def to_nltk_tokens(
    df,
    sentence_cols=["sentence1", "sentence2"],
    token_cols=["sentence1_tokens", "sentence2_tokens"],
):
    """
    This function converts a sentence to word tokens using nltk.

    Args:
        df (pd.DataFrame): Dataframe with columns sentence_cols to tokenize.
        sentence_cols (list, optional): Column names for the raw sentences.
        token_cols (list, optional): Column names for the tokenized sentences.

    Returns:
    pd.DataFrame: Dataframe with new columns token_cols, each containing a
    list of tokens for their respective sentences.
    """
    text_df = df[sentence_cols]
    tok_df = text_df.applymap(lambda sentence: nltk.word_tokenize(sentence))
    tok_df.columns = token_cols
    tokenized = pd.concat([df, tok_df], axis=1)
    return tokenized

    This function converts a sentence to word tokens using nltk.

    Args:
        df (pd.DataFrame): Dataframe with columns sentence_cols to tokenize.
        sentence_cols (list, optional): Column names for the raw sentences.
        token_cols (list, optional): Column names for the tokenized sentences.

    Returns:
    pd.DataFrame: Dataframe with new columns token_cols, each containing a
    list of tokens for their respective sentences.
    
utils_nlp/dataset/preprocess.py