def to_spacy_tokens(
    df,
    sentence_cols=["sentence1", "sentence2"],
    token_cols=["sentence1_tokens", "sentence2_tokens"],
):
    """
    This function tokenizes the sentence pairs using spaCy, defaulting to the
    spaCy en_core_web_sm model

    Args:
        df (pd.DataFrame): Dataframe with columns sentence_cols to tokenize.
        sentence_cols (list, optional): Column names of the raw sentence pairs.
        token_cols (list, optional): Column names for the tokenized sentences.

    Returns:
        pd.DataFrame: Dataframe with new columns token_cols, each containing
                            a list of tokens for their respective sentences.
    """
    nlp = spacy.load("en_core_web_sm")
    text_df = df[sentence_cols]
    nlp_df = text_df.applymap(lambda x: nlp(x))
    tok_df = nlp_df.applymap(lambda doc: [token.text for token in doc])
    tok_df.columns = token_cols
    tokenized = pd.concat([df, tok_df], axis=1)
    return tokenized

    This function tokenizes the sentence pairs using spaCy, defaulting to the
    spaCy en_core_web_sm model

    Args:
        df (pd.DataFrame): Dataframe with columns sentence_cols to tokenize.
        sentence_cols (list, optional): Column names of the raw sentence pairs.
        token_cols (list, optional): Column names for the tokenized sentences.

    Returns:
        pd.DataFrame: Dataframe with new columns token_cols, each containing
                            a list of tokens for their respective sentences.
    
utils_nlp/dataset/preprocess.py