def _preprocess(split_map, data_path, column_names):
    """
    Method to save the tokens for each split in a snli_1.0_{split}.txt.clean file,
    with the sentence pairs and scores tab-separated and the tokens separated by a
    single space.

    Args:
        split_map(dict) : A dictionary containing train, test and dev
        tokenized dataframes.
        data_path(str): Path to the data folder.
        column_names(list): List of column names for the new columns created.

    """

    for file_split, df in split_map.items():
        base_txt_path = os.path.join(
            data_path, SNLI_CLEAN_PATH, "snli_1.0_{}.txt".format(file_split)
        )

        df[column_names[0]] = df["sentence1_tokens"].apply(
            lambda x: " " "".join(x)
        )
        df[column_names[1]] = df["sentence2_tokens"].apply(
            lambda x: " " "".join(x)
        )
        df[column_names[0]].to_csv(
            "{}.s1.tok".format(base_txt_path),
            sep=" ",
            header=False,
            index=False,
        )
        df[column_names[1]].to_csv(
            "{}.s2.tok".format(base_txt_path),
            sep=" ",
            header=False,
            index=False,
        )
        df[column_names[2]].to_csv(
            "{}.lab".format(base_txt_path), sep=" ", header=False, index=False
        )
        df_clean = df[column_names]
        df_clean.to_csv(
            "{}.clean".format(base_txt_path),
            sep="\t",
            header=False,
            index=False,
        )
        # remove rows with blank scores
        df_noblank = df_clean.loc[df_clean[column_names[2]] != "-"].copy()
        df_noblank.to_csv(
            "{}.clean.noblank".format(base_txt_path),
            sep="\t",
            header=False,
            index=False,
        )

    Method to save the tokens for each split in a snli_1.0_{split}.txt.clean file,
    with the sentence pairs and scores tab-separated and the tokens separated by a
    single space.

    Args:
        split_map(dict) : A dictionary containing train, test and dev
        tokenized dataframes.
        data_path(str): Path to the data folder.
        column_names(list): List of column names for the new columns created.

    
utils_nlp/models/gensen/preprocess_utils.py