def prepare_dataset(tokenizer, text_file_path: str):
    """
    Tokenizes a text file where each line is a different example.
    Padding is applied to each example.
    """
    # Each line is a different example
    dataset = load_dataset("text", data_files={"train": text_file_path})

    def tokenize_function(examples):
        return tokenizer(examples["text"], padding="max_length", truncation=True, max_length=128)

    tokenized_dataset = dataset.map(tokenize_function, batched=True, remove_columns=["text"])
    return tokenized_dataset["train"]

    Tokenizes a text file where each line is a different example.
    Padding is applied to each example.
    
test/resources/scripts/generate_tokenized_dataset.py