def text_generation_huggingface_pre_process(
    dataset, model_name: str, trust_remote_code: Optional[bool] = None, **kwargs
):
    """Pre-process data for text generation task.

    Args:
        dataset (object): Data to be pre-processed, reserved for internal dataset assignment.
        model_name (str): Name of the huggingface model.
        trust_remote_code (bool, optional): Whether or not to allow for custom models defined on the Hub in their own
            modeling files. Defaults to None.
        **kwargs: Additional arguments.
            The common arguments are the fields in olive.data.component.text_generation.TextGenParams.

    """
    all_kwargs = deepcopy(kwargs)
    # task is not used in the pre-process function. Will pop it so that the config validation doesn't warn about
    # unused kwargs
    all_kwargs.pop("task", None)

    tokenizer = get_tokenizer(model_name, trust_remote_code=trust_remote_code)

    return text_gen_pre_process(dataset, tokenizer, all_kwargs)
Pre-process data for text generation task.

    Args:
        dataset (object): Data to be pre-processed, reserved for internal dataset assignment.
        model_name (str): Name of the huggingface model.
        trust_remote_code (bool, optional): Whether or not to allow for custom models defined on the Hub in their own
            modeling files. Defaults to None.
        **kwargs: Additional arguments.
            The common arguments are the fields in olive.data.component.text_generation.TextGenParams.

    
olive/data/component/pre_process_data.py