def huggingface_pre_process(
    dataset, model_name, input_cols, label_col="label", max_samples=None, trust_remote_code=None, **kwargs
):
    """Pre-process data.

    Args:
        dataset (object): Data to be pre-processed, reserved for internal dataset assignment.
        model_name (str): Name of the huggingface model.
        input_cols (list): List of input columns.
        label_col (str): Label column. Defaults to "label".
        max_samples (int, optional): Max number of samples to use. Defaults to None.
        trust_remote_code (bool, optional): Whether or not to allow for custom models defined on the Hub in their own
            modeling files. Defaults to None.
        **kwargs: Additional arguments.

    Returns:
        object: Pre-processed data.

    """

    def _tokenizer_and_align_labels(examples):
        tokenizer = get_tokenizer(model_name, trust_remote_code=trust_remote_code)
        tokenized_inputs = tokenizer(
            *[examples[input_col] for input_col in input_cols if examples[input_col]],
            padding=kwargs.get("padding", True),
            truncation=kwargs.get("truncation", True),
            max_length=kwargs.get("max_length"),
            is_split_into_words=kwargs.get("is_split_into_words", False),
            add_special_tokens=kwargs.get("add_special_tokens", True),
        )
        tokenized_inputs["label"] = examples[label_col]
        # huggingface dataset api limit to return dict and arrow table
        return tokenized_inputs

    model_config_path = kwargs.pop("model_config_path", None)
    # TODO(trajep): add the complete data operation mapping like:
    # align_labels -> align_labels_with_mapping
    # Also to support customized operation arguments from users
    if kwargs.pop("align_labels", False):
        model_hf_config = get_model_config(model_config_path or model_name, trust_remote_code=trust_remote_code)
        if model_hf_config and model_hf_config.label2id:
            dataset = dataset.align_labels_with_mapping(model_hf_config.label2id, label_col)

    tokenized_datasets = _huggingface_pre_process_helper(dataset, _tokenizer_and_align_labels, max_samples, **kwargs)
    # label_col is "label" since we added label_col as "label" to tokenized_inputs
    return BaseDataset(tokenized_datasets, label_col="label", max_samples=max_samples)
Pre-process data.

    Args:
        dataset (object): Data to be pre-processed, reserved for internal dataset assignment.
        model_name (str): Name of the huggingface model.
        input_cols (list): List of input columns.
        label_col (str): Label column. Defaults to "label".
        max_samples (int, optional): Max number of samples to use. Defaults to None.
        trust_remote_code (bool, optional): Whether or not to allow for custom models defined on the Hub in their own
            modeling files. Defaults to None.
        **kwargs: Additional arguments.

    Returns:
        object: Pre-processed data.

    
olive/data/component/pre_process_data.py