def _huggingface_pre_process_helper(dataset, map_func, max_samples, **kwargs):
    """Apply a map function to the dataset.

    Args:
        dataset (object): Data to be pre-processed.
        map_func (function): Function to be applied to the dataset.
        max_samples (int): Max number of samples to use.
        **kwargs: Additional arguments.

    Returns:
        object: Pre-processed data.

    """
    if max_samples is not None:
        # select the data beforehand to avoid tokenizing the whole dataset
        dataset = dataset.select(range(min(len(dataset), max_samples)))

    # output type is list
    tokenized_datasets = dataset.map(
        map_func,
        batched=kwargs.get("batched", True),
        remove_columns=dataset.column_names,
    )
    tokenized_datasets.set_format("torch", output_all_columns=True)
    return tokenized_datasets
Apply a map function to the dataset.

    Args:
        dataset (object): Data to be pre-processed.
        map_func (function): Function to be applied to the dataset.
        max_samples (int): Max number of samples to use.
        **kwargs: Additional arguments.

    Returns:
        object: Pre-processed data.

    
olive/data/component/pre_process_data.py