def build_from_hf(
        self,
        dataset_name: str,
        split: str,
        safe_load: bool = False,
        max_seq_len: int = 2048,
        mapping_fn: Callable = tokenize_formatted_example,
        preprocessing_fn: Optional[Callable[[dict[str, Any]], Example]] = None,
        tokenizer: Optional[PreTrainedTokenizerBase] = None,
        target_prompts: str = DEFAULT_TARGET_PROMPTS,
        target_responses: str = DEFAULT_TARGET_RESPONSES,
        decoder_only_format: bool = True,
        hf_kwargs: Optional[dict[str, Any]] = None,
    ) -> Union[hf_datasets.DatasetDict, hf_datasets.Dataset,
               hf_datasets.IterableDatasetDict, hf_datasets.IterableDataset]:
        """Load a HuggingFace Datasets, preprocess, and tokenize.

        Note: This function will drop examples where the prompt is longer than the max_seq_len

        Args:
            dataset_name (str): The name of the HuggingFace dataset
                to use. Can also be a remote http(s) directory or object store bucket
                containing the file {split}.jsonl in the format (prompt, response),
                in which case the builder will create a HuggingFace dataset.
            split (str): The split of the HuggingFace dataset.
            safe_load (bool, optional): Whether to enforce safe loading of the dataset.
                If `None`, will default to not applying any safe loading.
            max_seq_len (int): The maximum length of sequences
                in the batch. See :class:`Seq2SeqFinetuningCollator` docstring
                for details.
            mapping_fn (Callable): The mapping function to use for mapping the data
                examples.
            preprocessing_fn (Callable, optional): The preprocessing function to use for
                formatting the data examples.
            tokenizer (PreTrainedTokenizerBase): The tokenizer to be used for tokenizing
                the HuggingFace dataset.
            target_prompts (str): Which prompts are used as training targets.
                Defaults to "none", meaning prompts are never used as training targets.
                See :class:`Seq2SeqFinetuningCollator` docstring for details.
            target_responses (str): Which responses are used as training targets.
                Defaults to "last", meaning only the final response in multi-turn examples
                will serve as training targets. See :class:`Seq2SeqFinetuningCollator` docstring for
                details.
            decoder_only_format (bool): Whether to format the
                examples for a decoder-only model. See :class:`Seq2SeqFinetuningCollator`
                docstring for details.
            hf_kwargs (DictConfig, optional): Additional kwargs to
                pass to `datasets.load_dataset`, which can be used to load
                a dataset from local files.

        Returns:
            Dataset: The tokenized dataset.
        """
        if hf_kwargs is None:
            hf_kwargs = {}

        # None is checked in the function, because argument defaults were added after the function was written and we want
        # to preserve the ordering of the arguments for backwards compatibility.
        if tokenizer is None:
            raise ValueError('A tokenizer must be provided.')

        signal_file_path = dist.get_node_signal_file_name()

        download_folder = dist_mkdtemp()

        # Non local rank 0 ranks will wait here for local rank 0 to finish the data processing.
        # Once local rank 0 is done, the datasets are all cached on disk, and all other ranks
        # can just read them.
        if dist.get_local_rank() != 0:
            log.debug('Waiting for local_rank 0 to finish data prep')
            with dist.local_rank_zero_download_and_wait(signal_file_path):
                pass

        hf_tokenization_logger = logging.getLogger(
            'transformers.tokenization_utils_base',
        )
        sequence_length_warning_filter = SpecificWarningFilter(
            'Token indices sequence length is longer than the specified maximum sequence length',
        )

        # We will trim examples later in the collate_fn, so we want to silence this warning from Hugging Face
        hf_tokenization_logger.addFilter(sequence_length_warning_filter)

        error: Optional[Exception] = None
        filtered_dataset = None
        try:
            if safe_load:
                if not os.path.isdir(dataset_name):
                    # dataset_name is not a local dir path, download if needed.
                    local_dataset_dir = os.path.join(
                        download_folder,
                        dataset_name,
                    )

                    log.debug(
                        f'Downloading dataset {dataset_name} to {local_dataset_dir}.',
                    )

                    if _is_empty_or_nonexistent(dirpath=local_dataset_dir):
                        # Safely load a dataset from HF Hub with restricted file types.
                        hf_hub.snapshot_download(
                            dataset_name,
                            repo_type='dataset',
                            allow_patterns=[
                                '*' + ext for ext in SUPPORTED_EXTENSIONS
                            ],
                            token=hf_kwargs.get('token', None),
                            revision=hf_kwargs.get('revision', None),
                            local_dir_use_symlinks=False,
                            local_dir=local_dataset_dir,
                        )
                        if _is_empty_or_nonexistent(dirpath=local_dataset_dir):
                            raise InvalidFileExtensionError(
                                dataset_name,
                                SUPPORTED_EXTENSIONS,
                            )
                    # Set dataset_name to the downloaded location.
                    dataset_name = local_dataset_dir

                # dataset_name is a local dir path. Use the abspath to prevent confusion.
                dataset_name = os.path.abspath(dataset_name)

                # Ensure that the local dir contains only allowed file types.
                dataset_files = [
                    f for _, _, files in os.walk(dataset_name) for f in files
                ]
                if not all(
                    Path(f).suffix in SUPPORTED_EXTENSIONS +
                    HUGGINGFACE_FOLDER_EXTENSIONS or f == '.gitignore'
                    for f in dataset_files
                ):
                    raise InvalidFileExtensionError(
                        dataset_name,
                        SUPPORTED_EXTENSIONS,
                    )

            dataset = hf_datasets.load_dataset(
                dataset_name,
                split=split,
                **hf_kwargs,
            )

            def dataset_mapper(example: dict):
                if preprocessing_fn is not None:
                    return mapping_fn(
                        preprocessing_fn(example),
                        tokenizer,
                    )
                return mapping_fn(example, tokenizer)

            num_proc = _get_num_processes()
            if len(dataset) < num_proc:
                num_proc = 1

            columns_to_remove = list(dataset[0].keys())
            tokenized_dataset = dataset.map(
                dataset_mapper,
                batched=False,
                remove_columns=columns_to_remove,
                num_proc=num_proc,
                desc='Tokenizing dataset',
            )

            filtered_dataset = tokenized_dataset.filter(
                partial(
                    is_valid_ift_example,
                    max_seq_len,
                    target_prompts,
                    target_responses,
                    decoder_only_format,
                ),
                num_proc=num_proc,
                desc='Filtering out long prompts',
            )

            examples_removed = len(tokenized_dataset) - len(filtered_dataset)
            if examples_removed > 0:
                warnings.warn(
                    f'Dropped {examples_removed} examples where the prompt was longer than {max_seq_len}, '
                    +
                    'the prompt or response was empty, or the response was all padding tokens.',
                )
            if len(filtered_dataset) == 0:
                raise InvalidDatasetError(
                    f'No valid examples found after filtering out prompts longer than {max_seq_len}, '
                    +
                    'examples with empty prompts or responses, and examples with responses that are all padding tokens.',
                )
        except Exception as e:
            error = e
        # Now local rank 0 indicates to the other ranks that it is done
        if dist.get_local_rank() == 0:
            log.debug('Local rank 0 finished data prep')
            with open(signal_file_path, 'wb') as f:
                f.write(b'local_rank0_completed_data_prep')

        # All ranks sync up at this barrier, having completed data processing
        dist.barrier()

        # Last, local rank 0 cleans up the signal file
        if dist.get_local_rank() == 0:
            os.remove(signal_file_path)

        if isinstance(error, hf_exceptions.DatasetGenerationError):
            log.error('Huggingface DatasetGenerationError during data prep.')
            raise MisconfiguredHfDatasetError(
                dataset_name=dataset_name,
                split=split,
            ) from error
        if error is not None:
            log.error('Error during data prep')
            raise error
        log.debug('All ranks finished data prep')

        hf_tokenization_logger.removeFilter(sequence_length_warning_filter)

        assert filtered_dataset is not None
        return filtered_dataset
Load a HuggingFace Datasets, preprocess, and tokenize.

        Note: This function will drop examples where the prompt is longer than the max_seq_len

        Args:
            dataset_name (str): The name of the HuggingFace dataset
                to use. Can also be a remote http(s) directory or object store bucket
                containing the file {split}.jsonl in the format (prompt, response),
                in which case the builder will create a HuggingFace dataset.
            split (str): The split of the HuggingFace dataset.
            safe_load (bool, optional): Whether to enforce safe loading of the dataset.
                If `None`, will default to not applying any safe loading.
            max_seq_len (int): The maximum length of sequences
                in the batch. See :class:`Seq2SeqFinetuningCollator` docstring
                for details.
            mapping_fn (Callable): The mapping function to use for mapping the data
                examples.
            preprocessing_fn (Callable, optional): The preprocessing function to use for
                formatting the data examples.
            tokenizer (PreTrainedTokenizerBase): The tokenizer to be used for tokenizing
                the HuggingFace dataset.
            target_prompts (str): Which prompts are used as training targets.
                Defaults to "none", meaning prompts are never used as training targets.
                See :class:`Seq2SeqFinetuningCollator` docstring for details.
            target_responses (str): Which responses are used as training targets.
                Defaults to "last", meaning only the final response in multi-turn examples
                will serve as training targets. See :class:`Seq2SeqFinetuningCollator` docstring for
                details.
            decoder_only_format (bool): Whether to format the
                examples for a decoder-only model. See :class:`Seq2SeqFinetuningCollator`
                docstring for details.
            hf_kwargs (DictConfig, optional): Additional kwargs to
                pass to `datasets.load_dataset`, which can be used to load
                a dataset from local files.

        Returns:
            Dataset: The tokenized dataset.
        
llmfoundry/data/finetuning/tasks.py