def convert_dataset_hf(
    dataset: str,
    data_subset: Optional[str],
    splits: list[str],
    out_root: str,
    compression: Optional[str],
    concat_tokens: Optional[int],
    tokenizer: Optional[str],
    tokenizer_kwargs: dict[str, Any],
    bos_text: str,
    eos_text: str,
    no_wrap: bool,
    num_workers: Optional[int],
) -> None:
    """Converts HuggingFace datasets to MDS format.

    Args:
        dataset (str): Name of the dataset
        data_subset (Optional[str]): Subset of the dataset (e.g., "all" or "en")
        splits (list[str]): Comma-separated list of dataset splits
        out_root (str): Output root directory
        compression (Optional[str]): Compression type
        concat_tokens (Optional[int]): Concatenate tokens up to this many tokens
        tokenizer (Optional[str]): Tokenizer name
        tokenizer_kwargs (dict[str, Any]): Tokenizer keyword arguments
        bos_text (str): BOS text
        eos_text (str): EOS text
        no_wrap (bool): Do not wrap text across max_length boundaries
        num_workers (Optional[int]): Number of workers

    Raises:
        KeyError: If constants are not defined for the split
    """
    try:
        dataset_constants = CONSTS[dataset]
    except KeyError:
        raise ValueError(
            f'Constants for dataset "{dataset}" not found. Currently only "the_pile" and "allenai/c4" are supported.',
        )

    if concat_tokens is not None and tokenizer is not None:
        mode = ConcatMode.CONCAT_TOKENS
        built_tokenizer = build_tokenizer(tokenizer, tokenizer_kwargs)
        # we will enforce length, so suppress warnings about sequences too long for the model
        built_tokenizer.model_max_length = int(1e30)
        columns = {'tokens': 'ndarray:int32'}
    else:
        mode = ConcatMode.NO_CONCAT
        built_tokenizer = None
        columns = {'text': 'str'}

    for split_name in splits:
        try:
            split = dataset_constants.splits[split_name]
        except KeyError:
            raise KeyError(f'Constants not defined for split {split_name}.')
        hf_split = split.hf_split
        folder_split = split.folder_split
        expected_num_samples = split.raw_samples
        truncate_num_samples = split.truncated_samples
        # Only generate the splits requested
        if folder_split not in splits:
            continue

        # Get samples
        hf_dataset = build_hf_dataset(
            dataset_name=dataset,
            data_subset=data_subset,
            split=hf_split,
            mode=mode,
            max_length=concat_tokens,
            bos_text=bos_text,
            eos_text=eos_text,
            no_wrap=no_wrap,
            tokenizer=built_tokenizer,
        )
        loader = build_dataloader(
            dataset=hf_dataset,
            batch_size=512,
            num_workers=num_workers,
        )
        samples = generate_samples(
            loader,
            truncate_num_samples=truncate_num_samples,
        )

        if expected_num_samples is not None and concat_tokens is not None:
            denominator = truncate_num_samples if truncate_num_samples is not None else _est_progress_denominator(
                total_samples=expected_num_samples,
                chars_per_sample=dataset_constants.chars_per_sample,
                chars_per_token=dataset_constants.chars_per_token,
                mode=mode,
                max_length=concat_tokens,
            )
        else:
            denominator = None

        # Write samples
        print(f'Converting {folder_split} to MDS format...')
        print(
            f'Note: the progress bar is based on the dataset length before tokenization, and may finish at a value before 100%.',
        )
        with MDSWriter(
            columns=columns,
            out=os.path.join(out_root, folder_split),
            compression=compression,
        ) as out:
            if denominator is not None:
                for sample in tqdm(
                    samples,
                    desc=folder_split,
                    total=denominator,
                ):
                    out.write(sample)
            else:
                for sample in tqdm(samples, desc=folder_split):
                    out.write(sample)
Converts HuggingFace datasets to MDS format.

    Args:
        dataset (str): Name of the dataset
        data_subset (Optional[str]): Subset of the dataset (e.g., "all" or "en")
        splits (list[str]): Comma-separated list of dataset splits
        out_root (str): Output root directory
        compression (Optional[str]): Compression type
        concat_tokens (Optional[int]): Concatenate tokens up to this many tokens
        tokenizer (Optional[str]): Tokenizer name
        tokenizer_kwargs (dict[str, Any]): Tokenizer keyword arguments
        bos_text (str): BOS text
        eos_text (str): EOS text
        no_wrap (bool): Do not wrap text across max_length boundaries
        num_workers (Optional[int]): Number of workers

    Raises:
        KeyError: If constants are not defined for the split
    
llmfoundry/command_utils/data_prep/convert_dataset_hf.py