def convert_dataset_hf_from_args(
    dataset: str,
    data_subset: Optional[str],
    splits: list[str],
    out_root: str,
    compression: Optional[str],
    concat_tokens: Optional[int],
    tokenizer: Optional[str],
    tokenizer_kwargs: Optional[str],
    bos_text: Optional[str],
    eos_text: Optional[str],
    no_wrap: bool,
    num_workers: Optional[int],
) -> None:
    """A wrapper for `convert_dataset_hf` that parses arguments.

    Args:
        dataset (str): Name of the dataset
        data_subset (Optional[str]): Subset of the dataset (e.g., "all" or "en")
        splits (list[str]): Comma-separated list of dataset splits
        out_root (str): Output root directory
        compression (Optional[str]): Compression type
        concat_tokens (Optional[int]): Concatenate tokens up to this many tokens
        tokenizer (Optional[str]): Tokenizer name
        tokenizer_kwargs (Optional[str]): Tokenizer keyword arguments in JSON format
        bos_text (Optional[str]): BOS text
        eos_text (Optional[str]): EOS text
        no_wrap (bool): Do not wrap text across max_length boundaries
        num_workers (Optional[int]): Number of workers

    Raises:
        ValueError: If the output directory already contains the requested splits
        ValueError: If `concat_tokens` is set but `tokenizer` is not
    """
    os.environ['WORLD_SIZE'] = '1'
    if tokenizer_kwargs:
        parsed_tokenizer_kwargs = json.loads(tokenizer_kwargs)
    else:
        parsed_tokenizer_kwargs = {}

    if os.path.isdir(out_root) and len(
        set(os.listdir(out_root)).intersection(set(splits)),
    ) > 0:
        raise ValueError(
            f'--out_root={out_root} contains {os.listdir(out_root)} which cannot overlap with the requested splits {splits}.',
        )

    # Make sure we have needed concat options
    if (
        concat_tokens is not None and isinstance(concat_tokens, int) and
        tokenizer is None
    ):
        raise ValueError(
            'When setting --concat_tokens, you must specify a --tokenizer',
        )

    # now that we have validated them, change BOS/EOS to strings and convert
    convert_dataset_hf(
        dataset=dataset,
        data_subset=data_subset,
        splits=splits,
        out_root=out_root,
        compression=compression,
        concat_tokens=concat_tokens,
        tokenizer=tokenizer,
        tokenizer_kwargs=parsed_tokenizer_kwargs,
        bos_text=bos_text if bos_text else '',
        eos_text=eos_text if eos_text else '',
        no_wrap=no_wrap,
        num_workers=num_workers,
    )
A wrapper for `convert_dataset_hf` that parses arguments.

    Args:
        dataset (str): Name of the dataset
        data_subset (Optional[str]): Subset of the dataset (e.g., "all" or "en")
        splits (list[str]): Comma-separated list of dataset splits
        out_root (str): Output root directory
        compression (Optional[str]): Compression type
        concat_tokens (Optional[int]): Concatenate tokens up to this many tokens
        tokenizer (Optional[str]): Tokenizer name
        tokenizer_kwargs (Optional[str]): Tokenizer keyword arguments in JSON format
        bos_text (Optional[str]): BOS text
        eos_text (Optional[str]): EOS text
        no_wrap (bool): Do not wrap text across max_length boundaries
        num_workers (Optional[int]): Number of workers

    Raises:
        ValueError: If the output directory already contains the requested splits
        ValueError: If `concat_tokens` is set but `tokenizer` is not
    
llmfoundry/command_utils/data_prep/convert_dataset_hf.py