def convert_dataset_json_from_args(
    path: str,
    out_root: str,
    compression: Optional[str],
    concat_tokens: Optional[int],
    split: str,
    tokenizer: Optional[str] = None,
    bos_text: Optional[str] = None,
    eos_text: Optional[str] = None,
    no_wrap: bool = False,
    num_workers: Optional[int] = None,
) -> None:
    """A wrapper for `convert_dataset_json` that parses arguments.

    Args:
        path (str): Path to the input data file
        out_root (str): Output root directory
        compression (Optional[str]): Compression type, if any
        concat_tokens (Optional[int]): Convert text to tokens and concatenate up to this many tokens
        split (str): Dataset split to process
        tokenizer (Optional[str]): Tokenizer name
        bos_text (Optional[str]): Text to insert at the beginning of each sequence
        eos_text (Optional[str]): Text to insert at the end of each sequence
        no_wrap (bool): Do not wrap text across max_length boundaries
        num_workers (Optional[int]): Number of workers for data loading

    Raises:
        ValueError: If the out_root directory exists and contains files that overlap with the requested splits
        ValueError: If concat_tokens is set and a tokenizer is not provided
    """
    os.environ['WORLD_SIZE'] = '1'
    if os.path.isdir(out_root) and len(
        set(os.listdir(out_root)).intersection(set(split)),
    ) > 0:
        raise ValueError(
            f'--out_root={out_root} contains {os.listdir(out_root)} which cannot overlap with the requested splits {split}.',
        )

    # Make sure we have needed concat options
    if (
        concat_tokens is not None and isinstance(concat_tokens, int) and
        tokenizer is None
    ):
        ValueError(
            'When setting --concat_tokens, you must specify a --tokenizer',
        )

    # now that we have validated them, change BOS/EOS to strings
    if bos_text is None:
        bos_text = ''
    if eos_text is None:
        eos_text = ''

    convert_dataset_json(
        path=path,
        out_root=out_root,
        compression=compression,
        concat_tokens=concat_tokens,
        split=split,
        tokenizer=tokenizer,
        bos_text=bos_text,
        eos_text=eos_text,
        no_wrap=no_wrap,
        num_workers=num_workers,
    )
A wrapper for `convert_dataset_json` that parses arguments.

    Args:
        path (str): Path to the input data file
        out_root (str): Output root directory
        compression (Optional[str]): Compression type, if any
        concat_tokens (Optional[int]): Convert text to tokens and concatenate up to this many tokens
        split (str): Dataset split to process
        tokenizer (Optional[str]): Tokenizer name
        bos_text (Optional[str]): Text to insert at the beginning of each sequence
        eos_text (Optional[str]): Text to insert at the end of each sequence
        no_wrap (bool): Do not wrap text across max_length boundaries
        num_workers (Optional[int]): Number of workers for data loading

    Raises:
        ValueError: If the out_root directory exists and contains files that overlap with the requested splits
        ValueError: If concat_tokens is set and a tokenizer is not provided
    
llmfoundry/command_utils/data_prep/convert_dataset_json.py