def convert_dataset_json(
    path: Annotated[str, Option(..., help='Path to the input data file')],
    out_root: Annotated[str, Option(..., help='Output root directory')],
    concat_tokens: Annotated[
        int,
        Option(
            ...,
            help='Convert text to tokens and concatenate up to this many tokens',
        )],
    tokenizer: Annotated[str, Option(..., help='Tokenizer name')],
    compression: Annotated[Optional[str],
                           Option(help='Compression type, if any')] = 'zstd',
    split: Annotated[str, Option(help='Dataset split to process')] = 'train',
    bos_text: Annotated[
        Optional[str],
        Option(help='Text to insert at the beginning of each sequence')] = None,
    eos_text: Annotated[
        Optional[str],
        Option(help='Text to insert at the end of each sequence')] = None,
    no_wrap: Annotated[
        bool,
        Option(help='Do not wrap text across max_length boundaries')] = False,
    num_workers: Annotated[
        Optional[int],
        Option(help='Number of workers for data loading')] = None,
):
    """Convert a dataset from JSON to MDS streaming format."""
    convert_dataset_json_from_args(
        path=path,
        split=split,
        out_root=out_root,
        compression=compression,
        concat_tokens=concat_tokens,
        tokenizer=tokenizer,
        bos_text=bos_text,
        eos_text=eos_text,
        no_wrap=no_wrap,
        num_workers=num_workers,
    )
Convert a dataset from JSON to MDS streaming format.
llmfoundry/cli/data_prep_cli.py