def convert_dataset_json(
    path: str,
    out_root: str,
    compression: Optional[str],
    concat_tokens: Optional[int],
    split: str,
    tokenizer: Optional[str] = None,
    bos_text: str = '',
    eos_text: str = '',
    no_wrap: bool = False,
    num_workers: Optional[int] = None,
) -> None:
    """Create C4/pile streaming dataset.

    Args:
        path (str): Path to the input data file
        out_root (str): Output root directory
        compression (Optional[str]): Compression type, if any
        concat_tokens (Optional[int]): Convert text to tokens and concatenate up to this many tokens
        split (str): Dataset split to process
        tokenizer (Optional[str]): Tokenizer name
        bos_text (str): Text to insert at the beginning of each sequence
        eos_text (str): Text to insert at the end of each sequence
        no_wrap (bool): Do not wrap text across max_length boundaries
        num_workers (Optional[int]): Number of workers for data loading
    """
    if concat_tokens is not None:
        mode = ConcatMode.CONCAT_TOKENS
        assert tokenizer is not None
        built_tokenizer = AutoTokenizer.from_pretrained(tokenizer)
        # we will enforce length, so suppress warnings about sequences too long for the model
        built_tokenizer.model_max_length = int(1e30)
        columns = {'tokens': 'ndarray:int32'}
    else:
        mode = ConcatMode.NO_CONCAT
        built_tokenizer = None
        columns = {'text': 'str'}

    # Get samples
    dataset = build_hf_dataset(
        path=path,
        split=split,
        mode=mode,
        max_length=concat_tokens,
        bos_text=bos_text,
        eos_text=eos_text,
        no_wrap=no_wrap,
        tokenizer=built_tokenizer,
    )

    # Write samples
    print(f'Converting to MDS format...')
    print(
        f'Note that the progress bar is based on the dataset length before tokenization.',
    )
    print(f'It will finish at a value below 100% if tokenizing')
    with MDSWriter(
        columns=columns,
        out=os.path.join(out_root),
        compression=compression,
    ) as out:
        for sample in tqdm(dataset):
            out.write(sample)
Create C4/pile streaming dataset.

    Args:
        path (str): Path to the input data file
        out_root (str): Output root directory
        compression (Optional[str]): Compression type, if any
        concat_tokens (Optional[int]): Convert text to tokens and concatenate up to this many tokens
        split (str): Dataset split to process
        tokenizer (Optional[str]): Tokenizer name
        bos_text (str): Text to insert at the beginning of each sequence
        eos_text (str): Text to insert at the end of each sequence
        no_wrap (bool): Do not wrap text across max_length boundaries
        num_workers (Optional[int]): Number of workers for data loading
    
llmfoundry/command_utils/data_prep/convert_dataset_json.py