def convert_text_to_mds_from_args(
    output_folder: str,
    input_folder: str,
    compression: str,
    concat_tokens: int,
    tokenizer_name: str,
    bos_text: Optional[str],
    eos_text: Optional[str],
    use_tokenizer_eos: bool,
    no_wrap: bool,
    processes: int,
    reprocess: bool,
    trust_remote_code: bool,
    logging_level: str,
) -> None:
    """A wrapper for `convert_text_to_mds` to parse arguments.

    Args:
        output_folder (str): Folder to write MDS shards to
        input_folder (str): Folder of text files to process
        compression (str): The compression algorithm to use for MDS writing
        concat_tokens (int): Concatenate up to this many tokens
        tokenizer_name (str): The name of the tokenizer to use
        bos_text (Optional[str]): The text to prepend to each example to separate concatenated examples
        eos_text (Optional[str]): The text to append to each example to separate concatenated examples
        use_tokenizer_eos (bool): Use the EOS text from the tokenizer
        no_wrap (bool): Whether to let text examples wrap across multiple training examples
        processes (int): The number of processes to use to download and convert the dataset
        reprocess (bool): If true, reprocess the input_folder to MDS format. Otherwise, only reprocess upon changes to the input folder or dataset creation parameters.
        trust_remote_code (bool): If true, allows custom code to be executed to load the tokenizer
        logging_level (str): Logging level for the script. Default is INFO.

    Raises:
        ValueError: If `use_tokenizer_eos` is True and `eos_text` is not None
    """
    os.environ['WORLD_SIZE'] = '1'
    if use_tokenizer_eos:
        # Ensure that eos text is not specified twice.
        if eos_text is not None:
            ValueError(
                'Cannot set --eos_text with --use_tokenizer_eos. Please specify one.',
            )
        tokenizer = AutoTokenizer.from_pretrained(
            tokenizer_name,
            trust_remote_code=trust_remote_code,
        )
        eos_text = tokenizer.eos_token

    # now that we have validated them, change BOS/EOS to strings
    if bos_text is None:
        bos_text = ''
    if eos_text is None:
        eos_text = ''
    _configure_logging(logging_level)

    # Define args for _args_str
    args = {
        'tokenizer': tokenizer_name,
        'output_folder': output_folder,
        'input_folder': input_folder,
        'compression': compression,
        'concat_tokens': concat_tokens,
        'eos_text': eos_text,
        'bos_text': bos_text,
        'no_wrap': no_wrap,
        'processes': processes,
        'reprocess': reprocess,
        'trust_remote_code': trust_remote_code,
    }
    convert_text_to_mds(
        tokenizer_name=tokenizer_name,
        output_folder=output_folder,
        input_folder=input_folder,
        concat_tokens=concat_tokens,
        eos_text=eos_text,
        bos_text=bos_text,
        no_wrap=no_wrap,
        compression=compression,
        processes=processes,
        reprocess=reprocess,
        trust_remote_code=trust_remote_code,
        args_str=str(args),
    )
A wrapper for `convert_text_to_mds` to parse arguments.

    Args:
        output_folder (str): Folder to write MDS shards to
        input_folder (str): Folder of text files to process
        compression (str): The compression algorithm to use for MDS writing
        concat_tokens (int): Concatenate up to this many tokens
        tokenizer_name (str): The name of the tokenizer to use
        bos_text (Optional[str]): The text to prepend to each example to separate concatenated examples
        eos_text (Optional[str]): The text to append to each example to separate concatenated examples
        use_tokenizer_eos (bool): Use the EOS text from the tokenizer
        no_wrap (bool): Whether to let text examples wrap across multiple training examples
        processes (int): The number of processes to use to download and convert the dataset
        reprocess (bool): If true, reprocess the input_folder to MDS format. Otherwise, only reprocess upon changes to the input folder or dataset creation parameters.
        trust_remote_code (bool): If true, allows custom code to be executed to load the tokenizer
        logging_level (str): Logging level for the script. Default is INFO.

    Raises:
        ValueError: If `use_tokenizer_eos` is True and `eos_text` is not None
    
llmfoundry/command_utils/data_prep/convert_text_to_mds.py