def download_and_convert(
    file_names: list[str],
    output_folder: str,
    input_folder: str,
    tokenizer_name: str,
    concat_tokens: int,
    eos_text: str,
    bos_text: str,
    no_wrap: bool,
    compression: str,
    trust_remote_code: bool,
):
    """Downloads and converts text files to MDS format.

    Args:
        file_names (List[str]): Files to process
        output_folder (str): Folder to write MDS shards to
        input_folder (str): Folder of text files to process
        tokenizer_name (str): Name of tokenizer to use
        concat_tokens (int): Concatenate up to this many tokens
        eos_text (str): Text to append to each example to separate concatenated samples
        bos_text (str): Text to prepend to each example to separate concatenated samples
        no_wrap: (bool): Whether to let text examples wrap across multiple training examples
        compression (str): The compression algorithm to use for MDS writing
        trust_remote_code (bool): If true, allows custom code to be executed to load the tokenizer
    """
    log.info(f'Starting download and conversion for {len(file_names)} files')

    object_store = maybe_create_object_store_from_uri(input_folder)

    # Download file_names
    downloading_iter = DownloadingIterable(
        object_names=file_names,
        output_folder=None, # Downloads to temporary files.
        object_store=object_store,
    )
    log.info(f'Initializing tokenizer: {tokenizer_name}')
    tokenizer = AutoTokenizer.from_pretrained(
        tokenizer_name,
        trust_remote_code=trust_remote_code,
    )
    tokenizer.model_max_length = 5000000000  # Hack to prevent warnings from HuggingFace

    # Use the ConcatTokensDataset from LLM-foundry to concatenate sequences of tokens up
    # to the maximum sequence length
    dataset = ConcatTokensFromFilesDataset(
        files=downloading_iter,
        max_length=concat_tokens,
        tokenizer=tokenizer,
        eos_text=eos_text,
        bos_text=bos_text,
        no_wrap=no_wrap,
    )

    columns = {'tokens': 'ndarray:int32'}

    log.info('Converting to MDS format...')
    with MDSWriter(
        out=output_folder,
        columns=columns,
        compression=compression,
    ) as out:
        for sample in tqdm(dataset):
            out.write(sample)

    log.info(f'Completed download and conversion for {len(file_names)} files')
Downloads and converts text files to MDS format.

    Args:
        file_names (List[str]): Files to process
        output_folder (str): Folder to write MDS shards to
        input_folder (str): Folder of text files to process
        tokenizer_name (str): Name of tokenizer to use
        concat_tokens (int): Concatenate up to this many tokens
        eos_text (str): Text to append to each example to separate concatenated samples
        bos_text (str): Text to prepend to each example to separate concatenated samples
        no_wrap: (bool): Whether to let text examples wrap across multiple training examples
        compression (str): The compression algorithm to use for MDS writing
        trust_remote_code (bool): If true, allows custom code to be executed to load the tokenizer
    
llmfoundry/command_utils/data_prep/convert_text_to_mds.py