def create_c4_dataset_xxsmall(
    path: Path,
    request: pytest.FixtureRequest,
) -> str:
    """Creates a small mocked version of the C4 dataset."""
    tiny_text_hf_dataset = request.getfixturevalue('tiny_text_hf_dataset')

    with patch('datasets.load_dataset') as mock_load_dataset:
        mock_load_dataset.return_value = tiny_text_hf_dataset

        c4_dir = os.path.join(path, f'my-copy-c4')
        downloaded_split = 'val_xxsmall'  # very fast to convert

        # Hyperparameters from https://github.com/mosaicml/llm-foundry/blob/340a56658560ebceb2a3aa69d6e37813e415acd0/README.md#L188
        convert_dataset_hf(
            dataset='allenai/c4',
            data_subset='en',
            splits=[downloaded_split],
            out_root=c4_dir,
            compression=None,
            concat_tokens=128,
            tokenizer='EleutherAI/gpt-neox-20b',
            tokenizer_kwargs={},
            bos_text='',
            eos_text='<|endoftext|>',
            no_wrap=False,
            num_workers=8,
        )

    # copy the small downloaded_split to other c4 splits for mocking purposes
    mocked_splits = ['train', 'val']
    for mocked_split in mocked_splits:
        shutil.copytree(
            os.path.join(c4_dir, 'val_xxsmall'),
            os.path.join(c4_dir, mocked_split),
            dirs_exist_ok=True,
        )
    assert os.path.exists(c4_dir)
    return c4_dir
Creates a small mocked version of the C4 dataset.
tests/data_utils.py