def convert_retrieval_dataset(dataset, split, output_folder, *, transform=None, image_format='webp', max_count=10_000,
                              max_size=1_000_000_000, verbose=True):
    """
    Convert an iterable `dataset` of (image, [caption1, caption2, ...]) pairs to webdataset (.tar) format, and store in `output_folder/split`.

    Labels must be lists of strings, with no newlines.

    Read the documentation of `convert_dataset` for more information.
    """
    # Create output directory
    os.makedirs(os.path.join(output_folder, split), exist_ok=True)
    # Multiprocessed dataloader, should work with Dataset or list
    dataloader = torch.utils.data.DataLoader(
        dataset,
        batch_size=1,
        num_workers=8,
        collate_fn=lambda batch: batch[0]  # No collate, only for multiprocessing
    )
    if verbose:
        try:
            print(f'Dataset size: {len(dataset)}')
        except TypeError:
            print('IterableDataset has no len()')
    # No classnames
    # No zeroshot templates
    # Save dataset type
    type_fname = os.path.join(output_folder, 'dataset_type.txt')
    with open(type_fname, 'w') as type_file:
        print('retrieval', end='\n', file=type_file)
    if verbose:
        print("Saved dataset type to '%s'" % type_fname)
    # Write to TAR files
    data_fname = os.path.join(output_folder, split, r'%d.tar')
    sink = webdataset.ShardWriter(
        data_fname,
        maxcount=max_count,
        maxsize=max_size
    )
    nsamples = 0
    for index, (input, output) in enumerate(tqdm(dataloader, desc='Converting')):
        nsamples += 1
        if isinstance(input, str) and transform is path_to_bytes:
            # If copying file, determine image format from extension
            extension = os.path.splitext(input)[1].replace('.', '').lower().replace('jpeg', 'jpg') or image_format
        else:
            extension = image_format
        sink.write({
            '__key__': 's%07d' % index,
            extension: transform(input) if transform else input,
            'txt': '\n'.join(caption.replace('\n', r'\n') for caption in output),
        })
    num_shards = sink.shard
    sink.close()
    if verbose:
        print("Saved dataset to '%s'" % data_fname.replace(r'%d', '{0..%d}' % (num_shards - 1)))
    # Save number of shards
    nshards_fname = os.path.join(output_folder, split, 'nshards.txt')
    with open(nshards_fname, 'w') as nshards_file:
        print(num_shards, end='\n', file=nshards_file)
    if verbose:
        print("Saved number of shards = %d to '%s'" % (num_shards, nshards_fname))
    print('Final dataset size:', nsamples)

    Convert an iterable `dataset` of (image, [caption1, caption2, ...]) pairs to webdataset (.tar) format, and store in `output_folder/split`.

    Labels must be lists of strings, with no newlines.

    Read the documentation of `convert_dataset` for more information.
    
clip_benchmark/clip_benchmark/webdataset_builder.py