def make_supervised_data_module(tokenizer: transformers.PreTrainedTokenizer,
                                data_args,
                                tcs_loader) -> Dict:
    """Make dataset and collator for supervised fine-tuning."""
    datasets = []
    lengths = []
    ds_collections = json.loads(open(data_args.meta_path).read())
    for ds_name in ds_collections.keys():
        repeat_time = ds_collections[ds_name]['repeat_time']
        dataset = LazySupervisedDataset(
            meta=ds_collections[ds_name],
            tokenizer=tokenizer,
            tcs_loader=tcs_loader,
            data_args=data_args,
        )
        for i in range(repeat_time):
            datasets.append(dataset)
            lengths.append(math.sqrt(len(dataset)))
    total_length = sum(lengths)
    weights = [l / total_length for l in lengths]
    for idx, ds_name in enumerate(ds_collections.keys()):
        print(f"{ds_name}: {weights[idx]}")
    train_dataset = WeightedConcatDataset(datasets, weights)
    data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer)
    return dict(train_dataset=train_dataset,
                eval_dataset=None,
                data_collator=data_collator)
Make dataset and collator for supervised fine-tuning.
internvl_chat_llava/llava/train/train_custom.py