def test_train_multi_eval(
    tmp_path: pathlib.Path,
    request: pytest.FixtureRequest,
):
    """Test training run with multiple eval datasets."""
    c4_dataset_name = create_c4_dataset_xxsmall(tmp_path, request)
    test_cfg = gpt_tiny_cfg(c4_dataset_name, 'cpu')
    # Set up multiple eval dataloaders
    first_eval_loader = test_cfg.eval_loader
    first_eval_loader.label = 'allenai/c4'
    # Create second eval dataloader using the arxiv dataset.
    second_eval_loader = copy.deepcopy(first_eval_loader)
    second_eval_loader.label = 'arxiv'
    test_cfg.eval_loader = om.create([first_eval_loader, second_eval_loader])
    test_cfg.eval_subset_num_batches = 1  # -1 to evaluate on all batches

    test_cfg.max_duration = '1ba'
    test_cfg.eval_interval = '1ba'
    test_cfg.loggers = DictConfig({'inmemory': DictConfig({})})
    trainer = train(test_cfg)

    assert isinstance(trainer.logger.destinations, tuple)

    assert len(trainer.logger.destinations) > 0
    inmemorylogger = trainer.logger.destinations[
        0]  # pyright: ignore [reportGeneralTypeIssues]
    assert isinstance(inmemorylogger, InMemoryLogger)

    # Checks for first eval dataloader
    assert 'metrics/eval/allenai/c4/LanguageCrossEntropy' in inmemorylogger.data.keys(
    )
    assert isinstance(
        inmemorylogger.data['metrics/eval/allenai/c4/LanguageCrossEntropy'],
        list,
    )
    assert len(
        inmemorylogger.data['metrics/eval/allenai/c4/LanguageCrossEntropy'][-1],
    ) > 0
    assert isinstance(
        inmemorylogger.data['metrics/eval/allenai/c4/LanguageCrossEntropy'][-1],
        tuple,
    )

    # Checks for second eval dataloader
    assert 'metrics/eval/arxiv/LanguageCrossEntropy' in inmemorylogger.data.keys(
    )
    assert isinstance(
        inmemorylogger.data['metrics/eval/arxiv/LanguageCrossEntropy'],
        list,
    )
    assert len(
        inmemorylogger.data['metrics/eval/arxiv/LanguageCrossEntropy'][-1],
    ) > 0
    assert isinstance(
        inmemorylogger.data['metrics/eval/arxiv/LanguageCrossEntropy'][-1],
        tuple,
    )
Test training run with multiple eval datasets.
tests/a_scripts/train/test_train.py