def test_tp_train_with_moes(tp_degree: int, tp_strategy: str):
    """Test that tensor parallelism is not compatible with MoEs."""
    # Make `cfg` for MoE model, fsdp, and tp
    moe_yaml_path: str = 'scripts/train/yamls/pretrain/testing-moe.yaml'
    dataset_name = Path('')  # dummy dataset path
    train_cfg = get_cfg(dataset_name, tp_strategy, tp_degree, moe_yaml_path)

    # Expect an error
    with pytest.raises(
        ValueError,
        match='Tensor Parallelism is not currently supported for MoE models.',
    ):
        process_init_device(
            train_cfg.model,
            train_cfg.fsdp_config,
            train_cfg.tp_config,
        )
Test that tensor parallelism is not compatible with MoEs.
tests/tp/test_tp_strategies.py