def get_model_parallel_src_rank():
    """Calculate the global rank corresponding to a local rank zeor
    in the model parallel group."""
    global_rank = torch.distributed.get_rank()
    local_world_size = get_model_parallel_world_size()
    return (global_rank // local_world_size) * local_world_size
Calculate the global rank corresponding to a local rank zeor
    in the model parallel group.
megatron_lm/megatron/mpu/initialize.py