def initialize_model_parallel(
    tensor_model_parallel_size: int = 1,
    pipeline_model_parallel_size: int = 1,
    backend: Optional[str] = None,
) -> None:
    """
    NOTE: This method is a hack from the open-sourced version without
    asertion of world_size = tp * pp
    
    Initialize model parallel groups.

    Arguments:
        tensor_model_parallel_size: number of GPUs used for tensor model
            parallelism.
        pipeline_model_parallel_size: number of GPUs used for pipeline model
            parallelism.

    Let's say we have a total of 8 GPUs denoted by g0 ... g7 and we
    use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize
    the model pipeline. The present function will
    create 4 tensor model-parallel groups and 2 pipeline model-parallel groups:
        4 tensor model-parallel groups:
            [g0, g1], [g2, g3], [g4, g5], [g6, g7]
        2 pipeline model-parallel groups:
            [g0, g2, g4, g6], [g1, g3, g5, g7]
    Note that for efficiency, the caller should make sure adjacent ranks
    are on the same DGX box. For example if we are using 2 DGX-1 boxes
    with a total of 16 GPUs, rank 0 to 7 belong to the first box and
    ranks 8 to 15 belong to the second box.
    """
    # Get world size and rank. Ensure some consistencies.
    assert torch.distributed.is_initialized()
    world_size: int = torch.distributed.get_world_size()
    # get the backend of _DEVICE_WORLD_GROUP
    backend = backend or torch.distributed.get_backend()

    # NOTE(sgm) we don't assert world_size == tp * pp
    # DP is not managed by vllm but by the veRL WorkerGroup

    num_tensor_model_parallel_groups: int = (world_size // tensor_model_parallel_size)
    num_pipeline_model_parallel_groups: int = (world_size // pipeline_model_parallel_size)
    rank = torch.distributed.get_rank()

    # Build device mesh for TP
    if num_tensor_model_parallel_groups > 1:
        device_mesh = init_device_mesh("cuda", (num_tensor_model_parallel_groups, tensor_model_parallel_size),
                                       mesh_dim_names=("replicate", "tp_shard"))
    else:
        device_mesh = init_device_mesh("cuda", (tensor_model_parallel_size,), mesh_dim_names=["tp_shard"])
    shard_group = device_mesh.get_group(mesh_dim="tp_shard")

    # Build the tensor model-parallel groups.
    global _TP_DEVICE_GROUP, _TP_CPU_GROUP
    global _DEVICE_MESH
    assert _TP_DEVICE_GROUP is None, ("tensor model parallel group is already initialized")
    assert _DEVICE_MESH is None, ("device mesh in vllm is already initialized")

    _DEVICE_MESH = device_mesh
    # for i in range(num_tensor_model_parallel_groups):
    #     ranks = range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size)
    # group = torch.distributed.new_group(ranks, backend=backend)
    # cpu_group = torch.distributed.new_group(ranks, backend="gloo")
    # assert torch.distributed.get_process_group_ranks(shard_group) == torch.distributed.get_process_group_ranks(cpu_group)
    # ranks = torch.distributed.get_process_group_ranks(shard_group)
    # cpu_group = torch.distributed.new_group(ranks, backend="gloo") # TODO: this will hang
    # cpu_group = torch.distributed.new_group(, backend="gloo")
    # if rank == 0:
    #     print(f'rank: {rank}')
    #     print(f'ranks: {ranks}')
    #     print(f'torch.distributed.get_process_group_ranks(shard_group): {torch.distributed.get_process_group_ranks(shard_group)}')
    # if rank in ranks:
    _TP_DEVICE_GROUP = shard_group
    ps._TP_DEVICE_GROUP = _TP_DEVICE_GROUP
    # ps._TP_CPU_GROUP = cpu_group # TODO: will hang when used with device mesh

    # TODO: init using device mesh
    # Build the pipeline model-parallel groups.
    assert ps._PIPELINE_MODEL_PARALLEL_GROUP is None, ("pipeline model parallel group is already initialized")
    for i in range(num_pipeline_model_parallel_groups):
        ranks = range(i, world_size, num_pipeline_model_parallel_groups)
        group = torch.distributed.new_group(ranks, backend=backend)
        if rank in ranks:
            ps._PIPELINE_MODEL_PARALLEL_GROUP = group
            ps._PIPELINE_GLOBAL_RANKS = ranks

    NOTE: This method is a hack from the open-sourced version without
    asertion of world_size = tp * pp
    
    Initialize model parallel groups.

    Arguments:
        tensor_model_parallel_size: number of GPUs used for tensor model
            parallelism.
        pipeline_model_parallel_size: number of GPUs used for pipeline model
            parallelism.

    Let's say we have a total of 8 GPUs denoted by g0 ... g7 and we
    use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize
    the model pipeline. The present function will
    create 4 tensor model-parallel groups and 2 pipeline model-parallel groups:
        4 tensor model-parallel groups:
            [g0, g1], [g2, g3], [g4, g5], [g6, g7]
        2 pipeline model-parallel groups:
            [g0, g2, g4, g6], [g1, g3, g5, g7]
    Note that for efficiency, the caller should make sure adjacent ranks
    are on the same DGX box. For example if we are using 2 DGX-1 boxes
    with a total of 16 GPUs, rank 0 to 7 belong to the first box and
    ranks 8 to 15 belong to the second box.
    
verl/third_party/vllm/vllm_v_0_4_2/parallel_state.py