def concat_all_gather(tensor):
  """
  Performs all_gather operation on the provided tensors.
  *** Warning ***: torch.distributed.all_gather has no gradient.
  """
  if torch.cuda.device_count() > 1:
    return varsize_tensor_all_gather(tensor)
  else:
    output = tensor
  return output

  Performs all_gather operation on the provided tensors.
  *** Warning ***: torch.distributed.all_gather has no gradient.
  
infoxlm/src-infoxlm/infoxlm/utils.py