def metric_average(value, name):
    """
    Sync the validation loss with nodes.
    :param value:
    :param name:
    :return:
    """
    tensor = torch.tensor(value)
    avg_tensor = hvd.allreduce(tensor, name=name)
    return avg_tensor.item()

    Sync the validation loss with nodes.
    :param value:
    :param name:
    :return:
    
examples/sentence_similarity/gensen_train.py