def allreduce_grads_naive(all_grads, devices=None, average=True):
    """
    AllReduce the gradients with raw ops (instead of collective ops).

    Args:
        all_grads (K x N): A list of K lists. Each of the list is a list of N grad tuples.
            The variables have to be the same across the K lists.
        devices (list[str]): assign the averaging to these device in
            round-robin. Cannot be used together with ``colocation``.
        average (bool): do average or sum

    Returns:
        list[Tensor]: list of grads where each grad is averaged or summed over K.
    """
    if devices is not None:
        assert isinstance(devices, list), devices
        # device_setter = LeastLoadedDeviceSetter(None, devices)

    nr_tower = len(all_grads)
    if nr_tower == 1:
        return all_grads[0]

    def aggregate(grads):
        if average:
            return tf.multiply(tf.add_n(grads), 1.0 / nr_tower)
        else:
            return tf.add_n(grads)

    grads_ret = []  # N(rev) grads
    # reverse so the device placement makes the last part of model more balance?
    all_grads_rev = [x[::-1] for x in all_grads]   # K x N(rev)

    for idx, grads in enumerate(zip(*all_grads_rev)):
        # grads: K tensors
        if devices is None:
            grad = aggregate(grads)
        else:
            # dev = device_setter.place_with_balance(v.op)
            dev = devices[idx % len(devices)]
            with tf.device(dev):
                grad = aggregate(grads)
        grads_ret.append(grad)
    grads_ret = grads_ret[::-1]
    return grads_ret

    AllReduce the gradients with raw ops (instead of collective ops).

    Args:
        all_grads (K x N): A list of K lists. Each of the list is a list of N grad tuples.
            The variables have to be the same across the K lists.
        devices (list[str]): assign the averaging to these device in
            round-robin. Cannot be used together with ``colocation``.
        average (bool): do average or sum

    Returns:
        list[Tensor]: list of grads where each grad is averaged or summed over K.
    
tensorpack/graph_builder/utils.py