def _broadcast_fallback(self):
        """Sum gradients across devices using TensorFlow collective ops (slow fallback path)."""
        from tensorflow.python.ops import collective_ops # pylint: disable=no-name-in-module
        global _collective_ops_warning_printed, _collective_ops_group_key, _collective_ops_instance_key
        if all(x.shape.num_elements() == 0 for device in self._devices.values() for x in device.grad_clean.values()):
            return
        if not _collective_ops_warning_printed:
            print("------------------------------------------------------------------------")
            print("WARNING: Using slow fallback implementation for inter-GPU communication.")
            print("Please use TensorFlow 1.14 on Linux for optimal training performance.")
            print("------------------------------------------------------------------------")
            _collective_ops_warning_printed = True
        for device in self._devices.values():
            with tf.device(device.name):
                combo = [tf.reshape(x, [x.shape.num_elements()]) for x in device.grad_clean.values()]
                combo = tf.concat(combo, axis=0)
                combo = collective_ops.all_reduce(combo, merge_op='Add', final_op='Id',
                    group_size=len(self._devices), group_key=_collective_ops_group_key,
                    instance_key=_collective_ops_instance_key)
                cur_ofs = 0
                for var, grad_old in device.grad_clean.items():
                    grad_new = tf.reshape(combo[cur_ofs : cur_ofs + grad_old.shape.num_elements()], grad_old.shape)
                    cur_ofs += grad_old.shape.num_elements()
                    device.grad_clean[var] = grad_new
        _collective_ops_instance_key += 1
Sum gradients across devices using TensorFlow collective ops (slow fallback path).
converters/stylegan2ada_tf_official/dnnlib/tflib/optimizer.py