def __init__(self, gpus, average=True, mode=None):
        """
        Args:
            gpus (int or [int]): list of GPU ids.
            average (bool): whether to average or sum gradients.
            mode (str or None): Gradient aggregation mode.
                Supported values: ['nccl', 'hierarchical', 'cpu', 'gpu'].
                These modes may differ in speed.
                Default to pick automatically by heuristics.
                "hierarchical" mode was designed for DGX-like 8-GPU machines.
        """
        self.devices = gpus
        if mode is not None:
            mode = mode.lower()

        # Heuristics about mode selection:
        if mode == 'hierarchical' and len(gpus) != 8:
            logger.warn("mode='hierarchical' requires 8 GPUs. Will fallback to default mode.")
            mode = None
        if mode is None:
            if len(gpus) == 8:
                mode = 'hierarchical'
            else:
                # https://github.com/tensorflow/tensorflow/issues/41539
                mode = 'nccl' if get_tf_version_tuple() < (1, 15) else 'gpu'
        if mode == 'cpu' and get_tf_version_tuple() >= (2, 0):
            # cpu mode causes the entire model to get located on cpu
            mode = 'gpu'
        if mode == 'nccl' and get_tf_version_tuple() >= (1, 15):
            logger.warning(
                "NCCL in TensorFlow has a serious bug that is likely to trigger in TF>=1.15. "
                "Try 'mode=None' to use a better default mode.")

        self._builder = SyncMultiGPUReplicatedBuilder(gpus, average, mode)
        self.BROADCAST_EVERY_EPOCH = True

        super(SyncMultiGPUTrainerReplicated, self).__init__()

        Args:
            gpus (int or [int]): list of GPU ids.
            average (bool): whether to average or sum gradients.
            mode (str or None): Gradient aggregation mode.
                Supported values: ['nccl', 'hierarchical', 'cpu', 'gpu'].
                These modes may differ in speed.
                Default to pick automatically by heuristics.
                "hierarchical" mode was designed for DGX-like 8-GPU machines.
        
tensorpack/train/trainers.py