def call_for_each_tower(self, tower_fn):
        """
        Call the function `tower_fn` under :class:`TowerContext` for each tower.

        Returns:
            a list, contains the return values of `tower_fn` on each tower.
        """
        raw_devices = ['/gpu:{}'.format(k) for k in self.towers]
        if self.ps_device == 'gpu' or get_tf_version_tuple() >= (2, 0):
            devices = [LeastLoadedDeviceSetter(d, raw_devices) for d in raw_devices]
        else:
            # TODO not working in TF2 - cause model to run on CPUs
            devices = [tf.train.replica_device_setter(
                worker_device=d, ps_device='/cpu:0', ps_tasks=1) for d in raw_devices]

        return DataParallelBuilder.build_on_towers(self.towers, tower_fn, devices)

        Call the function `tower_fn` under :class:`TowerContext` for each tower.

        Returns:
            a list, contains the return values of `tower_fn` on each tower.
        
tensorpack/graph_builder/training.py