src/transformers/trainer_tf.py/TFTrainer/apply_gradients
class TFTrainer:    def apply_gradients(self, features, labels, nb_instances_in_global_batch):
        if self.args.gradient_accumulation_steps == 1:
            gradients = self.training_step(features, labels, nb_instances_in_global_batch)

            self.optimizer.apply_gradients(list(zip(gradients, self.model.trainable_variables)))
        else:
            for _ in tf.range(self.args.gradient_accumulation_steps):
                reduced_features = {
                    k: ft[: self.args.train_batch_size // self.args.n_replicas] for k, ft in features.items()
                }

                if tf.is_tensor(labels):
                    reduced_labels = labels[: self.args.train_batch_size // self.args.n_replicas]
                elif isinstance(labels, dict):
                    reduced_labels = {
                        k: lbl[: self.args.train_batch_size // self.args.n_replicas] for k, lbl in labels.items()
                    }
                else:
                    raise ValueError("The labels must be either a tf.Tensor or a dict.")

                self.training_step(reduced_features, reduced_labels, nb_instances_in_global_batch)

                features = {
                    k: tf.concat(
                        [ft[self.args.train_batch_size // self.args.n_replicas :], reduced_features[k]],
                        axis=0,
                    )
                    for k, ft in features.items()
                }

                if tf.is_tensor(labels):
                    labels = tf.concat(
                        [labels[self.args.train_batch_size // self.args.n_replicas :], reduced_labels], axis=0
                    )
                elif isinstance(labels, dict):
                    labels = {
                        k: tf.concat(
                            [lbl[self.args.train_batch_size // self.args.n_replicas :], reduced_labels[k]],
                            axis=0,
                        )
                        for k, lbl in labels.items()
                    }
                else:
                    raise ValueError("The labels must be either a tf.Tensor or a dict.")

            gradients = self.gradient_accumulator.gradients
            gradients = [
                (tf.clip_by_value(grad, -self.args.max_grad_norm, self.args.max_grad_norm)) for grad in gradients
            ]

            self.optimizer.apply_gradients(list(zip(gradients, self.model.trainable_variables)))
            self.gradient_accumulator.reset()