def _create_slots(self, var_list):
    """Create slot variables for Adam with accumulated gradients."""
    first_var = min(var_list, key=lambda x: x.name)
    self._create_non_slot_variable(
        initial_value=self._beta1, name="beta1_power", colocate_with=first_var)
    self._create_non_slot_variable(
        initial_value=self._beta2, name="beta2_power", colocate_with=first_var)
    # if iter is initialized as an int32, this optimizer could not run
    # with tensorflow_hub with a tensorflow-gpu version
    self._create_non_slot_variable(
        initial_value=0.0 if self._n == 1 else 1.0,
        name="iter",
        colocate_with=first_var)
    # Create slots for the first and second moments, as well as grad_acc.
    for v in var_list:
      self._zeros_slot(v, "m", self._name)
      self._zeros_slot(v, "v", self._name)
      self._zeros_slot(v, "grad_acc", self._name)
Create slot variables for Adam with accumulated gradients.
tensor2tensor/utils/multistep_with_adamoptimizer.py