def base_nmt():
  """Base hparams for LSTM-based NMT models."""
  hparams = common_hparams.basic_params1()

  # Architecture.
  hparams.shared_embedding_and_softmax_weights = True
  hparams.daisy_chain_variables = False
  hparams.batch_size = 1024
  hparams.hidden_size = 512
  hparams.initializer = "uniform_unit_scaling"
  hparams.initializer_gain = 1.0
  hparams.weight_decay = 0.0
  hparams.dropout = 0.2
  hparams.add_hparam("rnn_unit_type", "lstm")
  hparams.add_hparam("rnn_forget_bias", 1.0)
  hparams.add_hparam("pass_hidden_state", True)

  # Optimizer.
  # Adafactor uses less memory than Adam.
  # switch to Adafactor with its recommended learning rate scheme.
  hparams.optimizer = "Adafactor"
  hparams.learning_rate_schedule = "rsqrt_decay"
  hparams.learning_rate_warmup_steps = 10000

  # Decoder.
  hparams.add_hparam("beam_width", 4)
  hparams.add_hparam("decode_length", 50)
  hparams.add_hparam("decoder_type", "basic")
  hparams.add_hparam("decoder_continuous", False)

  # Training.
  hparams.add_hparam("scheduled_training", False)
  hparams.add_hparam("scheduled_sampling_probability", 0.2)
  hparams.add_hparam("scheduled_mixing_concentration", 10.)

  return hparams
Base hparams for LSTM-based NMT models.
language/labs/consistent_zero_shot_nmt/models/basic.py