def _build_decoder_lm_loss(self, central_lang_tag="<en>"):
    """Builds LM loss on the auxiliary decodings."""

    # Get target embeddigns and vocab size.
    target_modality = self._problem_hparams.modality["targets"]
    target_modality_scope = self._variable_scopes[target_modality.name]
    target_embeddings = model_utils.get_embeddings(
        modality=target_modality,
        outer_scope=target_modality_scope,
        inner_scope="shared")
    target_vocab_size = target_modality._vocab_size  # pylint: disable=protected-access

    # Build auxiliary sequences (if necessary).
    aux_keys = self._build_aux_sequences(
        target_embeddings, target_vocab_size,
        central_lang_tag=central_lang_tag)

    # Make sure LM loss does not affect embeddings.
    target_embeddings = tf.stop_gradient(target_embeddings)

    # Build loss.
    aux_loss = 0.
    with tf.name_scope("aux_lm_loss"):
      for key in aux_keys:
        dec_outputs = tf.expand_dims(
            self.dec_outputs[key]["rnn_output"], axis=2)
        dec_output_tags = tf.expand_dims(
            self.inputs[key][0][:, :1], axis=2)
        dec_lengths = self.dec_outputs[key]["length"]
        # Preprocess LM features.
        lm_features = {
            "targets": dec_outputs,
            "target_tags": dec_output_tags}
        inputs, inputs_length = self._build_lm_inputs(lm_features)
        # Build LM (with frozen weights in PREDICT mode).
        lm_outputs = self.language_model(
            inputs=inputs,
            inputs_length=inputs_length,
            mode=tf_estimator.ModeKeys.PREDICT,
            hparams=self._hparams,
            trainable=False,
            reuse=tf.AUTO_REUSE)
        # Compute logits.
        lm_logits = model_utils.build_logits(
            sequences=tf.expand_dims(lm_outputs, axis=2),
            embeddings=target_embeddings,
            vocab_size=target_vocab_size)
        # Compute decoder probabilities.
        dec_logits = model_utils.build_logits(
            sequences=dec_outputs,
            embeddings=target_embeddings,
            vocab_size=target_vocab_size)
        dec_probs = tf.nn.softmax(dec_logits, axis=-1)
        # Compute cross-entropy loss.
        aux_loss = aux_loss + losses.CrossEntropyLoss(sparse=False)(
            lm_logits, dec_probs, dec_lengths)

    aux_loss = self._hparams.lm_loss_coeff * aux_loss

    return aux_loss
Builds LM loss on the auxiliary decodings.
language/labs/consistent_zero_shot_nmt/models/agreement.py