def encoder(features, mode, vocab, hps):
  """Model function.

  Atttention seq2seq model, augmented with an encoder
  over the targets of the nearest neighbors.

  Args:
    features: Dictionary of input Tensors.
    mode: train or eval. Keys from tf.estimator.ModeKeys.
    vocab: A list of strings of words in the vocabulary.
    hps: Hyperparams.

  Returns:
    Encoder outputs.
  """

  # [batch_size, src_len]
  src_inputs = features["src_inputs"]
  src_len = features["src_len"]

  with tf.variable_scope("embeddings"):
    scale = (3.0 / hps.emb_dim) ** 0.5
    embeddings = tf.get_variable(
        "embeddings",
        [vocab.size(), hps.emb_dim],
        dtype=tf.float32,
        initializer=tf.random_uniform_initializer(minval=-scale, maxval=scale))

  # [batch_size, src_len, emb_dim]
  src_input_emb = tf.nn.embedding_lookup(embeddings, src_inputs)

  if mode == tf_estimator.ModeKeys.TRAIN and hps.emb_drop > 0.:
    src_input_emb = tf.nn.dropout(
        src_input_emb, keep_prob=1.0-hps.emb_drop)
  src_att_context, neighbor_att_context = None, None
  src_copy_context, neighbor_copy_context = None, None
  with tf.variable_scope("src_encoder"):

    # 2 * [batch_size, src_len, encoder_dim]
    src_encoder_outputs, src_encoder_states = tf.nn.bidirectional_dynamic_rnn(
        cell_fw=get_rnn_cell(
            mode=mode, hps=hps,
            input_dim=hps.emb_dim,
            num_units=hps.encoder_dim,
            num_layers=hps.num_encoder_layers,
            dropout=hps.encoder_drop,
            cell_type="lstm"),
        cell_bw=get_rnn_cell(
            mode=mode, hps=hps,
            input_dim=hps.emb_dim,
            num_units=hps.encoder_dim,
            num_layers=hps.num_encoder_layers,
            dropout=hps.encoder_drop,
            cell_type="lstm"),
        inputs=src_input_emb,
        dtype=tf.float32,
        sequence_length=src_len)

    # [batch_size, src_len, 2*encoder_dim]
    src_encoder_outputs = tf.concat(src_encoder_outputs, 2)

    with tf.variable_scope("src_att_context"):
      src_att_context = _build_context(
          hps=hps,
          encoder_outputs=src_encoder_outputs)
    if hps.use_copy:
      with tf.variable_scope("src_copy_context"):
        src_copy_context = _build_context(
            hps=hps,
            encoder_outputs=src_encoder_outputs)

  if hps.encode_neighbor or hps.att_neighbor or hps.sum_neighbor:
    # [batch_size, neighbor_len]
    neighbor_inputs = features["neighbor_inputs"]
    neighbor_len = features["neighbor_len"]

    # [batch_size, neighbor_len, emb_dim]
    neighbor_input_emb = tf.nn.embedding_lookup(
        embeddings, neighbor_inputs)

    if mode == tf_estimator.ModeKeys.TRAIN and hps.emb_drop > 0.:
      neighbor_input_emb = tf.nn.dropout(
          neighbor_input_emb, keep_prob=1.0-hps.emb_drop)
    if hps.binary_neighbor:
      neighbor_binary_input = features["neighbor_binary"]
      if hps.binary_dim == 1:
        neighbor_binary_emb = tf.to_float(neighbor_binary_input)
        neighbor_binary_emb = tf.expand_dims(neighbor_binary_emb, axis=-1)
      else:
        with tf.variable_scope("binary_emb"):
          scale = (3.0 / hps.binary_dim) ** 0.5
          binary_embeddings = tf.get_variable(
              "binary_emb",
              [2, hps.binary_dim],
              dtype=tf.float32,
              initializer=tf.random_uniform_initializer(
                  minval=-scale, maxval=scale))
        neighbor_binary_emb = tf.nn.embedding_lookup(
            binary_embeddings, neighbor_binary_input)

      neighbor_input_emb = tf.concat(
          [neighbor_input_emb, neighbor_binary_emb], axis=2)

    with tf.variable_scope("neighbor_encoder"):
      # 2 * [batch_size, neighbor_len, encoder_dim]
      input_dim = hps.emb_dim
      if hps.binary_neighbor:
        input_dim += hps.binary_dim
      neighbor_encoder_outputs, neighbor_encoder_states = \
          tf.nn.bidirectional_dynamic_rnn(
              cell_fw=get_rnn_cell(
                  mode=mode, hps=hps,
                  input_dim=input_dim,
                  num_units=hps.neighbor_dim,
                  num_layers=1,
                  dropout=hps.encoder_drop,
                  cell_type="lstm"),
              cell_bw=get_rnn_cell(
                  mode=mode, hps=hps,
                  input_dim=input_dim,
                  num_units=hps.neighbor_dim,
                  num_layers=1,
                  dropout=hps.encoder_drop,
                  cell_type="lstm"),
              inputs=neighbor_input_emb,
              dtype=tf.float32,
              sequence_length=neighbor_len)
      # [batch_size, neighbor_len, 2*encoder_dim]
      neighbor_encoder_outputs = tf.concat(neighbor_encoder_outputs, 2)
      if hps.att_neighbor:
        with tf.variable_scope("neighbor_att_context"):
          neighbor_att_context = _build_context(
              hps=hps,
              encoder_outputs=neighbor_encoder_outputs)
        if hps.use_copy:
          with tf.variable_scope("neighbor_copy_context"):
            neighbor_copy_context = _build_context(
                hps=hps,
                encoder_outputs=neighbor_encoder_outputs)
  att_context, copy_context = None, None
  if hps.att_neighbor:
    att_context = tf.concat([src_att_context, neighbor_att_context], 1)
    if hps.use_copy:
      copy_context = tf.concat(
          [src_copy_context, neighbor_copy_context], 1)
  else:
    att_context = src_att_context
    if hps.use_copy:
      copy_context = src_copy_context
  if hps.encode_neighbor:
    neighbor_fw_states, neighbor_bw_states = neighbor_encoder_states
    neighbor_h = tf.concat(
        [neighbor_fw_states[-1].h, neighbor_bw_states[-1].h], axis=1)
    if mode == tf_estimator.ModeKeys.TRAIN and hps.drop > 0.:
      neighbor_h = tf.nn.dropout(neighbor_h, keep_prob=1.0-hps.drop)
    mem_input = tf.layers.dense(
        neighbor_h, units=hps.decoder_dim,
        activation=tf.nn.tanh,
        use_bias=True,
        kernel_initializer=tf.contrib.layers.xavier_initializer(),
        name="mem_input")
    if mode == tf_estimator.ModeKeys.TRAIN and hps.drop > 0.:
      mem_input = tf.nn.dropout(
          mem_input, keep_prob=1.0-hps.drop)
  elif hps.sum_neighbor:

    src_fw_states, src_bw_states = src_encoder_states
    src_h = tf.concat([src_fw_states[-1].h, src_bw_states[-1].h], axis=1)

    if mode == tf_estimator.ModeKeys.TRAIN and hps.drop > 0.:
      src_h = tf.nn.dropout(src_h, keep_prob=1.0-hps.drop)
    src_h = tf.layers.dense(
        src_h, units=hps.decoder_dim,
        activation=tf.nn.tanh,
        use_bias=True,
        kernel_initializer=tf.contrib.layers.xavier_initializer(),
        name="proj_src_h")
    neighbor_encoder_outputs = tf.layers.dense(
        neighbor_encoder_outputs, units=hps.decoder_dim,
        activation=tf.nn.tanh,
        use_bias=True,
        kernel_initializer=tf.contrib.layers.xavier_initializer(),
        name="proj_neighbor_out")

    alpha = tf.tensordot(neighbor_encoder_outputs, src_h, axes=1)
    alpha = tf.einsum("bij,bj->bi", neighbor_encoder_outputs, src_h)
    alpha = tf.nn.softmax(alpha)
    mem_input = tf.reduce_sum(
        neighbor_encoder_outputs * tf.expand_dims(alpha, -1), 1)
    # mem_input = tf.reduce_mean(mem_input, axis=1)
    if mode == tf_estimator.ModeKeys.TRAIN and hps.drop > 0.:
      mem_input = tf.nn.dropout(mem_input, keep_prob=1.0-hps.drop)
  else:
    assert hps.rnn_cell != "hyper_lstm"
    assert hps.att_type != "hyper"
    mem_input = None

  if hps.use_bridge:
    with tf.variable_scope("bridge"):
      out_dim = hps.num_decoder_layers * hps.decoder_dim
      fw_states, bw_states = src_encoder_states
      c_states, h_states = [], []
      for (fw, bw) in zip(fw_states, bw_states):
        c_states.append(tf.concat((fw.c, bw.c), 1))
        h_states.append(tf.concat((fw.h, bw.h), 1))
      cs, hs = c_states[-1], h_states[-1]

      if mode == tf_estimator.ModeKeys.TRAIN and hps.drop > 0.:
        hs = tf.nn.dropout(hs, keep_prob=1.0-hps.drop)
        cs = tf.nn.dropout(cs, keep_prob=1.0-hps.drop)

      h_state = tf.layers.dense(
          hs, units=out_dim,
          activation=tf.nn.tanh,
          use_bias=True,
          kernel_initializer=tf.contrib.layers.xavier_initializer(),
          name="h_layer")
      c_state = tf.layers.dense(
          cs, units=out_dim,
          activation=tf.nn.tanh,
          use_bias=True,
          kernel_initializer=tf.contrib.layers.xavier_initializer(),
          name="c_layer")
  else:
    h_state, c_state = None, None
  # print (att_context)
  # dsadsa
  return EncoderOutputs(
      embeddings=embeddings,
      mem_input=mem_input,
      att_context=att_context,
      copy_context=copy_context,
      states=(h_state, c_state)
  )
Model function.

  Atttention seq2seq model, augmented with an encoder
  over the targets of the nearest neighbors.

  Args:
    features: Dictionary of input Tensors.
    mode: train or eval. Keys from tf.estimator.ModeKeys.
    vocab: A list of strings of words in the vocabulary.
    hps: Hyperparams.

  Returns:
    Encoder outputs.
  
language/labs/exemplar_decoding/models/hypernet.py