def __init__(self, num_units, mem_input,
               use_peepholes=False, cell_clip=None,
               initializer=None, num_proj=None, proj_clip=None,
               num_unit_shards=None, num_proj_shards=None,
               forget_bias=1.0, state_is_tuple=True,
               activation=None, reuse=None, name=None, dtype=None,
               use_beam=False,
               hps=None):
    """Initialize the HyperLSTM cell.

    Args:
      num_units: int, The number of units in the LSTM cell.
      mem_input: mem_input.
      use_peepholes: bool, use peephole connections or not.
      cell_clip: (optional) A float value, if provided the cell state is clipped
        by this value prior to the cell output activation.
      initializer: (optional) The initializer to use for the weight and
        projection matrices.
      num_proj: (optional) int, The output dimensionality for the projection
        matrices.  If None, no projection is performed.
      proj_clip: (optional) A float value.  If `num_proj > 0` and `proj_clip` is
        provided, then the projected values are clipped elementwise to within
        `[-proj_clip, proj_clip]`.
      num_unit_shards: Deprecated, will be removed by Jan. 2017.
        Use a variable_scope partitioner instead.
      num_proj_shards: Deprecated, will be removed by Jan. 2017.
        Use a variable_scope partitioner instead.
      forget_bias: float, The bias added to forget gates (see above).
        Must set to `0.0` manually when restoring from CudnnLSTM-trained
        checkpoints.
      state_is_tuple: If True, accepted and returned states are 2-tuples of
        the `c_state` and `m_state`.  If False, they are concatenated
        along the column axis.  The latter behavior will soon be deprecated.
      activation: Activation function of the inner states.  Default: `tanh`.
      reuse: (optional) Python boolean describing whether to reuse variables
        in an existing scope.  If not `True`, and the existing scope already has
        the given variables, an error is raised.
      name: String, the name of the layer. Layers with the same name will
        share weights, but to avoid mistakes we require reuse=True in such
        cases.
      dtype: Default dtype of the layer (default of `None` means use the type
        of the first input). Required when `build` is called before `call`.
      use_beam: Use beam search or not.
      hps: hyperparameters.
    """

    super(HyperLSTMCell, self).__init__(_reuse=reuse, name=name, dtype=dtype)
    if not state_is_tuple:
      tf.logging.warn("%s: Using a concatenated state is slower and will soon "
                      "be deprecated.  Use state_is_tuple=True.", self)
    if num_unit_shards is not None or num_proj_shards is not None:
      tf.logging.warn(
          "%s: The num_unit_shards and proj_unit_shards parameters are "
          "deprecated and will be removed in Jan 2017.  "
          "Use a variable scope with a partitioner instead.", self)

    assert not use_peepholes, "currently not supporting peephole connections"
    assert hps is not None
    # Inputs must be 2-dimensional.
    self.input_spec = tf.layers.InputSpec(ndim=2)

    self._num_units = num_units
    self._rank = hps.rank
    assert self._rank == self._num_units or self._rank == 2 * self._num_units
    self._use_peepholes = use_peepholes
    self._cell_clip = cell_clip
    self._initializer = initializer
    self._num_proj = num_proj
    self._proj_clip = proj_clip
    self._num_unit_shards = num_unit_shards
    self._num_proj_shards = num_proj_shards
    self._forget_bias = forget_bias
    self._state_is_tuple = state_is_tuple
    self._activation = activation or tf.tanh
    self._sigma_norm = hps.sigma_norm
    self._beam_width = hps.beam_width
    self._mem_input = mem_input
    self._use_beam = use_beam

    if num_proj:
      self._state_size = (
          tf.nn.rnn_cell.LSTMStateTuple(num_units, num_proj)
          if state_is_tuple else num_units + num_proj)
      self._output_size = num_proj
    else:
      self._state_size = (
          tf.nn.rnn_cell.LSTMStateTuple(num_units, num_units)
          if state_is_tuple else 2 * num_units)
      self._output_size = num_units

    input_depth = hps.emb_dim + hps.decoder_dim
    # if hps.encode_neighbor:
    #   input_depth += hps.decoder_dim
    h_depth = self._num_units if self._num_proj is None else self._num_proj

    maybe_partitioner = (
        tf.fixed_size_partitioner(self._num_unit_shards)
        if self._num_unit_shards is not None else None)

    # `u`s are matrices of [input_shape, rank], `v`s being [rank, hidden_size]
    # they are the collection of rank-1 parameter matrices.
    # The full parameter matrix is constructed by taking `U\sigma V`,
    # with diagonal matrix `\sigma` computed in the `self.initialize` function.

    redundant_rank = (self._rank > self._num_units)
    # `u`, `v` used to construct matrix from input `x` to input_gate `i`.
    u_xi, v_xi = self._orthogonal_init(
        shape=[input_depth, self._num_units],
        initializer=initializer,
        redundant_rank=redundant_rank)
    self._u_xi = tf.get_variable(
        "u_xi/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=u_xi,
        partitioner=maybe_partitioner)
    self._v_xi = tf.get_variable(
        "v_xi/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=v_xi,
        partitioner=maybe_partitioner)

    # `u`, `v` used to construct matrix that maps input `x` to cell_state `j`.
    u_xj, v_xj = self._orthogonal_init(
        shape=[input_depth, self._num_units],
        initializer=initializer,
        redundant_rank=redundant_rank)
    self._u_xj = tf.get_variable(
        "u_xj/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=u_xj,
        partitioner=maybe_partitioner)
    self._v_xj = tf.get_variable(
        "v_xj/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=v_xj,
        partitioner=maybe_partitioner)

    # `u`, `v` used to construct matrix
    # that maps input `x` to forget_gate `f`.
    u_xf, v_xf = self._orthogonal_init(
        shape=[input_depth, self._num_units],
        initializer=initializer,
        redundant_rank=redundant_rank)
    self._u_xf = tf.get_variable(
        "u_xf/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=u_xf,
        partitioner=maybe_partitioner)
    self._v_xf = tf.get_variable(
        "v_xf/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=v_xf,
        partitioner=maybe_partitioner)

    # `u`, `v` used to construct matrix
    # that maps input `x` to output_gate `o`.
    u_xo, v_xo = self._orthogonal_init(
        shape=[input_depth, self._num_units],
        initializer=initializer,
        redundant_rank=redundant_rank)
    self._u_xo = tf.get_variable(
        "u_xo/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=u_xo,
        partitioner=maybe_partitioner)
    self._v_xo = tf.get_variable(
        "v_xo/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=v_xo,
        partitioner=maybe_partitioner)

    # `u`, `v` used to construct matrix
    # that maps hid_state `h` to input_gate `i`.
    u_hi, v_hi = self._orthogonal_init(
        shape=[h_depth, self._num_units],
        initializer=initializer,
        redundant_rank=redundant_rank)
    self._u_hi = tf.get_variable(
        "u_hi/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=u_hi,
        partitioner=maybe_partitioner)
    self._v_hi = tf.get_variable(
        "v_hi/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=v_hi,
        partitioner=maybe_partitioner)

    # `u`, `v` used to construct matrix
    # that maps hid_state `h` to cell_state `j`.
    u_hj, v_hj = self._orthogonal_init(
        shape=[h_depth, self._num_units],
        initializer=initializer,
        redundant_rank=redundant_rank)
    self._u_hj = tf.get_variable(
        "u_hj/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=u_hj,
        partitioner=maybe_partitioner)
    self._v_hj = tf.get_variable(
        "v_hj/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=v_hj,
        partitioner=maybe_partitioner)

    # `u`, `v` used to construct matrix
    # that maps hid_state `h` to forget_gate `f`.
    u_hf, v_hf = self._orthogonal_init(
        shape=[h_depth, self._num_units],
        initializer=initializer,
        redundant_rank=redundant_rank)
    self._u_hf = tf.get_variable(
        "u_hf/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=u_hf,
        partitioner=maybe_partitioner)
    self._v_hf = tf.get_variable(
        "v_hf/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=v_hf,
        partitioner=maybe_partitioner)

    # `u`, `v` used to construct matrix
    # that maps hid_state `h` to output_gate `o`.
    u_ho, v_ho = self._orthogonal_init(
        shape=[h_depth, self._num_units],
        initializer=initializer,
        redundant_rank=redundant_rank)
    self._u_ho = tf.get_variable(
        "u_ho/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=u_ho,
        partitioner=maybe_partitioner)
    self._v_ho = tf.get_variable(
        "v_ho/%s" % _WEIGHTS_VARIABLE_NAME,
        initializer=v_ho,
        partitioner=maybe_partitioner)

    self._c = tf.get_variable(
        "c/%s" % _WEIGHTS_VARIABLE_NAME,
        shape=[self._num_units, self._rank],
        initializer=tf.contrib.layers.xavier_initializer(),
        partitioner=maybe_partitioner)

    initializer = tf.zeros_initializer(dtype=tf.float32)
    self._b = tf.get_variable(
        "b/%s" % _BIAS_VARIABLE_NAME,
        shape=[4 * h_depth, self._rank],
        initializer=initializer)

    if self._num_proj is not None:
      if self._num_proj_shards is not None:
        maybe_proj_partitioner = (
            tf.fixed_size_partitioner(self._num_proj_shards))
      else:
        maybe_proj_partitioner = (None)
      self._proj_kernel = self.add_variable(
          "projection/%s" % _WEIGHTS_VARIABLE_NAME,
          shape=[self._num_units, self._num_proj],
          initializer=tf.uniform_unit_scaling_initializer(),
          partitioner=maybe_proj_partitioner)
    self.initialize()
    self.built = True
Initialize the HyperLSTM cell.

    Args:
      num_units: int, The number of units in the LSTM cell.
      mem_input: mem_input.
      use_peepholes: bool, use peephole connections or not.
      cell_clip: (optional) A float value, if provided the cell state is clipped
        by this value prior to the cell output activation.
      initializer: (optional) The initializer to use for the weight and
        projection matrices.
      num_proj: (optional) int, The output dimensionality for the projection
        matrices.  If None, no projection is performed.
      proj_clip: (optional) A float value.  If `num_proj > 0` and `proj_clip` is
        provided, then the projected values are clipped elementwise to within
        `[-proj_clip, proj_clip]`.
      num_unit_shards: Deprecated, will be removed by Jan. 2017.
        Use a variable_scope partitioner instead.
      num_proj_shards: Deprecated, will be removed by Jan. 2017.
        Use a variable_scope partitioner instead.
      forget_bias: float, The bias added to forget gates (see above).
        Must set to `0.0` manually when restoring from CudnnLSTM-trained
        checkpoints.
      state_is_tuple: If True, accepted and returned states are 2-tuples of
        the `c_state` and `m_state`.  If False, they are concatenated
        along the column axis.  The latter behavior will soon be deprecated.
      activation: Activation function of the inner states.  Default: `tanh`.
      reuse: (optional) Python boolean describing whether to reuse variables
        in an existing scope.  If not `True`, and the existing scope already has
        the given variables, an error is raised.
      name: String, the name of the layer. Layers with the same name will
        share weights, but to avoid mistakes we require reuse=True in such
        cases.
      dtype: Default dtype of the layer (default of `None` means use the type
        of the first input). Required when `build` is called before `call`.
      use_beam: Use beam search or not.
      hps: hyperparameters.
    
language/labs/exemplar_decoding/models/hyperlstm.py