def _encode_initial_chars(self, char_embed_seq: tf.Tensor,
                            char_attention_mask: tf.Tensor) -> tf.Tensor:
    """Encode characters using shallow/low dim transformer."""
    with tf.variable_scope("initial_char_encoder"):
      return local_attention.local_transformer_model(
          input_tensor=char_embed_seq,
          attention_mask=char_attention_mask,
          hidden_size=self.config.hidden_size,
          num_hidden_layers=1,
          num_attention_heads=self.config.num_attention_heads,
          intermediate_size=self.config.intermediate_size,
          intermediate_act_fn=bert_modeling.get_activation(
              self.config.hidden_act),
          hidden_dropout_prob=self.config.hidden_dropout_prob,
          attention_probs_dropout_prob=(
              self.config.attention_probs_dropout_prob),
          initializer_range=self.config.initializer_range,
          always_attend_to_first_position=False,
          first_position_attends_to_all=False,
          attend_from_chunk_width=self.config.local_transformer_stride,
          attend_from_chunk_stride=self.config.local_transformer_stride,
          attend_to_chunk_width=self.config.local_transformer_stride,
          attend_to_chunk_stride=self.config.local_transformer_stride)
Encode characters using shallow/low dim transformer.
language/canine/modeling.py