def _chars_to_molecules(
      self,
      char_encoding: tf.Tensor,
      expected_molecule_seq_length: tf.Tensor) -> tf.Tensor:
    """Convert char seq to initial molecule seq."""

    del expected_molecule_seq_length  # Used by contract only.

    with tf.variable_scope("initial_char_encoder/chars_to_molecules"):
      downsampled = tf.layers.conv1d(
          inputs=char_encoding,
          filters=self.config.hidden_size,
          kernel_size=self.config.downsampling_rate,
          strides=self.config.downsampling_rate,
          padding="valid",
          activation=bert_modeling.get_activation(self.config.hidden_act),
          name="conv")

      # `cls_encoding`: [batch, 1, hidden_size]
      cls_encoding = char_encoding[:, 0:1, :]

      # Truncate the last molecule in order to reserve a position for [CLS].
      # Often, the last position is never used (unless we completely fill the
      # text buffer). This is important in order to maintain alignment on TPUs
      # (i.e. a multiple of 128).
      downsampled_truncated = downsampled[:, 0:-1, :]

      # We also keep [CLS] as a separate sequence position since we always
      # want to reserve a position (and the model capacity that goes along
      # with that) in the deep BERT stack.
      # `result`: [batch, molecule_seq, molecule_dim]
      result = tf.concat([cls_encoding, downsampled_truncated], axis=1)

      return bert_modeling.layer_norm(result)
Convert char seq to initial molecule seq.
language/canine/modeling.py