def _upsample_molecules_to_chars(self, final_char_input_seq: tf.Tensor,
                                   full_molecules: tf.Tensor) -> tf.Tensor:
    """Run a shallow/low-dim transformer to get a final character encoding."""
    _, char_seq_length, _ = bert_modeling.get_shape_list(final_char_input_seq)

    # `repeated_molecules`: [batch_size, char_seq_len, molecule_hidden_size]
    repeated_molecules = self._repeat_molecules(
        full_molecules, char_seq_length=char_seq_length)
    # `concat`:
    #     [batch_size, char_seq_len, molecule_hidden_size+char_hidden_final]
    concat = tf.concat([final_char_input_seq, repeated_molecules], axis=-1)

    # `upsampled`: [batch_size, char_seq_len, hidden_size]
    upsampled = tf.layers.conv1d(
        inputs=concat,
        filters=self.config.hidden_size,
        kernel_size=self.config.upsampling_kernel_size,
        strides=1,
        padding="same",
        activation=bert_modeling.get_activation(self.config.hidden_act),
        name="conv")
    upsampled = bert_modeling.layer_norm(upsampled)
    if self._is_training:
      upsampled = bert_modeling.dropout(upsampled,
                                        self.config.hidden_dropout_prob)
    return upsampled
Run a shallow/low-dim transformer to get a final character encoding.
language/canine/modeling.py