src/transformers/models/canine/modeling_canine.py/ConvProjection/forward
class ConvProjection:    def forward(self, inputs, final_seq_char_positions=None):
        # inputs has shape [batch, mol_seq, molecule_hidden_size+char_hidden_final]
        # we transpose it to be [batch, molecule_hidden_size+char_hidden_final, mol_seq]
        inputs = torch.transpose(inputs, 1, 2)

        # PyTorch < 1.9 does not support padding="same" (which is used in the original implementation),
        # so we pad the tensor manually before passing it to the conv layer
        # based on https://github.com/google-research/big_transfer/blob/49afe42338b62af9fbe18f0258197a33ee578a6b/bit_tf2/models.py#L36-L38
        pad_total = self.config.upsampling_kernel_size - 1
        pad_beg = pad_total // 2
        pad_end = pad_total - pad_beg

        pad = nn.ConstantPad1d((pad_beg, pad_end), 0)
        # `result`: shape (batch_size, char_seq_len, hidden_size)
        result = self.conv(pad(inputs))
        result = torch.transpose(result, 1, 2)
        result = self.activation(result)
        result = self.LayerNorm(result)
        result = self.dropout(result)
        final_char_seq = result

        if final_seq_char_positions is not None:
            # Limit transformer query seq and attention mask to these character
            # positions to greatly reduce the compute cost. Typically, this is just
            # done for the MLM training task.
            # TODO add support for MLM
            raise NotImplementedError("CanineForMaskedLM is currently not supported")
        else:
            query_seq = final_char_seq

        return query_seq