src/transformers/models/convbert/modeling_tf_convbert.py/TFConvBertSelfAttention/__init__
class TFConvBertSelfAttention:    def __init__(self, config, **kwargs):
        super().__init__(**kwargs)

        if config.hidden_size % config.num_attention_heads != 0:
            raise ValueError(
                f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
                f"heads ({config.num_attention_heads})"
            )

        new_num_attention_heads = int(config.num_attention_heads / config.head_ratio)
        if new_num_attention_heads < 1:
            self.head_ratio = config.num_attention_heads
            num_attention_heads = 1
        else:
            num_attention_heads = new_num_attention_heads
            self.head_ratio = config.head_ratio

        self.num_attention_heads = num_attention_heads
        self.conv_kernel_size = config.conv_kernel_size

        assert (
            config.hidden_size % self.num_attention_heads == 0
        ), "hidden_size should be divisible by num_attention_heads"

        self.attention_head_size = config.hidden_size // config.num_attention_heads
        self.all_head_size = self.num_attention_heads * self.attention_head_size
        self.query = tf.keras.layers.Dense(
            self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name="query"
        )
        self.key = tf.keras.layers.Dense(
            self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name="key"
        )
        self.value = tf.keras.layers.Dense(
            self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name="value"
        )

        self.key_conv_attn_layer = tf.keras.layers.SeparableConv1D(
            self.all_head_size,
            self.conv_kernel_size,
            padding="same",
            activation=None,
            depthwise_initializer=get_initializer(1 / self.conv_kernel_size),
            pointwise_initializer=get_initializer(config.initializer_range),
            name="key_conv_attn_layer",
        )

        self.conv_kernel_layer = tf.keras.layers.Dense(
            self.num_attention_heads * self.conv_kernel_size,
            activation=None,
            name="conv_kernel_layer",
            kernel_initializer=get_initializer(config.initializer_range),
        )

        self.conv_out_layer = tf.keras.layers.Dense(
            self.all_head_size,
            activation=None,
            name="conv_out_layer",
            kernel_initializer=get_initializer(config.initializer_range),
        )

        self.dropout = tf.keras.layers.Dropout(config.attention_probs_dropout_prob)