def get_decoder_layers(name,
                       input_layer,
                       encoded_layer,
                       head_num,
                       hidden_dim,
                       attention_activation=None,
                       feed_forward_activation='relu',
                       dropout_rate=0.0,
                       trainable=True,
                       use_adapter=False,
                       adapter_units=None,
                       adapter_activation='relu'):
    """Multi-head self-attention, multi-head query attention and feed-forward layer.

    :param name: Prefix of names for internal layers.
    :param input_layer: Input layer.
    :param encoded_layer: Encoded layer from encoder.
    :param head_num: Number of heads in multi-head self-attention.
    :param hidden_dim: Hidden dimension of feed forward layer.
    :param attention_activation: Activation for multi-head self-attention.
    :param feed_forward_activation: Activation for feed-forward layer.
    :param dropout_rate: Dropout rate.
    :param trainable: Whether the layers are trainable.
    :param use_adapter: Whether to use feed-forward adapters before each residual connections.
    :param adapter_units: The dimension of the first transformation in feed-forward adapter.
    :param adapter_activation: The activation after the first transformation in feed-forward adapter.
    :return: Output layer.
    """
    self_attention_name = '%s-MultiHeadSelfAttention' % name
    query_attention_name = '%s-MultiHeadQueryAttention' % name
    feed_forward_name = '%s-FeedForward' % name
    self_attention_layer = _common_wrap_layer(name=self_attention_name,
                                              input_layer=input_layer,
                                              build_func=build_attention(name=self_attention_name,
                                                                         head_num=head_num,
                                                                         activation=attention_activation,
                                                                         history_only=True,
                                                                         trainable=trainable, ),
                                              dropout_rate=dropout_rate,
                                              trainable=trainable,
                                              use_adapter=use_adapter,
                                              adapter_units=adapter_units,
                                              adapter_activation=adapter_activation,
                                              )
    query_attention_layer = _common_wrap_layer(name=query_attention_name,
                                               input_layer=[self_attention_layer, encoded_layer, encoded_layer],
                                               build_func=build_attention(name=query_attention_name,
                                                                          head_num=head_num,
                                                                          activation=attention_activation,
                                                                          history_only=False,
                                                                          trainable=trainable,
                                                                          ),
                                               dropout_rate=dropout_rate,
                                               trainable=trainable,
                                               use_adapter=use_adapter,
                                               adapter_units=adapter_units,
                                               adapter_activation=adapter_activation,
                                               )
    feed_forward_layer = _common_wrap_layer(name=feed_forward_name,
                                            input_layer=query_attention_layer,
                                            build_func=build_feed_forward(name=feed_forward_name,
                                                                          hidden_dim=hidden_dim,
                                                                          activation=feed_forward_activation,
                                                                          trainable=trainable,
                                                                          ),
                                            dropout_rate=dropout_rate,
                                            trainable=trainable,
                                            use_adapter=use_adapter,
                                            adapter_units=adapter_units,
                                            adapter_activation=adapter_activation,
                                            )
    return feed_forward_layer
Multi-head self-attention, multi-head query attention and feed-forward layer.

    :param name: Prefix of names for internal layers.
    :param input_layer: Input layer.
    :param encoded_layer: Encoded layer from encoder.
    :param head_num: Number of heads in multi-head self-attention.
    :param hidden_dim: Hidden dimension of feed forward layer.
    :param attention_activation: Activation for multi-head self-attention.
    :param feed_forward_activation: Activation for feed-forward layer.
    :param dropout_rate: Dropout rate.
    :param trainable: Whether the layers are trainable.
    :param use_adapter: Whether to use feed-forward adapters before each residual connections.
    :param adapter_units: The dimension of the first transformation in feed-forward adapter.
    :param adapter_activation: The activation after the first transformation in feed-forward adapter.
    :return: Output layer.
    
keras_textclassification/keras_layers/transformer.py