def forward(
        self,
        x,
        attention_mask=None,
        encoder_hidden_states: Optional[torch.Tensor] = None,
        encoder_attention_mask: Optional[torch.Tensor] = None,
        past_key_value: Optional[List[torch.Tensor]] = None,
        output_attentions: bool = False,
        use_cache: bool = False,
        self_attn_bias: Optional[torch.Tensor] = None,
        cross_attn_bias: Optional[torch.Tensor] = None,
    ):
        """Forward the decoder layer.

        Args:
            x (torch.tensor): The input to the layer of shape ``(B, L, C)``.
            attention_mask (torch.Tensor, optional): The attention mask of size
                ``(B, 1, L, L)``, where padding elements are indicated by very
                large negative values. Defaults to None.
            encoder_hidden_states (torch.Tensor, optional): The cross attention
                input to the layer of size ``(B, L, C)``. Defaults to None.
            encoder_attention_mask (torch.Tensor, optional): The cross
                attention mask where padding elements are indicated by very
                large negative values. Defaults to None.
            past_key_value (Tuple[torch.tensor], optional): The cached past key
                and value projection states. Defaults to none.
            output_attentions (bool): whether to return the attentions tensors
                of all attention layers. Defaults to False.
            use_cache (bool, optional): Whether to use cache.
                Defaults to False.
            self_attn_bias (torch.Tensor, optional): The self attention bias
                for positional information. Defaults to None.
            cross_attn_bias (torch.Tensor, optional): The cross attention bias
                for positional information. Defaults to None.

        Returns:
            List[torch.tensor]: The first element is the encoded output of
            shape ``(B, L, C)``. The following two elements can be the output
            self-attentions and cross-attentions if ``output_attentions=True``.
            The following one element can be the cached past key and value
            projection states.
        """
        residual = x

        if past_key_value is not None:
            self_past_key_value = past_key_value[:2]
            cross_past_key_value = past_key_value[2:]
        else:
            self_past_key_value, cross_past_key_value = None, None

        # Self-Attention block
        if self.pre_norm:
            x = self.self_attn_ln(x)
        x, self_attn_weights, present_key_value = self.self_attn(
            query=x,
            past_key_value=self_past_key_value,
            attn_mask=attention_mask,
            output_attentions=output_attentions,
            attn_bias=self_attn_bias,
        )
        if self.normformer:
            x = self.self_attn_mid_ln(x)
        x = self.dropout(x)
        x = residual + self.drop_path(x)
        if not self.pre_norm:
            x = self.self_attn_ln(x)

        # Cross-Attention block
        if encoder_hidden_states is not None:
            residual = x
            if self.pre_norm:
                x = self.cross_attn_ln(x)
            x, cross_attn_weights, cross_key_value = self.cross_attn.forward(
                query=x,
                key_value=encoder_hidden_states,
                attn_mask=encoder_attention_mask,
                past_key_value=cross_past_key_value,
                output_attentions=output_attentions,
                attn_bias=cross_attn_bias)
            if self.normformer:
                x = self.cross_attn_mid_ln(x)
            x = self.dropout(x)
            x = residual + self.drop_path(x)
            if not self.pre_norm:
                x = self.cross_attn_ln(x)

            present_key_value = present_key_value + cross_key_value

        residual = x

        # FFN block
        if self.pre_norm:
            x = self.ffn_ln(x)
        x = self.act(self.fc1(x))
        x = self.act_drop(x)
        if self.normformer:
            x = self.ffn_mid_ln(x)
        x = self.fc2(x)
        x = self.dropout(x)
        x = residual + self.drop_path(x)
        if not self.pre_norm:
            x = self.ffn_ln(x)

        outputs = [x]

        if output_attentions:
            outputs.extend([self_attn_weights, cross_attn_weights])

        if use_cache:
            outputs.append(present_key_value)

        return outputs
Forward the decoder layer.

        Args:
            x (torch.tensor): The input to the layer of shape ``(B, L, C)``.
            attention_mask (torch.Tensor, optional): The attention mask of size
                ``(B, 1, L, L)``, where padding elements are indicated by very
                large negative values. Defaults to None.
            encoder_hidden_states (torch.Tensor, optional): The cross attention
                input to the layer of size ``(B, L, C)``. Defaults to None.
            encoder_attention_mask (torch.Tensor, optional): The cross
                attention mask where padding elements are indicated by very
                large negative values. Defaults to None.
            past_key_value (Tuple[torch.tensor], optional): The cached past key
                and value projection states. Defaults to none.
            output_attentions (bool): whether to return the attentions tensors
                of all attention layers. Defaults to False.
            use_cache (bool, optional): Whether to use cache.
                Defaults to False.
            self_attn_bias (torch.Tensor, optional): The self attention bias
                for positional information. Defaults to None.
            cross_attn_bias (torch.Tensor, optional): The cross attention bias
                for positional information. Defaults to None.

        Returns:
            List[torch.tensor]: The first element is the encoded output of
            shape ``(B, L, C)``. The following two elements can be the output
            self-attentions and cross-attentions if ``output_attentions=True``.
            The following one element can be the cached past key and value
            projection states.
        
mmpretrain/models/multimodal/ofa/ofa_modules.py