def forward(self,
                input_ids=None,
                images=None,
                images_mask=None,
                sample_patch_num=None,
                decoder_input_ids=None,
                code_masks=None,
                attention_mask=None,
                encoder_outputs=None,
                past_key_values=None,
                use_cache=False,
                output_attentions=False,
                output_hidden_states=False,
                constrain_fn=None,
                return_dict=False):
        """Forword the module.

        Args:
            input_ids (torch.Tensor): The indices of the input tokens in the
                vocabulary, and padding will be ignored by default. The indices
                can be obtained using :class:`OFATokenizer`.
                The shape is (B, L).
            images (torch.Tensor): The input images. The shape is (B, 3, H, W).
            images_mask (torch.Tensor): The mask of all available images. The
                shape is (B, ).
            sample_patch_num (int): The number of patches to sample for the
                images. Defaults to None, which means to use all patches.
            decoder_input_ids (torch.Tensor): The indices of the input tokens
                for the decoder.
            code_masks (torch.Tensor): The mask of all samples for image
                generation. The shape is (B, ).
            attention_mask (torch.Tensor): The attention mask for decoding.
                The shape is (B, L).
            encoder_outputs (OFAEncoderOutput): The encoder outputs with hidden
                states, positional embeddings, and padding masks.
            past_key_values (Tuple[Tuple[torch.Tensor]]): If use cache, the
                parameter is a tuple of length ``num_layers``. Every item is
                also a tuple with four tensors, two for the key and value of
                self-attention, two for the key and value of cross-attention.
            use_cache (bool): Whether to use cache for faster inference.
                Defaults to False.
            output_attentions (bool): Whether to output attention weights.
                Defaults to False.
            output_hidden_states (bool): Whether to output hidden states.
                Defaults to False.
            constrain_fn (Callable, optional): The function to constrain the
                output logits. Defaults to None.
            return_dict (bool): Not used, it's only for compat with the
                interface of the ``generate`` of ``transformers``.

        Returns:
            Seq2SeqLMOutput:

            - logits (``torch.Tensor``): The last decoder hidden states.
              The shape is (B, L, C).
            - past_key_values (``Tuple[Tuple[torch.Tensor]]``): The past keys
              and values for faster inference.
            - decoder_hidden_states (``Tuple[torch.Tensor]``): the decoder
              hidden states of all layers.
            - decoder_attentions (``Tuple[torch.Tensor]``): The self-attention
              weights of all layers in the decoder.
            - cross_attentions (``Tuple[torch.Tensor]``): The cross-attention
              weights of all layers in the decoder.
            - encoder_last_hidden_state (``torch.Tensor``): The last encoder
              hidden states.
            - encoder_hidden_states (``Tuple[torch.Tensor]``): The encoder
              hidden states of all layers, including the embeddings.
            - encoder_attentions (``Tuple[torch.Tensor]``): The self-attention
              weights of all layers in the encoder.
        """

        if encoder_outputs is None:
            encoder_outputs = self.encoder(
                input_ids=input_ids,
                images=images,
                images_mask=images_mask,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                sample_patch_num=sample_patch_num,
            )

        if decoder_input_ids.eq(self.padding_idx).any():
            attention_mask = decoder_input_ids.eq(self.padding_idx)

        encoder_hidden_states = encoder_outputs.last_hidden_state
        encoder_attention_mask = _expand_mask(encoder_outputs.padding_mask,
                                              encoder_hidden_states.dtype,
                                              decoder_input_ids.shape[-1])
        src_pos_embed = encoder_outputs.position_embedding

        decoder_outputs = self.decoder(
            input_ids=decoder_input_ids,
            attention_mask=attention_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
            code_masks=code_masks,
            encoder_pos_embedding=src_pos_embed,
            past_key_values=past_key_values,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
        )

        # The constrain operation for fine-tuned model in OFA is applied
        # before log_softmax, therefore we cannot use
        # `prefix_allowed_tokens_fn` to implement it.
        if constrain_fn is not None:
            logits = constrain_fn(decoder_input_ids,
                                  decoder_outputs.last_hidden_state)
        else:
            logits = decoder_outputs.last_hidden_state

        return Seq2SeqLMOutput(
            logits=logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )
Forword the module.

        Args:
            input_ids (torch.Tensor): The indices of the input tokens in the
                vocabulary, and padding will be ignored by default. The indices
                can be obtained using :class:`OFATokenizer`.
                The shape is (B, L).
            images (torch.Tensor): The input images. The shape is (B, 3, H, W).
            images_mask (torch.Tensor): The mask of all available images. The
                shape is (B, ).
            sample_patch_num (int): The number of patches to sample for the
                images. Defaults to None, which means to use all patches.
            decoder_input_ids (torch.Tensor): The indices of the input tokens
                for the decoder.
            code_masks (torch.Tensor): The mask of all samples for image
                generation. The shape is (B, ).
            attention_mask (torch.Tensor): The attention mask for decoding.
                The shape is (B, L).
            encoder_outputs (OFAEncoderOutput): The encoder outputs with hidden
                states, positional embeddings, and padding masks.
            past_key_values (Tuple[Tuple[torch.Tensor]]): If use cache, the
                parameter is a tuple of length ``num_layers``. Every item is
                also a tuple with four tensors, two for the key and value of
                self-attention, two for the key and value of cross-attention.
            use_cache (bool): Whether to use cache for faster inference.
                Defaults to False.
            output_attentions (bool): Whether to output attention weights.
                Defaults to False.
            output_hidden_states (bool): Whether to output hidden states.
                Defaults to False.
            constrain_fn (Callable, optional): The function to constrain the
                output logits. Defaults to None.
            return_dict (bool): Not used, it's only for compat with the
                interface of the ``generate`` of ``transformers``.

        Returns:
            Seq2SeqLMOutput:

            - logits (``torch.Tensor``): The last decoder hidden states.
              The shape is (B, L, C).
            - past_key_values (``Tuple[Tuple[torch.Tensor]]``): The past keys
              and values for faster inference.
            - decoder_hidden_states (``Tuple[torch.Tensor]``): the decoder
              hidden states of all layers.
            - decoder_attentions (``Tuple[torch.Tensor]``): The self-attention
              weights of all layers in the decoder.
            - cross_attentions (``Tuple[torch.Tensor]``): The cross-attention
              weights of all layers in the decoder.
            - encoder_last_hidden_state (``torch.Tensor``): The last encoder
              hidden states.
            - encoder_hidden_states (``Tuple[torch.Tensor]``): The encoder
              hidden states of all layers, including the embeddings.
            - encoder_attentions (``Tuple[torch.Tensor]``): The self-attention
              weights of all layers in the encoder.
        
mmpretrain/models/multimodal/ofa/ofa_modules.py