def forward(
        self,
        pixel_values: torch.FloatTensor,
        audio_values: torch.FloatTensor,
        pixel_mask: Optional[torch.FloatTensor] = None,
        audio_mask: Optional[torch.FloatTensor] = None,
        mask_pixel: bool = False,
        mask_audio: bool = False,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[torch.FloatTensor], TvltModelOutput]:
        r"""
        Returns:

        Examples:

        ```python
        >>> from transformers import TvltProcessor, TvltModel
        >>> import numpy as np
        >>> import torch

        >>> num_frames = 8
        >>> images = list(np.random.randn(num_frames, 3, 224, 224))
        >>> audio = list(np.random.randn(10000))

        >>> processor = TvltProcessor.from_pretrained("ZinengTang/tvlt-base")
        >>> model = TvltModel.from_pretrained("ZinengTang/tvlt-base")

        >>> input_dict = processor(images, audio, sampling_rate=44100, return_tensors="pt")

        >>> outputs = model(**input_dict)
        >>> loss = outputs.loss
        ```"""

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        pixel_embedding_output, pixel_mask = self.pixel_embeddings(pixel_values, pixel_mask)

        audio_embedding_output, audio_mask = self.audio_embeddings(audio_values, audio_mask)

        # Mask pixel if mask_pixel is True
        pixel_label_masks = None
        pixel_ids_restore = None
        if mask_pixel:
            pixel_mask_noise, pixel_len_keep = generate_pixel_mask_noise(
                pixel_embedding_output, pixel_mask=pixel_mask, mask_ratio=self.config.pixel_mask_ratio
            )
            pixel_embedding_output, pixel_mask, pixel_label_masks, pixel_ids_restore = random_masking(
                pixel_embedding_output,
                pixel_mask_noise,
                pixel_len_keep,
                attention_masks=pixel_mask,
            )

        # Mask audio if mask_audio is True
        audio_label_masks = None
        audio_ids_restore = None
        if mask_audio:
            num_freq_patches = self.config.frequency_length // self.config.audio_patch_size[1]
            audio_mask_noise, audio_len_keep = generate_audio_mask_noise(
                audio_embedding_output,
                audio_mask=audio_mask,
                mask_ratio=self.config.audio_mask_ratio,
                mask_type=self.config.audio_mask_type,
                freq_len=num_freq_patches,
            )
            audio_embedding_output, audio_mask, audio_label_masks, audio_ids_restore = random_masking(
                audio_embedding_output,
                audio_mask_noise,
                audio_len_keep,
                attention_masks=audio_mask,
            )

        # Prepare for encoder inputs and attention masks
        batch_size = pixel_values.size(0)
        embedding_output = torch.cat(
            [self.cls_embedding.repeat(batch_size, 1, 1), pixel_embedding_output, audio_embedding_output], 1
        )
        masked_pixel_len = pixel_embedding_output.size(1)

        attention_mask = None
        if pixel_mask is not None and audio_mask is not None:
            attention_mask = torch.cat([pixel_mask[:, :1], pixel_mask, audio_mask], 1)

        input_shape = embedding_output.size()
        extended_attention_mask = None
        if attention_mask is not None:
            extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)

        encoder_outputs = self.encoder(
            embedding_output,
            attention_mask=extended_attention_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
        sequence_output = encoder_outputs[0]
        if self.layernorm is not None:
            sequence_output = self.layernorm(sequence_output)

        pixel_sequence_output = sequence_output[:, 1 : 1 + masked_pixel_len]
        audio_sequence_output = sequence_output[:, 1 + masked_pixel_len :]
        if not return_dict:
            return (
                sequence_output,
                pixel_sequence_output,
                audio_sequence_output,
                pixel_label_masks,
                audio_label_masks,
                pixel_ids_restore,
                audio_ids_restore,
            ) + encoder_outputs[1:]

        return TvltModelOutput(
            last_hidden_state=sequence_output,
            last_pixel_hidden_state=pixel_sequence_output,
            last_audio_hidden_state=audio_sequence_output,
            pixel_label_masks=pixel_label_masks,
            audio_label_masks=audio_label_masks,
            pixel_ids_restore=pixel_ids_restore,
            audio_ids_restore=audio_ids_restore,
            hidden_states=encoder_outputs.hidden_states,
            attentions=encoder_outputs.attentions,
        )

        Returns:

        Examples:

        ```python
        >>> from transformers import TvltProcessor, TvltModel
        >>> import numpy as np
        >>> import torch

        >>> num_frames = 8
        >>> images = list(np.random.randn(num_frames, 3, 224, 224))
        >>> audio = list(np.random.randn(10000))

        >>> processor = TvltProcessor.from_pretrained("ZinengTang/tvlt-base")
        >>> model = TvltModel.from_pretrained("ZinengTang/tvlt-base")

        >>> input_dict = processor(images, audio, sampling_rate=44100, return_tensors="pt")

        >>> outputs = model(**input_dict)
        >>> loss = outputs.loss
        ```
dpkd/transformers/src/transformers/models/tvlt/modeling_tvlt.py