def forward(
        self,
        pixel_values: torch.FloatTensor,
        audio_values: torch.FloatTensor,
        pixel_mask: Optional[torch.FloatTensor] = None,
        audio_mask: Optional[torch.FloatTensor] = None,
        labels: Optional[torch.LongTensor] = None,
        pixel_values_mixed: Optional[torch.FloatTensor] = None,
        pixel_mask_mixed: Optional[torch.FloatTensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[torch.FloatTensor], TvltForPreTrainingOutput]:
        r"""
        pixel_values_mixed (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, height, width)`):
            Pixel values that mix positive and negative samples in Tvlt vision-audio matching. Audio values can be
            obtained using [`TvltProcessor`]. See [`TvltProcessor.__call__`] for details.

        pixel_mask_mixed (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel masks of pixel_values_mixed. Pixel values mixed can be obtained using [`TvltProcessor`]. See
            [`TvltProcessor.__call__`] for details.

        labels (`torch.LongTensor` of shape `(batch_size, num_labels)`, *optional*):
            Labels for computing the vision audio matching loss. Indices should be in `[0, 1]`. num_labels has to be 1.

        Return:

        Examples:

        ```python
        >>> from transformers import TvltProcessor, TvltForPreTraining
        >>> import numpy as np
        >>> import torch

        >>> num_frames = 8
        >>> images = list(np.random.randn(num_frames, 3, 224, 224))
        >>> images_mixed = list(np.random.randn(num_frames, 3, 224, 224))
        >>> audio = list(np.random.randn(10000))
        >>> processor = TvltProcessor.from_pretrained("ZinengTang/tvlt-base")
        >>> model = TvltForPreTraining.from_pretrained("ZinengTang/tvlt-base")
        >>> input_dict = processor(
        ...     images, audio, images_mixed, sampling_rate=44100, mask_pixel=True, mask_audio=True, return_tensors="pt"
        ... )

        >>> outputs = model(**input_dict)
        >>> loss = outputs.loss
        ```"""
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        total_loss = 0.0

        if self.task_matching:
            if labels is None:
                raise ValueError("Matching task requires labels")
            if pixel_values_mixed is None:
                raise ValueError("Matching task requires pixel_values_mixed")

            outputs = self.tvlt(
                pixel_values_mixed,
                audio_values,
                pixel_mask=pixel_mask_mixed,
                audio_mask=audio_mask,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )

            sequence_output = outputs[0]
            matching_logits = self.matching_head(sequence_output)

            loss_fct = BCEWithLogitsLoss()
            loss = loss_fct(matching_logits.view(-1), labels.view(-1))
            total_loss += loss

        pixel_logits = None
        audio_logits = None
        if self.task_mae and self.training:
            outputs = self.tvlt(
                pixel_values,
                audio_values,
                pixel_mask=pixel_mask,
                audio_mask=audio_mask,
                mask_pixel=True,
                mask_audio=True,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
            pixel_sequence_output = outputs.last_pixel_hidden_state if return_dict else outputs[1]
            audio_sequence_output = outputs.last_audio_hidden_state if return_dict else outputs[2]
            pixel_label_masks = outputs.pixel_label_masks if return_dict else outputs[3]
            audio_label_masks = outputs.audio_label_masks if return_dict else outputs[4]
            pixel_ids_restore = outputs.pixel_ids_restore if return_dict else outputs[5]
            audio_ids_restore = outputs.audio_ids_restore if return_dict else outputs[6]

            pixel_decoder_input = self.encoder_to_decoder(
                pixel_sequence_output
            )  # [batch_size, num_masked_pixel_patches, decoder_hidden_size]
            audio_decoder_input = self.encoder_to_decoder(
                audio_sequence_output
            )  # [batch_size, num_masked_audio_patches, decoder_hidden_size]
            num_frames = pixel_values.size(1)
            pixel_decoder_input = self.concatenate_mask(self.pixel_mask_token, pixel_decoder_input, pixel_ids_restore)
            pixel_decoder_input = pixel_decoder_input + self.decoder_pixel_pos_embed.repeat(1, num_frames, 1)
            pixel_decoder_input = pixel_decoder_input + torch.repeat_interleave(
                self.decoder_temporal_embed[:, :num_frames], self.num_patches_per_image, dim=1
            )
            pixel_decoder_input = pixel_decoder_input + self.decoder_pixel_type_embed
            pixel_decoder_outputs = self.decoder(pixel_decoder_input)
            pixel_logits = self.pixel_mae_head(pixel_decoder_outputs.logits)

            audio_decoder_input = self.concatenate_mask(self.audio_mask_token, audio_decoder_input, audio_ids_restore)
            num_time_patches = audio_decoder_input.size(1) // self.num_freq_patches
            audio_decoder_input = audio_decoder_input + self.decoder_freq_embed.repeat(1, num_time_patches, 1)
            audio_decoder_input = audio_decoder_input + torch.repeat_interleave(
                self.decoder_audio_pos_embed[:, :num_time_patches], self.num_freq_patches, dim=1
            )
            audio_decoder_input = audio_decoder_input + self.decoder_audio_type_embed
            audio_decoder_outputs = self.decoder(audio_decoder_input)
            audio_logits = self.audio_mae_head(audio_decoder_outputs.logits)

            loss = self.pixel_mae_loss(pixel_values, pixel_logits, pixel_label_masks) + self.audio_mae_loss(
                audio_values, audio_logits, audio_label_masks
            )
            total_loss += loss

        if not return_dict:
            output = (matching_logits, pixel_logits, audio_logits) + outputs[7:]
            return ((total_loss,) + output) if loss is not None else output

        return TvltForPreTrainingOutput(
            loss=total_loss,
            matching_logits=matching_logits,
            pixel_logits=pixel_logits,
            audio_logits=audio_logits,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

        pixel_values_mixed (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, height, width)`):
            Pixel values that mix positive and negative samples in Tvlt vision-audio matching. Audio values can be
            obtained using [`TvltProcessor`]. See [`TvltProcessor.__call__`] for details.

        pixel_mask_mixed (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel masks of pixel_values_mixed. Pixel values mixed can be obtained using [`TvltProcessor`]. See
            [`TvltProcessor.__call__`] for details.

        labels (`torch.LongTensor` of shape `(batch_size, num_labels)`, *optional*):
            Labels for computing the vision audio matching loss. Indices should be in `[0, 1]`. num_labels has to be 1.

        Return:

        Examples:

        ```python
        >>> from transformers import TvltProcessor, TvltForPreTraining
        >>> import numpy as np
        >>> import torch

        >>> num_frames = 8
        >>> images = list(np.random.randn(num_frames, 3, 224, 224))
        >>> images_mixed = list(np.random.randn(num_frames, 3, 224, 224))
        >>> audio = list(np.random.randn(10000))
        >>> processor = TvltProcessor.from_pretrained("ZinengTang/tvlt-base")
        >>> model = TvltForPreTraining.from_pretrained("ZinengTang/tvlt-base")
        >>> input_dict = processor(
        ...     images, audio, images_mixed, sampling_rate=44100, mask_pixel=True, mask_audio=True, return_tensors="pt"
        ... )

        >>> outputs = model(**input_dict)
        >>> loss = outputs.loss
        ```
dpkd/transformers/src/transformers/models/tvlt/modeling_tvlt.py