def forward(
        self,
        caps,
        cmasks,
        vfeats,
        vmasks,
        attention_mask,
        video_label=None,
        text_label=None,
        **kwargs
    ):
        """use pre-trained LM header for generation."""
        attention_mask, token_type_ids = self._mm_on_the_fly(
            cmasks, vmasks, attention_mask)

        outputs = self.mm_encoder(
            input_ids=caps,
            input_video_embeds=vfeats,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            masked_lm_labels=text_label,
        )
        return {"logits": outputs[0]}
use pre-trained LM header for generation.
understand_icl/fairseq/examples/MMPT/mmpt/models/mmfusionnlg.py