def predict_loop(
        self,
        model,
        eval_dataloader,
        output_file="retrieval.npy"
    ):
        """on-the-fly prediction on a single gpu."""
        full_scores = []
        texts = []
        model.eval()
        model = model.cuda()
        with torch.no_grad():
            for data in eval_dataloader:
                # convert to dict.
                if not isinstance(data, dict):
                    data = {
                        "caps": data[0],
                        "cmasks": data[1],
                        "vfeats": data[2],
                        "vmasks": data[3],
                        "video_id": data[4]
                    }
                data = self.to_ctx(data)
                outputs = model(**data)
                outputs.update(data)
                self(outputs, full_scores)
                for _cap in data["caps"]:
                    texts.append(
                        self.tokenizer.decode(_cap, skip_special_tokens=True)
                    )

        return self.finalize(full_scores, texts, output_file)
on-the-fly prediction on a single gpu.
understand_icl/fairseq/examples/MMPT/mmpt/evaluators/predictor.py