def processTextInput(self, text):
        """Generate source tokens from text input"""
        if self.pre_tokenizer is not None:
            text = self.pre_tokenizer.encode(text)
        if self.bpe_tokenizer is not None:
            text = self.bpe_tokenizer.encode(text)
        target = self.src_dict.encode_line(
            text, add_if_not_exist=False, append_eos=True
        ).long()
        target = fairseq_data_utils.collate_tokens(
            [target],
            self.src_dict.pad(),
            self.src_dict.eos(),
            left_pad=False,
            move_eos_to_beginning=False,
        )
        src_lengths = torch.tensor([target.size(1)], dtype=torch.long)
        prev_output_tokens = None
        sample = {
            "net_input": {
                "src_tokens": target,
                "src_lengths": src_lengths,
                "prev_output_tokens": prev_output_tokens,
            }
        }
        sample = utils.move_to_cuda(sample) if self.use_cuda else sample
        return sample
Generate source tokens from text input
understand_icl/fairseq/examples/speech_to_speech/benchmarking/core.py