def encode_source(self, src):
        """Method to generate source tokens from a string"""
        if self.tokenizer is not None:
            src = self.tokenizer.encode(src)
        if self.bpe is not None:
            src = self.bpe.encode(src)
        src_tokens = self.task.source_dictionary.encode_line(src).long()
        src_lens = src_tokens.size(0)
        return {
            "net_input": {
                "src_tokens": src_tokens.view(1, src_lens),
                "src_lengths": torch.tensor([src_lens]),
            }
        }
Method to generate source tokens from a string
understand_icl/fairseq/examples/speech_to_speech/benchmarking/core.py