def _tokenize(self, data):
        """
        data:
        {
            'source': list[Path],
            'source_lang': str,
            'count': int,
            'weight': float,
            'name': str,
        }
        """
        dataset = list(
            zip(
                data["source"],
                itertools.repeat(data["source_lang"]),
            )
        )

        if self.shuffle:
            chunk_files = iterators.InfinitePermutationSourceIterator(
                dataset,
                seed=self.seed,
                shuffle=self.shuffle,
                num_instances=self.num_shards,
                instance_rank=self.shard_id,
            )
        else:
            chunk_files = iterators.ChunkedSourceIterator(
                dataset,
                num_instances=self.num_shards,
                instance_rank=self.shard_id,
            )

        tokenized_lines = iterators.SelectManyIterator(
            chunk_files, lambda files: self._read_from_files(*files)
        )
        tokenized_lines = iterators.SamplingRandomMapIterator(
            tokenized_lines, self._prepare, self.seed
        )

        return tokenized_lines

        data:
        {
            'source': list[Path],
            'source_lang': str,
            'count': int,
            'weight': float,
            'name': str,
        }
        
kosmos-2/torchscale/examples/fairseq/tasks/data/mlm_loader.py