def _generate_samples(self, data_dir, tmp_dir, dataset_split,
                        auxiliary_tags=None,
                        compile_data_fn=None):
    """Compile data and return an iterator over samples."""
    del data_dir  # Unused.
    if auxiliary_tags is None:
      auxiliary_tags = LANG_TAGS
    if compile_data_fn is None:
      compile_data_fn = translate.compile_data
    datasets = self.source_data_files(dataset_split)
    tag = "train" if dataset_split == problem.DatasetSplit.TRAIN else "dev"
    data_path = compile_data_fn(
        tmp_dir, datasets, "%s-compiled-%s" % (self.name, tag))
    return text2text_txt_iterator(
        source_txt_path=data_path + ".src",
        target_txt_path=data_path + ".tgt",
        auxiliary_tags=auxiliary_tags)
Compile data and return an iterator over samples.
language/labs/consistent_zero_shot_nmt/data_generators/translate_multilingual.py