def text2text_generate_encoded(sample_generator, vocab, targets_vocab=None):
  """Encode TranslateMultilingual samples from the generator with the vocab."""
  targets_vocab = targets_vocab or vocab
  for sample in sample_generator:
    # Encode sequences.
    sample["inputs"] = vocab.encode(sample["inputs"])
    sample["targets"] = targets_vocab.encode(sample["targets"])
    # Add EOS ids.
    sample["inputs"].append(text_encoder.EOS_ID)
    sample["targets"].append(text_encoder.EOS_ID)
    # Encode language tags.
    sample["input_tags"] = [get_tag_id(sample["input_tags"])]
    sample["target_tags"] = [get_tag_id(sample["target_tags"])]
    sample["all_tags"] = [get_tag_id(tag) for tag in sample["all_tags"]]
    yield sample
Encode TranslateMultilingual samples from the generator with the vocab.
language/labs/consistent_zero_shot_nmt/data_generators/translate_multilingual.py