def _compile_data(data_path, tmp_dir, datasets, filename):
  """Concatenate all `datasets` and save to `filename`."""
  filename = os.path.join(tmp_dir, filename)
  src_fname = filename + ".src"
  tgt_fname = filename + ".tgt"
  if tf.gfile.Exists(src_fname) and tf.gfile.Exists(tgt_fname):
    tf.logging.info("Skipping compile data, found files:\n%s\n%s",
                    src_fname, tgt_fname)
    return filename
  with tf.gfile.GFile(src_fname, mode="w") as src_resfile:
    with tf.gfile.GFile(tgt_fname, mode="w") as tgt_resfile:
      for d in datasets:
        logging.info("Loading %s-%s...", d["src_lang"], d["tgt_lang"])
        # Load source and target lines.
        src_fpath = os.path.join(data_path + d["src_fname"])
        tgt_fpath = os.path.join(data_path + d["tgt_fname"])
        src_lines = _parse_lines(src_fpath)
        tgt_lines = _parse_lines(tgt_fpath)
        assert len(src_lines) == len(tgt_lines)
        logging.info("...loaded %d parallel sentences", len(src_lines))
        # Prepend tags to each source and target line.
        src_lines = [d["src_lang"] + l for l in src_lines]
        tgt_lines = [d["tgt_lang"] + l for l in tgt_lines]
        # Write preprocessed source and target lines.
        logging.info("...writing preprocessed files")
        for src_line, tgt_line in zip(src_lines, tgt_lines):
          src_resfile.write(src_line)
          src_resfile.write("\n")
          tgt_resfile.write(tgt_line)
          tgt_resfile.write("\n")
  return filename
Concatenate all `datasets` and save to `filename`.
language/labs/consistent_zero_shot_nmt/data_generators/translate_uncorpus.py