def _compile_data(tmp_dir, datasets, filename):
  """Concatenate all `datasets` and save to `filename`."""
  filename = os.path.join(tmp_dir, filename)
  src_fname = filename + ".src"
  tgt_fname = filename + ".tgt"
  if tf.gfile.Exists(src_fname) and tf.gfile.Exists(tgt_fname):
    tf.logging.info("Skipping compile data, found files:\n%s\n%s",
                    src_fname, tgt_fname)
    return filename
  with tf.gfile.GFile(src_fname, mode="w") as src_resfile:
    with tf.gfile.GFile(tgt_fname, mode="w") as tgt_resfile:
      for d in datasets:
        logging.info("Loading %s-%s...", d["src_lang"], d["tgt_lang"])
        # Load source and target lines.
        src_fpath = os.path.join(FLAGS.iwslt17_orig_data_path, d["src_fname"])
        tgt_fpath = os.path.join(FLAGS.iwslt17_orig_data_path, d["tgt_fname"])
        src_lines = _parse_lines(src_fpath)
        tgt_lines = _parse_lines(tgt_fpath)
        assert len(src_lines) == len(tgt_lines)
        logging.info("...loaded %d parallel sentences", len(src_lines))
        # Filter overlap, if necessary.
        if "src_remove" in d:
          logging.info("...filtering src overlap")
          src_remove_path = os.path.join(FLAGS.iwslt17_overlap_data_path,
                                         d["src_remove"])
          src_remove = set(_parse_lines(src_remove_path))
          logging.info("...total overlapping lines: %d", len(src_remove))
          logging.info("...lines before filtering: %d", len(src_lines))
          src_tgt_lines = [
              (src_line, tgt_line)
              for src_line, tgt_line in zip(src_lines, tgt_lines)
              if src_line not in src_remove]
          src_lines, tgt_lines = map(list, zip(*src_tgt_lines))
          logging.info("...lines after filtering: %d", len(src_lines))
        if "tgt_remove" in d:
          logging.info("...filtering tgt overlap")
          tgt_remove_path = os.path.join(FLAGS.iwslt17_overlap_data_path,
                                         d["tgt_remove"])
          tgt_remove = set(_parse_lines(tgt_remove_path))
          logging.info("...total overlapping lines: %d", len(tgt_remove))
          logging.info("...lines before filtering: %d", len(src_lines))
          src_tgt_lines = [
              (src_line, tgt_line)
              for src_line, tgt_line in zip(src_lines, tgt_lines)
              if tgt_line not in tgt_remove]
          src_lines, tgt_lines = map(list, zip(*src_tgt_lines))
          logging.info("...lines after filtering: %d", len(src_lines))
        assert len(src_lines) == len(tgt_lines)
        # Prepend tags to each source and target line.
        src_lines = [d["src_lang"] + l for l in src_lines]
        tgt_lines = [d["tgt_lang"] + l for l in tgt_lines]
        # Write preprocessed source and target lines.
        logging.info("...writing preprocessed files")
        for src_line, tgt_line in zip(src_lines, tgt_lines):
          src_resfile.write(src_line)
          src_resfile.write("\n")
          tgt_resfile.write(tgt_line)
          tgt_resfile.write("\n")
  return filename
Concatenate all `datasets` and save to `filename`.
language/labs/consistent_zero_shot_nmt/data_generators/translate_iwslt17.py