def read_file(filename_tuple):
  """Read the contents of filename (str) and split into documents by chapter."""

  filename, other_filenames = filename_tuple

  all_documents = []
  document = []
  with tf.gfile.GFile(filename, "r") as reader:
    for line in reader:
      line = line.strip()
      if not line:
        continue
      if line.lower()[:7] == "chapter":
        if document:
          all_documents.append(document)
          document = []
      else:
        document.append(line)
    if document:
      all_documents.append(document)

  # Remove empty documents
  all_documents = [x for x in all_documents if x]

  for otherfilename in other_filenames:
    other_documents = []
    document = []
    with tf.gfile.GFile(otherfilename, "r") as reader:
      for line in reader:
        line = line.strip()
        if not line:
          continue
        if line.lower()[:7] == "chapter":
          if document:
            other_documents.append(document)
            document = []
        else:
          document.append(line)
      if document:
        other_documents.append(document)

  # Remove empty documents
  other_documents = [x for x in other_documents if x]

  # for each document grab 3 random docs
  all_docs_with_rands = []
  for doc in all_documents:
    all_docs_with_rands.append((doc, other_documents))

  return all_docs_with_rands
Read the contents of filename (str) and split into documents by chapter.
language/conpono/cpc/preproc/books_preproc_pipeline.py