def read_file(filename):
  """Read the contents of filename (str) and split into documents by chapter."""

  all_documents = []
  # Input file format:
  # See internal README for an example
  # Each file is a list of lines. Each line is a sentence. Empty lines denote
  # delimiters between paragraphs. Each paragraph is its own document.

  all_documents = [[]]
  with tf.gfile.GFile(filename, "r") as reader:
    for line in reader:
      line = line.strip()
      if not line:
        if all_documents[-1]:
          all_documents.append([])
        continue
      all_documents[-1].append(line)
      if len(all_documents[-1]) == FLAGS.max_para_length:
        all_documents.append([])

  # Remove empty documents
  all_documents = [x for x in all_documents if len(x) >= 8]

  return all_documents
Read the contents of filename (str) and split into documents by chapter.
language/conpono/cpc/preproc/ccnews_preproc_pipeline.py