def read_file(filename):
  """Read the contents of filename (str) and split into documents by chapter."""

  all_documents = []
  document = []
  with tf.gfile.GFile(filename, "r") as reader:
    for line in reader:
      line = line.strip()
      line = tokenization.convert_to_unicode(line)
      line = line.replace(u"\u2018", "'").replace(u"\u2019", "'")
      sents = split_line_by_sentences(line)
      for sent_line in sents:
        if not sent_line or len(sent_line) < 4:  # Arbitrary min length for line
          continue
        if sent_line.lower()[:7] == "chapter":
          if document:
            all_documents.append(document)
            document = []
        else:
          document.append(sent_line)
        if len(document) == FLAGS.max_para_length:
          all_documents.append(document)
          document = []
    if document:
      all_documents.append(document)

  # Remove small documents
  all_documents = [x for x in all_documents if len(x) >= 8]

  return all_documents
Read the contents of filename (str) and split into documents by chapter.
language/conpono/cpc/preproc/raw_books_preproc_pipeline.py