def truncate_text(self, text):
    """Truncates text to last complete sentence that fits in window."""
    # Break into sentences
    sentences = nltk.sent_tokenize(text)
    tokenized_sentences = [self.tokenizer.tokenize(x) for x in sentences]
    lengths = [x.shape[0] for x in tokenized_sentences]
    cutoff_index = 0
    cumulative_length = 0
    for length in lengths:
      cumulative_length += length
      if cumulative_length > self.max_article_length:
        break
      cutoff_index += 1
    return " ".join(sentences[:cutoff_index])
Truncates text to last complete sentence that fits in window.
language/fruit/beam_pipelines.py