def generate_sentences(self, title, text):
    """Generate sentences in each block from text."""
    title_length = len(self._tokenizer.tokenize(title))
    current_token_count = 0
    current_block_sentences = []
    for sentence in self._sentence_splitter.tokenize(text):
      num_tokens = len(self._tokenizer.tokenize(sentence))
      # Hypothetical sequence [CLS] <title> [SEP] <current> <next> [SEP].
      hypothetical_length = 3 + title_length + current_token_count + num_tokens
      if hypothetical_length <= self._max_block_length:
        current_token_count += num_tokens
        current_block_sentences.append(sentence)
      else:
        yield current_block_sentences
        current_token_count = num_tokens
        current_block_sentences = []
        current_block_sentences.append(sentence)
    if current_block_sentences:
      yield current_block_sentences
Generate sentences in each block from text.
language/orqa/preprocessing/wiki_preprocessor.py