def preprocess(self, story_lines, summary_lines=None):
        """preprocess multiple data points

           Args:
              story_lines (list of strings): List of sentences.
              targets (list of strings, optional): List of sentences.
                  Defaults to None, which means it doesn't include summary and is
                  not training data.

            Returns:
                If summary_lines is None, return list of list of token ids. Otherwise,
                return a tuple of (list of list of token ids, list of list of token ids).

        """
        story_lines_token_ids = []
        for line in story_lines:
            try:
                if len(line) <= 0:
                    continue
                story_lines_token_ids.append(
                    self.tokenizer.encode(line, max_length=self.max_src_len)
                )
            except:
                print(line)
                raise
        story_token_ids = [
            token for sentence in story_lines_token_ids for token in sentence
        ]
        if summary_lines:
            summary_lines_token_ids = []
            for line in summary_lines:
                try:
                    if len(line) <= 0:
                        continue
                    summary_lines_token_ids.append(
                        self.tokenizer.encode(line, max_length=self.max_tgt_len)
                    )
                except:
                    print(line)
                    raise
            summary_token_ids = [
                token for sentence in summary_lines_token_ids for token in sentence
            ]
            return story_token_ids, summary_token_ids
        else:
            return story_token_ids, None
preprocess multiple data points

           Args:
              story_lines (list of strings): List of sentences.
              targets (list of strings, optional): List of sentences.
                  Defaults to None, which means it doesn't include summary and is
                  not training data.

            Returns:
                If summary_lines is None, return list of list of token ids. Otherwise,
                return a tuple of (list of list of token ids, list of list of token ids).

        
utils_nlp/models/transformers/abstractive_summarization_bertsum.py