def __init__(
            self,
            corpus: Union[ParsedCorpus, OffsetCorpus],
            model: PreTrainedModel,
            tokenizer: PreTrainedTokenizerFast,
            feats_from_spacy_doc: Optional[FeatsFromSpacyDoc] = None,
            doc_segmenter: Optional[Callable[[spacy.tokens.doc.Doc], Iterable[spacy.tokens.span.Span]]] = None,
            token_normalizer: Optional[Callable[[spacy.tokens.Token], str]] = None,
            use_all_tokens_for_category_embeddings: bool = True,
            weights_to_embeddings: Callable = last_hidden_state_embeddings
    ):
        """

        :param corpus:
        :param model:
        :param tokenizer:
        :param feats_from_spacy_doc: if none, use offsets
        :param doc_segmenter:
        :param token_normalizer: Function which takes spacy token and returns a string.
          Not used for embedding calculation, just feature representation. Lowercases all by default.
        :param use_all_tokens_for_category_embeddings: bool, Use all tokens in segment for category embeddings?
        """
        self.corpus = corpus
        self.model = model
        self.tokenizer = tokenizer
        self.feats_from_spacy_doc = feats_from_spacy_doc
        if not isinstance(corpus, OffsetCorpus):
            raise Exception("Since the corpus object is not an offset corpus, feats_from_spacy_doc needs to be passed.")
        self.doc_segmenter = doc_sentence_splitter if doc_segmenter is None else doc_segmenter
        self.token_normalizer = lambda x: x.lower_ if token_normalizer is None else token_normalizer
        self.weights_to_embeddings = weights_to_embeddings
        self.use_all_tokens_for_category_embeddings = use_all_tokens_for_category_embeddings


        :param corpus:
        :param model:
        :param tokenizer:
        :param feats_from_spacy_doc: if none, use offsets
        :param doc_segmenter:
        :param token_normalizer: Function which takes spacy token and returns a string.
          Not used for embedding calculation, just feature representation. Lowercases all by default.
        :param use_all_tokens_for_category_embeddings: bool, Use all tokens in segment for category embeddings?
        
scattertext/contextual_embeddings/corpus_runing_stats_factory.py