def _count_tokens(text: str) -> int:
        """
        This can take a surprisingly long time on longer docs so we cache it, and we need to call it on every doc, regardless of size.
        """
        encoder = Settings.tokenizer
        tokens = encoder(text)
        return len(tokens)

        This can take a surprisingly long time on longer docs so we cache it, and we need to call it on every doc, regardless of size.
        
llama-index-core/llama_index/core/extractors/document_context.py