def get_default_tokenizer() -> Callable:
    """
    Get default tokenizer.

    NOTE: taken from https://www.pinecone.io/learn/hybrid-search-intro/.

    """
    from transformers import BertTokenizerFast

    orig_tokenizer = BertTokenizerFast.from_pretrained("bert-base-uncased")

    def _tokenizer(texts: List[str]) -> List[List[int]]:
        return orig_tokenizer(texts, padding=True, truncation=True, max_length=512)[
            "input_ids"
        ]

    return _tokenizer

    Get default tokenizer.

    NOTE: taken from https://www.pinecone.io/learn/hybrid-search-intro/.

    
llama-index-integrations/vector_stores/llama-index-vector-stores-pinecone/llama_index/vector_stores/pinecone/utils.py