src/transformers/models/bertweet/tokenization_bertweet.py/BertweetTokenizer/_tokenize
class BertweetTokenizer:    def _tokenize(self, text):
        """Tokenize a string."""
        if self.normalization:  # Perform Tweet normalization before performing BPE
            text = self.normalizeTweet(text)

        split_tokens = []
        words = re.findall(r"\S+\n?", text)
        for token in words:
            split_tokens.extend([t for t in self.bpe(token).split(" ")])
        return split_tokens