src/transformers/models/roformer/tokenization_roformer.py/RoFormerTokenizer/_tokenize
class RoFormerTokenizer:    def _tokenize(self, text, use_jieba=True):
        split_tokens = []
        if use_jieba:
            for wholword in self.jieba.cut(text, False):
                if wholword in self.vocab:
                    split_tokens.append(wholword)
                else:
                    # use bert tokenizer to _tokenize
                    char_list = self._tokenize(wholword, use_jieba=False)
                    split_tokens.extend(char_list)
        else:
            if self.do_basic_tokenize:
                for token in self.basic_tokenizer.tokenize(text, never_split=self.all_special_tokens):
                    # If the token is part of the never_split set
                    if token in self.basic_tokenizer.never_split:
                        split_tokens.append(token)
                    else:
                        split_tokens += self.wordpiece_tokenizer.tokenize(token)
            else:
                split_tokens = self.wordpiece_tokenizer.tokenize(text)
        return split_tokens