def tokenize(self, doc):
        '''
        doc: str, text to be tokenized
        '''

        sents = []
        if self.lower_case:
            doc = doc.lower()
        decoded_text = self.decoder(doc)
        tokens = self.tokenizer.convert_ids_to_tokens(
            self.tokenizer(decoded_text)['input_ids'],
            skip_special_tokens=True
        )

        last_idx = 0
        toks = []
        for raw_token in tokens:
            token_surface_string = self._get_surface_string(raw_token)
            if ord(raw_token[0]) == 266:  # skip new lines
                last_idx += len(raw_token)
                continue
            try:
                token_idx = decoded_text.index(token_surface_string, last_idx)
            except Exception as e:
                print(decoded_text)
                print(token_surface_string)
                raise e
            toks.append(Tok(_get_pos_tag(token_surface_string),
                            token_surface_string.lower(),
                            raw_token.lower(),
                            ent_type='' if self.entity_type is None else self.entity_type.get(token_surface_string, ''),
                            tag='' if self.tag_type is None else self.tag_type.get(token_surface_string, ''),
                            idx=token_idx))
            last_idx = token_idx + len(token_surface_string)
            if token_surface_string in ['.', '!', '?']:  # idiot's sentence splitter
                sents.append(toks)
                toks = []

        if len(toks) > 0:
            sents.append(toks)
        return Doc(sents, decoded_text)

        doc: str, text to be tokenized
        
scattertext/tokenizers/transformers.py