src/transformers/models/biogpt/tokenization_biogpt.py/BioGptTokenizer/_tokenize
class BioGptTokenizer:    def _tokenize(self, text, bypass_tokenizer=False):
        """Returns a tokenized string."""
        if bypass_tokenizer:
            text = text.split()
        else:
            text = self.moses_tokenize(text, self.lang)

        split_tokens = []
        for token in text:
            if token:
                split_tokens.extend(list(self.bpe(token).split(" ")))

        return split_tokens