src/transformers/models/speech_to_text_2/tokenization_speech_to_text_2.py/Speech2Text2Tokenizer/_tokenize
class Speech2Text2Tokenizer:    def _tokenize(self, text):
        """Tokenize a string."""

        if self.bpe_ranks is None:
            raise ValueError(
                "This tokenizer was instantiated without a `merges.txt` file, so"
                " that it can only be used for decoding, not for encoding."
                "Make sure to provide `merges.txt` file at instantiation to enable "
                "encoding."
            )

        if self.do_lower_case:
            text = text.lower()

        text = text.split()

        split_tokens = []
        for token in text:
            if token:
                split_tokens.extend(list(self.bpe(token).split(" ")))

        return split_tokens