src/transformers/models/gptsan_japanese/tokenization_gptsan_japanese.py/GPTSanJapaneseTokenizer/prepare_for_tokenization
class GPTSanJapaneseTokenizer:    def prepare_for_tokenization(self, text, prefix_text=None, add_sep_token=None, **kwargs):
        # GPTSAN inserts extra SEP tokens in Prefix-LM in addition to SOT for text generation.
        # SOT at the beginning of the text, and SEP at the separator between the Prefix part and the rest.
        if add_sep_token is None:
            add_sep_token = self.sep_token not in text  # If insert un-prefix position explicitly
        prepared = self.bos_token if self.bos_token in self.vocab else ""
        prepared += prefix_text if prefix_text is not None else ""
        if add_sep_token:
            prepared += self.sep_token if self.sep_token in self.vocab else ""
        prepared += text
        return (prepared, kwargs)