def __init__(
        self,
        model_name: Optional[str] = None,
        encoding_name: Optional[str] = None,
        add_bos_token: bool = False,
        add_eos_token: bool = False,
        use_default_system_prompt: bool = False,
        unk_token: Optional[str] = '<|endoftext|>',
        eos_token: Optional[str] = '<|endoftext|>',
        bos_token: Optional[str] = '<|endoftext|>',
        pad_token: Optional[str] = None,
        errors: str = 'replace',
        chat_template: Optional[str] = None,
        **kwargs: Any,
    ):
        """Constructor creates a tiktoken tokenizer to use as the underlying.

        tokenizer.

        Args:
            model_name (Optional[str], optional): The name of the model to load from tiktoken. Defaults to None.
                Either model_name or encoding_name must be set, but not both.
            encoding_name (Optional[str], optional): The name of the encoding to load from tiktoken. Defaults to None.
                Either model_name or encoding_name must be set, but not both.
            add_bos_token (bool, optional): Whether to add bos tokens. Defaults to False.
            add_eos_token (bool, optional): Whether to add eos tokens. Defaults to False.
            use_default_system_prompt (bool, optional): Use the default system prompt or not. Defaults to False.
            unk_token (Optional[str], optional): The unk token. Defaults to '<|endoftext|>'.
            eos_token (Optional[str], optional): The eos token. Defaults to '<|endoftext|>'.
            bos_token (Optional[str], optional): The bos token. Defaults to '<|endoftext|>'.
            pad_token (Optional[str], optional): The pad token. Defaults to None.
            errors (str, optional): Paradigm to follow when decoding bytes to UTF-8. See
                [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
                Defaults to `"replace"`.
            chat_template (Optional[str], optional): The Hugging Face chat template. Default will use the ``default_chat_template``
                set on this class.
            kwargs (Any): Other relevant keyword arguments.
        """
        try:
            import tiktoken
        except:
            raise ImportError(
                'You need to install tiktoken to use TiktokenTokenizerWrapper.',
            )

        # Workaround to make tiktokenizer picklable.
        # https://github.com/huggingface/datasets/issues/5536#issuecomment-1682309347
        # There is an open PR from HF to add this to tiktoken: https://github.com/openai/tiktoken/pull/181
        import copyreg
        import functools

        from tiktoken import Encoding  # type: ignore (thirdParty)

        def pickle_Encoding(enc: Encoding):
            return (
                functools.partial(
                    Encoding,
                    enc.name,
                    pat_str=enc._pat_str,
                    mergeable_ranks=enc._mergeable_ranks,
                    special_tokens=enc._special_tokens,
                ),
                (),
            )

        copyreg.pickle(Encoding, pickle_Encoding)

        if model_name is not None and encoding_name is not None:
            raise ValueError(
                'You need to specify either model_name or encoding_name, not both.',
            )

        self.model_name = model_name
        self.encoding_name = encoding_name

        if self.model_name is not None:
            self.encoding = tiktoken.encoding_for_model(  # type: ignore (thirdParty)
                self.model_name)
        elif self.encoding_name is not None:
            self.encoding = tiktoken.get_encoding(  # type: ignore (thirdParty)
                self.encoding_name)
        else:
            raise ValueError(
                'You need to specify either model_name or encoding_name.',
            )

        self.add_bos_token = add_bos_token
        self.add_eos_token = add_eos_token
        self.use_default_system_prompt = use_default_system_prompt

        self.byte_encoder = bytes_to_unicode()
        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
        self.errors = errors

        self.decoder: dict[int, str] = {}
        for i in range(self.encoding.n_vocab):
            try:
                self.encoding.decode_single_token_bytes(i)
            except KeyError:
                continue
            # Taken from
            # https://gist.github.com/xenova/a452a6474428de0182b17605a98631ee
            decoding = ''.join([
                bytes_to_unicode()[ord(char)] for char in
                self.encoding.decode_single_token_bytes(i).decode('latin-1')
            ])
            self.decoder[i] = decoding

        self.encoder: dict[str, int] = {}
        for i in range(self.encoding.n_vocab):
            if i in self.decoder:
                self.encoder[self.decoder[i]] = i

        super().__init__(
            model_name=model_name,
            encoding_name=encoding_name,
            add_bos_token=add_bos_token,
            add_eos_token=add_eos_token,
            use_default_system_prompt=use_default_system_prompt,
            unk_token=unk_token,
            eos_token=eos_token,
            bos_token=bos_token,
            pad_token=pad_token,
            errors=errors,
            chat_template=chat_template or self.default_chat_template,
            **kwargs,
        )
Constructor creates a tiktoken tokenizer to use as the underlying.

        tokenizer.

        Args:
            model_name (Optional[str], optional): The name of the model to load from tiktoken. Defaults to None.
                Either model_name or encoding_name must be set, but not both.
            encoding_name (Optional[str], optional): The name of the encoding to load from tiktoken. Defaults to None.
                Either model_name or encoding_name must be set, but not both.
            add_bos_token (bool, optional): Whether to add bos tokens. Defaults to False.
            add_eos_token (bool, optional): Whether to add eos tokens. Defaults to False.
            use_default_system_prompt (bool, optional): Use the default system prompt or not. Defaults to False.
            unk_token (Optional[str], optional): The unk token. Defaults to '<|endoftext|>'.
            eos_token (Optional[str], optional): The eos token. Defaults to '<|endoftext|>'.
            bos_token (Optional[str], optional): The bos token. Defaults to '<|endoftext|>'.
            pad_token (Optional[str], optional): The pad token. Defaults to None.
            errors (str, optional): Paradigm to follow when decoding bytes to UTF-8. See
                [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
                Defaults to `"replace"`.
            chat_template (Optional[str], optional): The Hugging Face chat template. Default will use the ``default_chat_template``
                set on this class.
            kwargs (Any): Other relevant keyword arguments.
        
llmfoundry/tokenizers/tiktoken.py