def create_fixed_tokenizer():
  """Function needed for extending tokenizer with additional chat tokens"""
  import extra.junk.sentencepiece_model_pb2 as spb2
  tokenizer_path = fetch("https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v0.4/resolve/main/tokenizer.model")
  if SentencePieceProcessor(model_file=str(tokenizer_path)).vocab_size() != 32003:
    print("creating fixed tokenizer")
    mp = spb2.ModelProto()
    mp.ParseFromString(tokenizer_path.read_bytes())
    # https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v0.4/blob/main/added_tokens.json
    mp.pieces.append(spb2.ModelProto.SentencePiece(piece="[PAD]", score=0))
    mp.pieces.append(spb2.ModelProto.SentencePiece(piece="<|im_start|>", score=0))
    mp.pieces.append(spb2.ModelProto.SentencePiece(piece="<|im_end|>", score=0))
    tokenizer_path.write_bytes(mp.SerializeToString())
  return tokenizer_path
Function needed for extending tokenizer with additional chat tokens
examples/conversation.py