def load_vq_tokenizer(config: ml_collections.ConfigDict,
                      *,
                      tokenizer_only: bool = False,
                      log_parameters: bool = True) -> Dict[str, TokenizerFn]:
  """Loads the pretrained the VQ model as tokenizer."""
  if config.get('debug_pseudo_vq', False):
    return get_pseudo_tokenizer_fn(config)
  vq_params, vq_config = load_vq_model_params(config)
  sync_vq_model_config(config, vq_config)
  vq_model = train_utils.get_vq_model(vq_config)['generator']
  if log_parameters:
    logging.info('logging vq_model parameters')
    parameter_overview.log_parameter_overview(vq_params)
  enc_params, _ = vq_params.pop('decoder')
  tokenizer_fn = functools.partial(
      vq_model.apply, {'params': enc_params}, method=vq_model.encode_to_indices)
  if tokenizer_only:
    return {'tokenizer': tokenizer_fn}
  dec_params, _ = vq_params.pop('encoder')
  detokenizer_fn = functools.partial(
      vq_model.apply, {'params': dec_params},
      method=vq_model.decode_from_indices)
  return {'tokenizer': tokenizer_fn, 'detokenizer': detokenizer_fn}
Loads the pretrained the VQ model as tokenizer.
videogvt/train_lib/maskgvt_train_lib.py