def load_featurizer():
  """Loads a featurizer from hyperparams specified in model_dir."""
  params_path = os.path.join(FLAGS.model_dir, "estimator_params.json")
  with tf.gfile.GFile(params_path) as f:
    params = json.load(f)

  tokenizer = featurization.Tokenizer(
      vocab_path=params["vocab_path"], do_lower_case=params["do_lower_case"])

  featurizer = featurization.Featurizer(
      query_seq_len=params["query_seq_len"],
      candidate_seq_len=params["candidate_seq_len"],
      num_candidates=params["num_candidates"],
      max_masks=params["max_masks"],
      tokenizer=tokenizer,
      separate_candidate_segments=params["separate_candidate_segments"])

  logging.info("Loaded featurizer.")
  return featurizer
Loads a featurizer from hyperparams specified in model_dir.
language/realm/refresh_doc_embeds.py