def generate_realm_examples_with_model_refresh():
  """Generates REALM examples, checking for new models periodically."""
  featurizer = load_featurizer()

  # Load a list of Documents (candidates for retrieval).
  doc_shard_paths = tf.io.matching_files(FLAGS.retrieval_corpus_path).numpy()
  doc_shard_paths = sorted([p.decode() for p in doc_shard_paths])
  docs = retrieval.load_documents_from_shards(doc_shard_paths, num_processes=12)

  while True:
    retriever, model_timestamp = load_latest_retriever(docs, featurizer)
    examples = generate_realm_examples(featurizer, retriever, model_timestamp)
    for ex in examples:
      yield ex
Generates REALM examples, checking for new models periodically.
language/realm/example_generator.py