def generate_queries_and_candidates(featurizer, retriever):
  """Returns a generator over (Query, List[Document]) pairs."""
  queries = generate_queries(featurizer)

  for query_batch in batch(queries, FLAGS.retrieval_batch_size):
    # candidates_batch has shape [batch_size, num_candidates]
    # Each element is a Document.
    candidates_batch = retriever.retrieve(query_batch)

    for query, cands in zip(query_batch, candidates_batch):
      # If the retrieval index contains duplicate docs, we may end up retrieving
      # duplicates. Skip such examples.
      cand_uids = [c.uid for c in cands]
      if len(cand_uids) != len(set(cand_uids)):
        STATS['duplicate_candidates'] += 1
        continue

      yield (query, postprocess_candidates(cands, query))
Returns a generator over (Query, List[Document]) pairs.
language/realm/example_generator.py