def generate_samples_v2(queries,
                        passages,
                        neighbors_path,
                        is_training,
                        processor,
                        q_ids,
                        train_eval=False):
  """Enum. (query, List<passage>, List<passage>, List<label>, List<score>)."""

  n_processed = 0
  n_found = 0
  seq_qids = set(q_ids)
  with open(neighbors_path) as fid:
    neighbors = json.load(fid)

  for q_id, q_text in queries.items():
    # get the retrieved neighbors
    if q_id not in seq_qids:
      continue
    if n_processed % 10000 == 0:
      print("processed " + str(n_processed) + " found " + str(n_found))
    n_processed = n_processed + 1
    if not have_neighbors(q_id, neighbors):
      print("neighbors for " + str(q_id) + " not found.")
      continue
    cand_list_ids, cand_list_labels, cand_list_scores, rand_start \
        = augmented_neighbors_list(
            q_id, neighbors, is_training, processor, train_eval=train_eval)
    # get the text of the neigbors
    if not cand_list_ids:
      continue
    if 1 in cand_list_labels:
      n_found = n_found + 1
    # sub-select

    positive, negative = sub_select_examples(cand_list_ids, cand_list_labels,
                                             cand_list_scores, rand_start,
                                             is_training)

    # get information for the positive passages

    passage_infos_positive = wrap_with_passage_texts(positive, passages)
    passage_infos_negative = wrap_with_passage_texts(negative, passages)
    yield QueryInfo(q_id, q_text), passage_infos_positive, \
        passage_infos_negative
Enum. (query, List<passage>, List<passage>, List<label>, List<score>).
language/multivec/preprocessing/create_training_data.py