def get_examples_v2_iterate(self, is_train, max_cand, query_info,
                              passage_infos_positive, passage_infos_negative):
    """Get examples as a sequence of grouped candidates."""
    # we need to iterate over the candidates in groups of max_cand
    # for training, each example needs to have a positive and we have
    # a limit of train_records_per_query
    # for eval, we just list them out in any order
    if is_train:
      random.shuffle(passage_infos_positive)
      random.shuffle(passage_infos_negative)
      n_available = len(passage_infos_negative)
      n_instances = FLAGS.train_records_per_query
      if n_instances * (max_cand - 1) > n_available:
        n_instances = (int)(n_available / (max_cand - 1))
        print("reduced candidates " + str(n_instances))
      # start generating the instances
      instances = []
      for _ in range(n_instances):
        # initialize with empty lists
        instances.append([])
      # first add the positives
      pos_index = 0
      max_pos_index = 0  # the maximum index of a positive element added
      for i in range(n_instances):
        instances[i].append(passage_infos_positive[pos_index])
        if pos_index > max_pos_index:
          max_pos_index = pos_index
        pos_index += 1
        if pos_index >= len(passage_infos_positive):
          pos_index = 0

      # see if we have posititives left over
      if max_pos_index < len(passage_infos_positive) - 1:
        # there are some positives that have not been added
        print("Extra positives! " + str(len(passage_infos_positive)))
        have_room = True
        n_added = 0
        pos_index = max_pos_index + 1
        while have_room and pos_index < len(passage_infos_positive):
          for i in range(n_instances):
            if len(instances[i]) > max_cand * .5:
              have_room = False
              break
            instances[i].append(passage_infos_positive[pos_index])
            n_added += 1
            pos_index += 1
            if pos_index >= len(passage_infos_positive):
              break
        print("Total positives added " + str(pos_index) + " out of " +
              str(len(passage_infos_positive)))

      # positives have been added, fill in negatives
      pos_index = 0
      for i in range(n_instances):
        n_needed = max_cand - len(instances[i])
        instances[i].extend(passage_infos_negative[pos_index:pos_index +
                                                   n_needed])
        yield self.emit(query_info, instances[i])
        pos_index += n_needed

    else:
      # this is for eval, just emit all of them
      rotate_group = []
      rotate_group.extend(passage_infos_positive)
      rotate_group.extend(passage_infos_negative)
      start = 0
      while start < len(rotate_group):
        end = start + max_cand
        if end > len(rotate_group):
          move = end - len(rotate_group)
          start = start - move
          end = end - move
        passages = rotate_group[start:end]
        yield self.emit(query_info, passages)
        start = end
Get examples as a sequence of grouped candidates.
language/multivec/preprocessing/create_training_data.py