def preprocess_metaqa(kb, entity2id, data_in_filename, data_out_filename,
                      num_hop):
  """Runner of the preprocessing code.

  Args:
    kb: kb dict
    entity2id: entity to int id
    data_in_filename: input filename
    data_out_filename: output filename
    num_hop: num hop
  """
  num_found = 0
  num_data = 0
  with open(data_in_filename) as f_in, open(data_out_filename, 'w') as f_out:
    for line in tqdm(f_in):
      num_data += 1
      line = line.strip()
      question, answers_str = line.split('\t')
      topic_ent = get_topic_ent(question)
      answers = set(answers_str.split('|'))
      candidate_chains = get_shortest_path(kb, topic_ent, answers, num_hop)

      best_chain, unused_intermediate_entities = postprocess_candidate_chains(
          kb, topic_ent, answers, candidate_chains, num_hop)
      num_found += int(best_chain is not None)

      out_example = {
          'question': question.replace('[' + topic_ent + ']', '__ent__'),
          'answers': list(answers),
          'question_entities': _link_entity_list([topic_ent], entity2id),
      }

      f_out.write('%s\n' % json.dumps(out_example))

  print('shortest path found: %d / %d' % (num_found, num_data))
Runner of the preprocessing code.

  Args:
    kb: kb dict
    entity2id: entity to int id
    data_in_filename: input filename
    data_out_filename: output filename
    num_hop: num hop
  
language/emql/preprocess/metaqa_preprocess.py