def hotpot_eval_fn(dataset, results, name_map, output_prediction_file,
                   **kwargs):
  """Compute evaluation metrics for HotpotQADataset.

  Args:
    dataset: An object of type HotpotQADataset.
    results: A list of result dicts from running estimator.predict.
    name_map: A mapping from prediction indices to text strings.
    output_prediction_file: File to store predictions to.
    **kwargs: Variable keyword arguments.

  Returns:
    metrics: A dict mapping metric names to values.
  """
  del kwargs

  # Collect ground truth answers.
  gt_answer = {ex.qas_id: ex.answer_entity for ex in dataset.examples}
  gt_types = {ex.qas_id: ex.inference_chain for ex in dataset.examples}

  # Compute basic metrics.
  num_correct = {2: 0., 5: 0., 10: 0., 20: 0.}
  aps = []
  no_answer = 0.
  all_predictions = {}
  bridge_acc, comp_acc = 0., 0.
  bridge_tot, comp_tot = 0, 0
  single_acc = 0.
  layer_weights = np.zeros_like(results[0]["layer_probs"])
  num_layer_entities = {i: 0. for i in range(layer_weights.shape[0])}
  num_new_entities = {i: 0. for i in range(layer_weights.shape[0])}
  for result in results:
    qas_id = result["qas_ids"].decode("utf-8")
    preds = result["top_idx"]
    scores = result["top_vals"]
    ans = gt_answer[qas_id]
    my_type = gt_types[qas_id]
    if my_type == "bridge":
      bridge_tot += 1
    else:
      comp_tot += 1
    ranks = np.where(np.in1d(preds, ans))[0]
    ranks = np.sort(ranks)
    ap = 0.
    cnt = 0.
    if any(rr < 10 for rr in ranks):
      single_acc += 1
    if ranks.shape[0] == 0:
      no_answer += 1
    for rr in ranks:
      cnt += 1
      ap += cnt / (rr + 1)
    if ans:
      aps.append(ap / len(ans))
    else:
      aps.append(0.)
    found = False
    for key in [2, 5, 10, 20]:
      if found or np.in1d(ans, preds[:key]).all():
        num_correct[key] += 1
        found = True
        if key == 10:
          if my_type == "bridge":
            bridge_acc += 1
          else:
            comp_acc += 1
    # Non-accuracy stats
    layer_weights += result["layer_probs"]
    layer_entities = {i: set() for i in range(layer_weights.shape[0])}
    all_predictions[qas_id] = {}
    for i in range(layer_weights.shape[0]):
      layer_entities[i] = set(
          [ee for ee in result["layer_%d_ent" % i] if ee != -1])
      num_layer_entities[i] += len(layer_entities[i])
      num_new_entities[i] += len(layer_entities[i] - layer_entities[0])
      # all_predictions[qas_id]["layer_%d" % i] = [
      #     name_map[str(ee)] for ee in layer_entities[i]]
    all_predictions[qas_id]["predictions"] = [
        (name_map[str(pred)], str(scores[i])) for i, pred in enumerate(preds)
    ]
  tf.logging.info("Evaluated %d items", len(all_predictions))
  accuracy = {
      key: (num_correct[key] / len(all_predictions)) for key in num_correct
  }

  # Compute advanced metrics.
  json.dump(all_predictions, tf.gfile.Open(output_prediction_file, "w"))

  # Return metrics.
  metrics = {"eval/@%d" % key: accuracy[key] for key in accuracy}
  metrics["accuracy"] = accuracy[10]
  metrics["eval/map"] = sum(aps) / len(all_predictions)
  metrics["eval/bridge_accuracy"] = bridge_acc / bridge_tot
  metrics["eval/comparison_accuracy"] = comp_acc / comp_tot
  metrics["analysis/single_accuracy"] = single_acc / len(all_predictions)
  metrics["analysis/no_answers"] = no_answer / len(all_predictions)
  for i in range(layer_weights.shape[0]):
    metrics["analysis/layer_weight_%d" %
            i] = layer_weights[i] / len(all_predictions)
    metrics["analysis/num_entities_%d" %
            i] = num_layer_entities[i] / len(all_predictions)
    metrics["analysis/num_new_entities_%d" %
            i] = num_new_entities[i] / len(all_predictions)

  return metrics
Compute evaluation metrics for HotpotQADataset.

  Args:
    dataset: An object of type HotpotQADataset.
    results: A list of result dicts from running estimator.predict.
    name_map: A mapping from prediction indices to text strings.
    output_prediction_file: File to store predictions to.
    **kwargs: Variable keyword arguments.

  Returns:
    metrics: A dict mapping metric names to values.
  
language/labs/drkit/evaluate.py