def surface_recall(targets, predictions):
  """Measures recall of generatable surfaces in predictions."""
  numerator = 0.0
  filtered_denominator = 0.0
  denominator = 0.0
  for target, prediction in zip(targets, predictions):
    input_text = target["inputs"]
    predicted_text = prediction["targets"]
    generatable_surfaces = target["generatable_surfaces"]
    for surfaces in generatable_surfaces.values():
      filtered = False  # True if surface is in evidence.
      appears = False
      for surface in surfaces:
        if surface in input_text:
          filtered = True
        if surface in predicted_text:
          appears = True
      denominator += 1.0
      filtered_denominator += 1.0 if filtered else 0.0
      numerator += 1.0 if appears and filtered else 0.0
  recall = numerator / (denominator + 1e-13)
  filtered_recall = numerator / (filtered_denominator + 1e-13)
  return {"surface_recall": recall, "filtered_surface_recall": filtered_recall}
Measures recall of generatable surfaces in predictions.
language/fruit/metrics.py