def partition_preds_by_scrape_type(verify_predictions,
                                   evidence_predictions,
                                   val_examples):
  """Partition predictions by which scrape_type they come from.

  The validation fold contains four sets of evidence: drqa, lucene, ukp_pred,
  and ukp_wiki. The intention is in this function to partition these into
  four sets so that they can each be scored separately to measure the
  difference between them on models that are trained on one of these
  (train_scrape).

  Args:
    verify_predictions: Claim verification predictions to partition, a 3-dim
      tensor of probabilities (one for each class)
    evidence_predictions: Evidence predictions to partition, a scalar
      probability of matching
    val_examples: Validation examples, typically all of
      FeverMetricsCallback._validation_flat

  Returns:
    Predictions and examples partitioned by scrape type
  """
  partitioned_verify = collections.defaultdict(list)
  partitioned_match = collections.defaultdict(list)
  partitioned_example = collections.defaultdict(list)
  for verify_probs, match_prob, example in zip(verify_predictions,
                                               evidence_predictions,
                                               val_examples):
    struct, _ = example
    metadata = json.loads(unwrap_tensor(struct['metadata']))
    scrape_type = metadata['scrape_type']
    partitioned_verify[scrape_type].append(verify_probs)
    partitioned_match[scrape_type].append(match_prob)
    partitioned_example[scrape_type].append(example)
  return partitioned_verify, partitioned_match, partitioned_example
Partition predictions by which scrape_type they come from.

  The validation fold contains four sets of evidence: drqa, lucene, ukp_pred,
  and ukp_wiki. The intention is in this function to partition these into
  four sets so that they can each be scored separately to measure the
  difference between them on models that are trained on one of these
  (train_scrape).

  Args:
    verify_predictions: Claim verification predictions to partition, a 3-dim
      tensor of probabilities (one for each class)
    evidence_predictions: Evidence predictions to partition, a scalar
      probability of matching
    val_examples: Validation examples, typically all of
      FeverMetricsCallback._validation_flat

  Returns:
    Predictions and examples partitioned by scrape type
  
language/serene/callbacks.py