def load_spans(filename, vocab):
  """Load spans from file.

  Wordpieces are converted back (approximately) into PTB-style tokens.

  Args:
    filename: Path to predicted spans file.
    vocab: Instance of text_utils.Vocab.

  Returns:
    spans: Dictionary of image_id --> question_id --> beam_id --> span.
  """
  spans = {}
  with tf.io.gfile.GFile(filename, "r") as f:
    for line in f:
      entry = json.loads(line)
      image_id = entry["image_id"]
      if image_id not in spans:
        spans[image_id] = {}
      question_id = entry["question_id"]
      if question_id not in spans[image_id]:
        spans[image_id][question_id] = {}
      beam_id = entry["beam_id"]
      span = vocab.detokenize([vocab.i2t(i) for i in entry["span"]])
      ptb_tokens = clean_tokens(nltk_utils.word_tokenize(span))
      spans[image_id][question_id][beam_id] = " ".join(ptb_tokens)

  # Flatten to lists.
  for image_id, qas in spans.items():
    for question_id, beams in qas.items():
      qas[question_id] = [beams[i] for i in range(len(beams))]

  return spans
Load spans from file.

  Wordpieces are converted back (approximately) into PTB-style tokens.

  Args:
    filename: Path to predicted spans file.
    vocab: Instance of text_utils.Vocab.

  Returns:
    spans: Dictionary of image_id --> question_id --> beam_id --> span.
  
language/capwap/utils/metric_utils.py