def process_annotation(annotation_dict, allow_single_annotations=False):
  """Process annotation dictionary to generate references.

  Not all annotations will be a reference set to which prediction will be
  evaluated against. The median annotation will reserved to approximate human
  performance. In case there's only one reference annotation, the
  `allow_single_annotations` argument determines whether it is reserved.

  The output dictionaries will only contain examples where at least half the
  annotators found decontextualization is feasible.

  Args:
    annotation_dict: dictionary with a list of decontext labels.
    allow_single_annotations: bool, when there's only a single annotation,
      whether to pass that single annotation to the reference_sents_dict.

  Returns:
    For all dictionaries, key is the example id.

    original_sent_dict: value is the original sentence.
    reference_sents_dict: value is a list of reference decontextualized
    sentences.
    median_annotation_dict: key is an example id,
      value is the median length human annotation,
      considered to approximate human references.

  """
  reference_sents_dict = {}
  median_annotation_dict = {}
  original_sent_dict = {}

  for ex_id in list(annotation_dict.keys()):
    annotations = annotation_dict[ex_id]
    if not annotations:
      raise ValueError('Empty annotations encountered while processing '
                       'reference annotations.')
    not_impossible = [
        get_sent(label)
        for label in annotations
        if label.category != 'IMPOSSIBLE'
    ]
    not_impossible = sorted(not_impossible, key=len)
    if 2 * len(not_impossible) < len(annotations):
      # Skip the example if the majority of annotators marked it as IMPOSSIBLE.
      continue
    median_val = (len(not_impossible) - 1) // 2
    if len(not_impossible) == 1 and allow_single_annotations:
      median_annotation_dict[ex_id] = not_impossible[median_val]
    else:
      median_annotation_dict[ex_id] = not_impossible.pop(median_val)
    reference_sents_dict[ex_id] = not_impossible
    original_sent_dict[ex_id] = annotations[0].original_sentence
  logging.info('%d examples needs to be decontextualized out of %d',
               len(median_annotation_dict), len(annotation_dict))
  return original_sent_dict, reference_sents_dict, median_annotation_dict
Process annotation dictionary to generate references.

  Not all annotations will be a reference set to which prediction will be
  evaluated against. The median annotation will reserved to approximate human
  performance. In case there's only one reference annotation, the
  `allow_single_annotations` argument determines whether it is reserved.

  The output dictionaries will only contain examples where at least half the
  annotators found decontextualization is feasible.

  Args:
    annotation_dict: dictionary with a list of decontext labels.
    allow_single_annotations: bool, when there's only a single annotation,
      whether to pass that single annotation to the reference_sents_dict.

  Returns:
    For all dictionaries, key is the example id.

    original_sent_dict: value is the original sentence.
    reference_sents_dict: value is a list of reference decontextualized
    sentences.
    median_annotation_dict: key is an example id,
      value is the median length human annotation,
      considered to approximate human references.

  
language/decontext/eval_util.py