def _annotate_dataset(context_export_dir, entity_export_dir, data_paths):
  """Load examples to annotate.

  Args:
    context_export_dir: String for context model export directory.
    entity_export_dir: String for entity model export directory.
    data_paths: List of data paths.

  Returns:
    List of dictionaries for annotated examples.
  """
  context_predict_fn = contrib_predictor.from_saved_model(context_export_dir)
  entity_predict_fn = contrib_predictor.from_saved_model(entity_export_dir)
  num_processed_examples = 0
  annotated_examples = []
  for data_path in data_paths:
    with GzipFile(fileobj=tf.gfile.GFile(data_path)) as data_file:
      for line in data_file:
        if num_processed_examples % 10 == 0:
          print("Processed %d examples." % num_processed_examples)
        example_id, question, contexts, context_indices = _get_example(line)
        long_answer_idx, long_answer_score = (
            _annotate_long_answer(context_predict_fn, question, contexts))

        if long_answer_idx != _NULL_INDEX:
          long_answer_start_token, long_answer_end_token = (
              context_indices[long_answer_idx])

          # Unlike in long answer, we tokenize before calling the short answer
          # model.
          question_tokens = question.split(" ")
          pred_context_tokens = contexts[long_answer_idx].split(" ")

          (short_answer_start_token,
           short_answer_end_token, short_answer_score) = _annotate_short_answer(
               entity_predict_fn, question_tokens, pred_context_tokens)

          # Offset the start/end indices by the start of the long answer.
          short_answer_start_token += long_answer_start_token
          short_answer_end_token += long_answer_start_token
          short_answer_dict = [{
              "start_token": short_answer_start_token,
              "end_token": short_answer_end_token,
              "start_byte": -1,
              "end_byte": -1
          }]
        else:
          # If long answer is NULL then short answer must also be NULL.
          long_answer_start_token = _NULL_INDEX
          long_answer_end_token = _NULL_INDEX
          short_answer_dict = []
          short_answer_score = -1.0

        annotated_example = {
            "example_id": example_id,
            "long_answer_score": long_answer_score,
            "short_answers_score": short_answer_score,
            "long_answer": {
                "start_token": long_answer_start_token,
                "end_token": long_answer_end_token,
                "start_byte": -1,
                "end_byte": -1
            },
            # While in theory there may be multiple spans indicating the short
            # answer, our model only outputs one span.
            "short_answers": short_answer_dict,
        }
        annotated_examples.append(annotated_example)
        num_processed_examples += 1

  return annotated_examples
Load examples to annotate.

  Args:
    context_export_dir: String for context model export directory.
    entity_export_dir: String for entity model export directory.
    data_paths: List of data paths.

  Returns:
    List of dictionaries for annotated examples.
  
language/question_answering/decatt_docreader/experiments/nq_export_scorer.py