def read_squad_examples(input_file,
                        is_training,
                        include_entity_in_question=True,
                        p=1.0):
  """Read a SQuAD json file into a list of SquadExample."""

  def is_whitespace(c):
    if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
      return True
    return False

  with tf.gfile.Open(input_file, "r") as reader:
    examples = []
    for line in tqdm(reader):
      item = json.loads(line.strip())
      paragraph_text = item["context"]
      doc_tokens = []
      char_to_word_offset = []
      prev_is_whitespace = True
      for c in paragraph_text:
        if is_whitespace(c):
          prev_is_whitespace = True
        else:
          if prev_is_whitespace:
            doc_tokens.append(c)
          else:
            doc_tokens[-1] += c
          prev_is_whitespace = False
        char_to_word_offset.append(len(doc_tokens) - 1)

      qas_id = item["id"]
      if item["subject"]["name"] is None or (is_training and
                                             random.uniform(0., 1.) < p):
        entity_text = random.choice(item["subject"]["mentions"])["text"]
      else:
        entity_text = item["subject"]["name"]
      if include_entity_in_question:
        question_text = (
            entity_text + " . " + random.choice(item["relation"]["text"]))
      else:
        question_text = random.choice(item["relation"]["text"])

      start_position = None
      end_position = None
      start_relation = None
      end_relation = None
      orig_answer_text = None
      orig_relation_text = None
      is_impossible = False
      has_entity = False
      has_relation = False
      if is_training:

        if FLAGS.version_2_with_negative:
          is_impossible = item["is_impossible"]
        if not is_impossible:
          answer = item["object"]["mention"]
          orig_answer_text = answer["text"]
          answer_offset = answer["start"]
          answer_length = len(orig_answer_text)
          start_position = char_to_word_offset[answer_offset]
          end_position = char_to_word_offset[answer_offset + answer_length - 1]
          start_relation = start_position
          end_relation = end_position
          orig_relation_text = answer["text"]
          has_entity = True
          has_relation = True
          # Only add answers where the text can be exactly recovered from the
          # document. If this CAN'T happen it's likely due to weird Unicode
          # stuff so we will just skip the example.
          #
          # Note that this means for training mode, every example is NOT
          # guaranteed to be preserved.
          actual_text = " ".join(doc_tokens[start_position:(end_position + 1)])
          cleaned_answer_text = " ".join(
              tokenization.whitespace_tokenize(orig_answer_text))
          if actual_text.find(cleaned_answer_text) == -1:
            tf.logging.warning("Example %d", len(examples))
            tf.logging.warning(json.dumps(item, indent=2))
            tf.logging.warning("Could not find answer: '%s' vs. '%s'",
                               actual_text, cleaned_answer_text)
            continue
        else:
          start_position = -1
          end_position = -1
          orig_answer_text = ""
          start_relation = -1
          end_relation = -1
          if item["context_type"] == "entity negative":
            has_entity = True
          elif item["context_type"] == "relation negative":
            answer = item["object"]["mention"]
            orig_relation_text = answer["text"]
            answer_offset = answer["start"]
            answer_length = len(orig_answer_text)
            start_relation = char_to_word_offset[answer_offset]
            end_relation = char_to_word_offset[answer_offset + answer_length -
                                               1]
            has_relation = True

      example = SquadExample(
          qas_id=qas_id,
          question_text=question_text,
          entity_text=entity_text,
          doc_tokens=doc_tokens,
          orig_answer_text=orig_answer_text,
          start_position=start_position,
          end_position=end_position,
          start_relation=start_relation,
          end_relation=end_relation,
          orig_relation_text=orig_relation_text,
          is_impossible=is_impossible,
          has_entity=has_entity,
          has_relation=has_relation)
      examples.append(example)

  return examples
Read a SQuAD json file into a list of SquadExample.
language/labs/drkit/run_dualencoder_lsf.py