def read_examples(input_file):
  """Read a SQuAD-like json file into a list of Examples."""

  def is_whitespace(c):
    if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
      return True
    return False

  reader = tf.gfile.Open(input_file, "rb")
  if input_file.endswith(".gz"):
    reader = gzip.GzipFile(fileobj=reader)
  next(reader)
  examples = []
  for line in tqdm(reader):
    item = json.loads(line.strip())
    paragraph_text = item["context"]
    doc_tokens = []
    char_to_word_offset = []
    prev_is_whitespace = True
    for c in paragraph_text:
      if is_whitespace(c):
        prev_is_whitespace = True
      else:
        if prev_is_whitespace:
          doc_tokens.append(c)
        else:
          doc_tokens[-1] += c
        prev_is_whitespace = False
      char_to_word_offset.append(len(doc_tokens) - 1)

    for qa in item["qas"]:
      qas_id = qa["qid"]
      question_text = qa["question"]

      start_position = None
      end_position = None
      orig_answer_text = None
      is_impossible = False

      start_position = -1
      end_position = -1
      orig_answer_text = ""
      if FLAGS.version_2_with_negative:
        is_impossible = qa["is_impossible"]
      if not is_impossible:
        answer_offset = qa["detected_answers"][0]["char_spans"][0][0]
        answer_end = qa["detected_answers"][0]["char_spans"][0][1]
        answer_length = answer_end - answer_offset + 1
        orig_answer_text = item["context"][answer_offset:answer_end + 1]
        start_position = char_to_word_offset[answer_offset]
        end_position = char_to_word_offset[answer_offset + answer_length - 1]
        # Only add answers where the text can be exactly recovered from the
        # document. If this CAN'T happen it's likely due to weird Unicode
        # stuff so we will just skip the example.
        #
        # Note that this means for training mode, every example is NOT
        # guaranteed to be preserved.
        actual_text = " ".join(doc_tokens[start_position:(end_position + 1)])
        cleaned_answer_text = " ".join(
            tokenization.whitespace_tokenize(orig_answer_text))
        if actual_text.find(cleaned_answer_text) == -1:
          tf.logging.warning("Example %d", len(examples))
          tf.logging.warning(json.dumps(item, indent=2))
          tf.logging.warning("Could not find answer: '%s' vs. '%s'",
                             actual_text, cleaned_answer_text)
          continue

      example = Example(
          qas_id=qas_id,
          question_text=question_text,
          doc_tokens=doc_tokens,
          orig_answer_text=orig_answer_text,
          start_position=start_position,
          end_position=end_position,
          is_impossible=is_impossible)
      examples.append(example)
  reader.close()

  return examples
Read a SQuAD-like json file into a list of Examples.
language/labs/drkit/hotpotqa/preprocessing/create_tfrecords.py