def read_docs_to_embed(input_file):
  """Read json file containing Wikipedia articles into list of SquadExample."""

  def is_whitespace(c):
    if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
      return True
    return False

  with tf.gfile.Open(input_file, "r") as reader:
    examples = []
    for line in tqdm(reader):
      item = json.loads(line.strip())
      paragraph_text = item["context"]
      doc_tokens = []
      prev_is_whitespace = True
      for c in paragraph_text:
        if is_whitespace(c):
          prev_is_whitespace = True
        else:
          if prev_is_whitespace:
            doc_tokens.append(c)
          else:
            doc_tokens[-1] += c
          prev_is_whitespace = False

      qas_id = item["id"]
      question_text = ""
      entity_text = ""
      is_impossible = True
      start_position = -1
      end_position = -1
      orig_answer_text = ""

      indices_to_keep = []
      if "mention_tokens" in item:
        indices_to_keep = item["mention_tokens"]

      example = SquadExample(
          qas_id=qas_id,
          question_text=question_text,
          entity_text=entity_text,
          doc_tokens=doc_tokens,
          indices_to_keep=indices_to_keep,
          orig_answer_text=orig_answer_text,
          start_position=start_position,
          end_position=end_position,
          is_impossible=is_impossible)
      examples.append(example)

  return examples
Read json file containing Wikipedia articles into list of SquadExample.
language/labs/drkit/run_dualencoder_lsf.py