def load_boolq_file(filename, num_par=2):
  """Build a tf.data.Data from a file of boolq examples."""
  tokenizer = tokenization.NltkTokenizer()
  examples = []
  with tf.gfile.Open(filename) as f:
    for line in f:
      obj = json.loads(line)
      context = tokenizer.tokenize(obj["passage"])
      if FLAGS.max_passage_len:
        context = context[:FLAGS.max_passage_len]
      question = tokenizer.tokenize(obj["question"])
      examples.append((question, context, obj["answer"]))

  def get_data():
    out = list(examples)
    np.random.shuffle(out)
    return out

  ds = tf.data.Dataset.from_generator(get_data, (tf.string, tf.string, tf.bool),
                                      ([None], [None], []))

  def to_dict(p, h, label):
    return {"hypothesis": p, "premise": h, "label": label}

  return ds.map(to_dict, num_parallel_calls=num_par)
Build a tf.data.Data from a file of boolq examples.
language/boolq/run_recurrent_model_boolq.py