def generate_examples_parallel():
  """Same as `generate_examples`, but runs in parallel."""
  create_worker = DocumentProcessor
  worker_kwargs = {
      'vocab_path': FLAGS.vocab_path,
      'do_lower_case': FLAGS.do_lower_case,
  }

  with parallel.Executor(
      create_worker=create_worker,
      queue_size=10000,
      worker_kwargs=worker_kwargs) as executor:

    # Feed inputs to the DocumentProcessors.
    executor.submit_from_generator(load_json_data, input_path=FLAGS.input_path)

    for i, result in enumerate(executor.results()):
      yield result
      if i == FLAGS.total_documents - 1:
        break
Same as `generate_examples`, but runs in parallel.
language/realm/generate_retrieval_corpus.py