def load_documents_from_shards(shard_paths, num_processes=None):
  """Loads Documents from a sharded, GZIP-ed TFRecords file into a Python list.

  Uses multiple processes to perform IO in parallel.

  Args:
    shard_paths: a list of paths, where each path points to a GZIP-ed TFRecords
      file. Documents loaded from each shard will be concatenated in the order
      of shard_paths.
    num_processes: number of Python processes to use in parallel. If None, will
      use all available CPUs.

  Returns:
    a list of Document instances.
  """
  num_shards = len(shard_paths)

  with parallel.Executor(
      create_worker=lambda: load_documents_from_shard,
      queue_size=num_shards,
      num_workers=num_processes) as executor:
    for shard_idx, shard_path in enumerate(shard_paths):
      executor.submit((shard_idx, shard_path))

    results = []
    for shard_idx, docs in executor.results(max_to_yield=num_shards):
      results.append((shard_idx, docs))
      logging.info('Loaded %d of %d document shards.', len(results), num_shards)

    # Sorts results by shard_idx.
    results.sort()

    logging.info('Combining data from all document shards.')
    all_docs = []
    for shard_idx, docs in results:
      all_docs.extend(docs)

    logging.info('Finished loading all shards.')
    return all_docs
Loads Documents from a sharded, GZIP-ed TFRecords file into a Python list.

  Uses multiple processes to perform IO in parallel.

  Args:
    shard_paths: a list of paths, where each path points to a GZIP-ed TFRecords
      file. Documents loaded from each shard will be concatenated in the order
      of shard_paths.
    num_processes: number of Python processes to use in parallel. If None, will
      use all available CPUs.

  Returns:
    a list of Document instances.
  
language/realm/retrieval.py