def count_tf_records_parallel(file_paths, num_processes=None):
  """Counts number of records in TFRecord files in parallel.

  Args:
    file_paths: a list of paths, where each path points to a GZIP-ed TFRecord
      file.
    num_processes: number of Python processes to use in parallel. If None, will
      use all available CPUs.

  Returns:
    shard_sizes: a list of ints.
  """
  num_files = len(file_paths)
  with parallel.Executor(
      create_worker=lambda: count_tf_records_parallel_helper,
      queue_size=num_files,
      num_workers=num_processes) as executor:
    for file_idx, file_path in enumerate(file_paths):
      executor.submit((file_idx, file_path))

    counts = [None] * num_files
    results = executor.results(max_to_yield=num_files)
    for i, (file_idx, count) in enumerate(results):
      counts[file_idx] = count
      logging.info('Counted %d / %d files.', i + 1, num_files)

  return counts
Counts number of records in TFRecord files in parallel.

  Args:
    file_paths: a list of paths, where each path points to a GZIP-ed TFRecord
      file.
    num_processes: number of Python processes to use in parallel. If None, will
      use all available CPUs.

  Returns:
    shard_sizes: a list of ints.
  
language/realm/retrieval.py