def _split_row_groups_from_footers(dataset):
    """Split the row groups by reading the footers of the parquet pieces"""

    logger.info('Recovering rowgroup information for the entire dataset. This can take a long time for datasets with '
                'large number of files. If this dataset was generated by Petastorm '
                '(i.e. by using "with materialize_dataset(...)") and you still see this message, '
                'this indicates that the materialization did not finish successfully.')

    thread_pool = futures.ThreadPoolExecutor()

    futures_list = [thread_pool.submit(_split_piece, piece, dataset.fs.open) for piece in dataset.pieces]
    result = [item for f in futures_list for item in f.result()]
    thread_pool.shutdown()
    return result
Split the row groups by reading the footers of the parquet pieces
petastorm/etl/dataset_metadata.py