def load_dataset_to_df(
        self,
        split="test",
        download=True,
    ) -> pd.DataFrame:
        """
        load BEIR dataset into dataframe with pre-processed fields to match expected TruLens schemas.
        Note this method loads the entire dataset into memory at once.
        Args:
            split (str, optional): Defaults to "test".
            download (bool, optional): If False, remove the downloaded dataset file after processing. Defaults to True.
        Returns:
            pd.DataFrame: DataFrame with the BEIR dataset
        """
        dataset_entries = []
        for chunk in self._process_dataset(split=split):
            dataset_entries.extend(chunk)

        if not download:
            logger.info(f"Cleaning up downloaded {self.dataset_name} dataset")
            os.system(
                f"rm -rf {os.path.join(self.data_folder, self.dataset_name)}"
            )
            os.system(
                f"rm -rf {os.path.join(self.data_folder, self.dataset_name)}.zip"
            )

        return pd.DataFrame(dataset_entries)

        load BEIR dataset into dataframe with pre-processed fields to match expected TruLens schemas.
        Note this method loads the entire dataset into memory at once.
        Args:
            split (str, optional): Defaults to "test".
            download (bool, optional): If False, remove the downloaded dataset file after processing. Defaults to True.
        Returns:
            pd.DataFrame: DataFrame with the BEIR dataset
        
src/benchmark/trulens/benchmark/benchmark_frameworks/dataset/beir_loader.py