def persist_dataset(
        self,
        session: core_session.TruSession,
        dataset_name: str,
        dataset_metadata: Optional[Dict[str, Any]] = None,
        split="test",
        download=True,
        chunk_size=1000,
    ):
        """Persist BEIR dataset into DB with pre-processed fields to match expected TruLens schemas.

        Note this method handle chunking of the dataset to avoid loading the entire dataset into memory at once by default.

        Args:
            split: Defaults to "test".

            session: TruSession instance to persist the dataset.

            dataset_name: Name of the dataset to be persisted - Note this can
              be different from the standardized BEIR dataset names.

            dataset_metadata: Metadata for the dataset.

            download: If False, remove the downloaded dataset file after
              processing. Defaults to True.

        Returns:
            DataFrame with the BEIR dataset

        """
        for chunk in self._process_dataset(split=split, chunk_size=chunk_size):
            df_chunk = pd.DataFrame(chunk)
            session.add_ground_truth_to_dataset(
                dataset_name=dataset_name,
                ground_truth_df=df_chunk,
                dataset_metadata=dataset_metadata,
            )

        if not download:
            logger.info(f"Cleaning up downloaded {self.dataset_name} dataset")
            os.system(
                f"rm -rf {os.path.join(self.data_folder, self.dataset_name)}"
            )
            os.system(
                f"rm -rf {os.path.join(self.data_folder, self.dataset_name)}.zip"
            )

        logger.info(
            f"Finished processing dataset {self.dataset_name} in chunks."
        )
Persist BEIR dataset into DB with pre-processed fields to match expected TruLens schemas.

        Note this method handle chunking of the dataset to avoid loading the entire dataset into memory at once by default.

        Args:
            split: Defaults to "test".

            session: TruSession instance to persist the dataset.

            dataset_name: Name of the dataset to be persisted - Note this can
              be different from the standardized BEIR dataset names.

            dataset_metadata: Metadata for the dataset.

            download: If False, remove the downloaded dataset file after
              processing. Defaults to True.

        Returns:
            DataFrame with the BEIR dataset

        
src/benchmark/trulens/benchmark/benchmark_frameworks/dataset/beir_loader.py