def _process_dataset(
        self, split="test", chunk_size=None
    ) -> Generator[List[Dict[str, Any]], None, None]:
        """
        Common method to process the BEIR dataset into entries.
        This method handles downloading, loading, and processing the dataset.

        Args:
            split (str, optional): Dataset split to load. Defaults to "test".
            chunk_size (int, optional): Number of records to process in each chunk. Defaults to None.

        Yields:
            List[Dict[str, Any]]: List of dataset entries (a chunk if chunk_size is specified).
        """

        url = f"https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/{self.dataset_name}.zip"

        logger.info(f"Downloading {self.dataset_name} dataset from {url}")
        download_and_unzip(url, self.data_folder)

        corpus_gen, queries_gen = self._load_generators(split=split)

        dataset_entries = []

        corpus_dict = {}
        for corpus in list(
            corpus_gen
        ):  # TODO: (Daniel) this is still going to be a memory hog for large datasets, but if we don't
            # load the entire corpus into memory, we'll have to re-read the entire corpus using generator for each query
            corpus_dict.update(corpus)

        # Iterate over queries generator and process entries
        for i, query in enumerate(queries_gen, 1):
            for query_id, (query_text, answer) in query.items():
                doc_to_rel = self.qrels.get(query_id, {})

                expected_chunks = []

                for corpus_id in doc_to_rel.keys():
                    if corpus_id in corpus_dict:
                        corpus_entry = corpus_dict[corpus_id]
                        expected_chunks.append({
                            "text": corpus_entry["text"],
                            "title": corpus_entry.get("title"),
                            "expected_score": doc_to_rel.get(
                                corpus_id, 0
                            ),  # Default relevance score to 0 if not found
                        })

                dataset_entries.append({
                    "query_id": query_id,
                    "query": query_text,
                    "expected_response": answer,  # expected response can be also be empty for IR datasets
                    "expected_chunks": expected_chunks,
                    "meta": {"source": self.dataset_name},
                })

                # Yield the chunk if chunk_size is specified
                if chunk_size and i % chunk_size == 0:
                    yield dataset_entries
                    dataset_entries = []

        if dataset_entries:
            yield dataset_entries

        Common method to process the BEIR dataset into entries.
        This method handles downloading, loading, and processing the dataset.

        Args:
            split (str, optional): Dataset split to load. Defaults to "test".
            chunk_size (int, optional): Number of records to process in each chunk. Defaults to None.

        Yields:
            List[Dict[str, Any]]: List of dataset entries (a chunk if chunk_size is specified).
        
src/benchmark/trulens/benchmark/benchmark_frameworks/dataset/beir_loader.py