async def generate_entities(self) -> AsyncGenerator[Union[CTTIWebEntity], None]:
        """Generate WebEntity instances for each nct_id in the AACT studies table."""
        try:
            # Get the connection pool
            pool = await self._ensure_pool()

            # Get the limit and skip from config
            limit = self.config.get("limit", 10000)
            skip = self.config.get("skip", 0)

            # Simple query - URL construction in Python is fine
            query = f'''
                SELECT nct_id
                FROM "{CTTISource.AACT_SCHEMA}"."{CTTISource.AACT_TABLE}"
                WHERE nct_id IS NOT NULL
                ORDER BY nct_id
                LIMIT {limit}
                OFFSET {skip}
            '''

            async def _execute_query():
                # Use connection from pool
                async with pool.acquire() as conn:
                    if skip > 0:
                        logger.info(
                            f"Executing query to fetch {limit} clinical trials from AACT database "
                            f"(skipping first {skip} records)"
                        )
                    else:
                        logger.info(
                            f"Executing query to fetch {limit} clinical trials from AACT database"
                        )
                    records = await conn.fetch(query)
                    logger.info(f"Successfully fetched {len(records)} clinical trial records")
                    return records

            # Use retry logic for query execution
            records = await _retry_with_backoff(_execute_query)

            logger.info(f"Starting to process {len(records)} records into entities")
            entities_created = 0

            # Process each nct_id
            for record in records:
                nct_id = record["nct_id"]

                # Skip if nct_id is empty or None
                if not nct_id or not str(nct_id).strip():
                    continue

                # Clean the nct_id (remove whitespace)
                clean_nct_id = str(nct_id).strip()

                # Create the ClinicalTrials.gov URL
                url = f"https://clinicaltrials.gov/study/{clean_nct_id}"

                # Create entity_id using the nct_id
                entity_id = f"CTTI:study:{clean_nct_id}"

                # Create WebEntity
                entity = CTTIWebEntity(
                    entity_id=entity_id,
                    url=url,
                    title=f"Clinical Trial {clean_nct_id}",
                    description=(
                        f"Clinical trial study from ClinicalTrials.gov with NCT ID: {clean_nct_id}"
                    ),
                    nct_id=clean_nct_id,
                    study_url=url,
                    data_source="ClinicalTrials.gov",
                    breadcrumbs=[
                        Breadcrumb(
                            entity_id="CTTI:source", name="CTTI Clinical Trials", type="source"
                        ),
                        Breadcrumb(
                            entity_id=entity_id,
                            name=f"Clinical Trial {clean_nct_id}",
                            type="clinical_trial",
                        ),
                    ],
                    metadata={
                        "source": "CTTI",
                        "database_host": self.AACT_HOST,
                        "database_name": self.AACT_DATABASE,
                        "database_schema": self.AACT_SCHEMA,
                        "database_table": self.AACT_TABLE,
                        "original_nct_id": nct_id,  # Keep original in case it had formatting
                        "limit_used": limit,
                        "skip_used": skip,
                        "total_fetched": len(records),
                    },
                )

                # TODO: For faster startup, consider batching entity creation
                # and checking Azure storage existence in parallel batches
                # This would reduce the sequential processing time significantly

                entities_created += 1

                # Log progress every 100 entities
                if entities_created % 100 == 0:
                    logger.info(f"Created {entities_created}/{len(records)} CTTI entities")

                # Yield control periodically to prevent blocking
                if entities_created % 10 == 0:
                    await asyncio.sleep(0)  # Allow other tasks to run

                yield entity

            logger.info(f"Completed creating all {entities_created} CTTI entities")

        except Exception as e:
            logger.error(f"Error in CTTI source generate_entities: {str(e)}")
            raise
        # Note: We don't close the pool here as it's shared across all CTTI instances
Generate WebEntity instances for each nct_id in the AACT studies table.
backend/airweave/platform/sources/ctti.py