async def _traverse_directory(
        self,
        client: httpx.AsyncClient,
        repo_name: str,
        path: str,
        breadcrumbs: List[Breadcrumb],
        owner: str,
        repo: str,
        branch: str,
        processed_paths: set,
    ) -> AsyncGenerator[ChunkEntity, None]:
        """Recursively traverse a directory using DFS.

        Args:
            client: HTTP client
            repo_name: Repository name
            path: Current path to traverse
            breadcrumbs: Current breadcrumb chain
            owner: Repository owner
            repo: Repository name
            branch: Branch name
            processed_paths: Set of already processed paths

        Yields:
            Directory and file entities
        """
        if path in processed_paths:
            return

        processed_paths.add(path)

        # Get contents of the current directory
        url = f"{self.BASE_URL}/repos/{repo_name}/contents/{path}"
        params = {"ref": branch}

        try:
            contents = await self._get_with_auth(client, url, params)

            # Handle pagination if needed (using GitHub's API)
            if isinstance(contents, List):
                items = contents
            else:
                items = [contents]

            # Process each item in the directory
            for item in items:
                item_path = item["path"]
                item_type = item["type"]

                if item_type == "dir":
                    # Create directory entity
                    dir_entity = GitHubDirectoryEntity(
                        entity_id=f"{repo_name}/{item_path}",
                        source_name="github",
                        path=item_path,
                        repo_name=repo,
                        repo_owner=owner,
                        content=f"Directory: {item_path}",
                        breadcrumbs=breadcrumbs.copy(),
                        url=item["html_url"],
                    )

                    # Create breadcrumb for this directory
                    dir_breadcrumb = Breadcrumb(
                        entity_id=dir_entity.entity_id,
                        name=Path(item_path).name,
                        type="directory",
                    )

                    # Yield the directory entity
                    yield dir_entity

                    # Create updated breadcrumb chain for children
                    dir_breadcrumbs = breadcrumbs.copy() + [dir_breadcrumb]

                    # Recursively traverse this directory (DFS)
                    async for child_entity in self._traverse_directory(
                        client,
                        repo_name,
                        item_path,
                        dir_breadcrumbs,
                        owner,
                        repo,
                        branch,
                        processed_paths,
                    ):
                        yield child_entity

                elif item_type == "file":
                    # Process the file and yield entities
                    async for file_entity in self._process_file(
                        client, repo_name, item_path, item, breadcrumbs, owner, repo, branch
                    ):
                        yield file_entity

        except Exception as e:
            logger.error(f"Error traversing path {path}: {str(e)}")
Recursively traverse a directory using DFS.

        Args:
            client: HTTP client
            repo_name: Repository name
            path: Current path to traverse
            breadcrumbs: Current breadcrumb chain
            owner: Repository owner
            repo: Repository name
            branch: Branch name
            processed_paths: Set of already processed paths

        Yields:
            Directory and file entities
        
backend/airweave/platform/sources/github.py