async def web_page_rag(state: AgentState):
    """Searches the web page for relevant information based on the User input"""
    page = state["page"]

    if state["is_pdf"]:
        result = await scrape_pdf(page)
    else:
        result = await scrape_text(page)

    if result == "Forbidden":
        return {"actions_taken": [f"Scraping the webpage {page.url} failed, should try another url"]}
    elif result == "No data found":
        return {"actions_taken": [f"No textual content found on the webpage {page.url}, try looking for url that has data"]}
    else:
        docs = await docs_from_text(result, page.url)
        print(len(docs))

        await store_doc_embeddings(docs)

        return {"actions_taken":[f"Scraped the url {page.url} and stored the information in a vector database for future reference"]}
Searches the web page for relevant information based on the User input
backend/app/deep_research_agent.py