Upload 6 files

__init__.py

@@ -0,0 +1,5 @@
+from .docparser import DocParser
+from .chunkers import Chunker, SemanticChunker, AgenticChunker
+from .imageprocessing import ImageProcessor
+from .doc_qa import AgenticQA
+from src.indexing import indexing

app_1.py

@@ -0,0 +1,518 @@
+from flask import Flask, request, render_template, session, url_for, redirect, jsonify
+from flask_session import Session
+from langchain_core.messages import HumanMessage, AIMessage
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+import os
+import logging
+import re
+import traceback
+import base64
+import shutil
+import zipfile
+from dotenv import load_dotenv
+from huggingface_hub import hf_hub_download
+from PIL import Image
+
+# --- Core Application Imports ---
+# Make sure you have an empty __init__.py file in your 'src' folder
+from api import api_bp
+from src.medical_swarm import run_medical_swarm
+from src.utils import load_rag_system, standardize_query, get_standalone_question, parse_agent_response, markdown_bold_to_html
+from langchain_google_genai import ChatGoogleGenerativeAI
+
+# Setup logging
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+
+# --- 1. DATABASE SETUP FUNCTION (For Deployment) ---
+def setup_database():
+    """Downloads and unzips the ChromaDB folder from Hugging Face Datasets."""
+    
+    # --- !!! IMPORTANT !!! ---
+    # YOU MUST CHANGE THIS to your Hugging Face Dataset repo ID
+    # For example: "your_username/your_database_repo_name"
+    DATASET_REPO_ID = "WanIrfan/atlast-db" 
+    # -------------------------
+
+    ZIP_FILENAME = "chroma_db.zip"
+    DB_DIR = "chroma_db"
+
+    if os.path.exists(DB_DIR) and os.listdir(DB_DIR):
+        logger.info("✅ Database directory already exists. Skipping download.")
+        return
+
+    logger.info(f"📥 Downloading database from HF Hub: {DATASET_REPO_ID}")
+    try:
+        zip_path = hf_hub_download(
+            repo_id=DATASET_REPO_ID,
+            filename=ZIP_FILENAME,
+            repo_type="dataset",
+            # You might need to add your HF token to secrets if the dataset is private
+            # token=os.getenv("HF_TOKEN") 
+        )
+        
+        logger.info(f"📦 Unzipping database from {zip_path}...")
+        with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+            zip_ref.extractall(".") # Extracts to the root, creating ./chroma_db
+            
+        logger.info("✅ Database setup complete!")
+        
+        # Clean up the downloaded zip file to save space
+        if os.path.exists(zip_path):
+            os.remove(zip_path)
+            
+    except Exception as e:
+        logger.error(f"❌ CRITICAL ERROR setting up database: {e}", exc_info=True)
+        # This will likely cause the RAG system to fail loading, which is expected
+        # if the database isn't available.
+
+# --- RUN DATABASE SETUP *BEFORE* INITIALIZING THE APP ---
+setup_database()
+
+
+# --- STANDARD FLASK APP INITIALIZATION ---
+app = Flask(__name__)
+app.secret_key = os.urandom(24)  # Set a secret key for session signing
+
+# --- CONFIGURE SERVER-SIDE SESSIONS ---
+app.config["SESSION_PERMANENT"] = False
+app.config["SESSION_TYPE"] = "filesystem"
+Session(app)
+
+google_api_key = os.getenv("GOOGLE_API_KEY")
+if not google_api_key:
+    logger.warning("⚠️ GOOGLE_API_KEY not found in environment variables. LLM calls will fail.")
+else:
+    logger.info("GOOGLE_API_KEY loaded successfully.")
+
+# Initialize LLM
+llm = ChatGoogleGenerativeAI(model="gemini-2.5-flash", temperature=0.05, google_api_key=google_api_key)
+
+# --- LOAD RAG SYSTEMS (AFTER DB SETUP) ---
+logger.info("🌟 Starting Multi-Domain AI Assistant...")
+try:
+    rag_systems = {
+        'medical': load_rag_system(collection_name="medical_csv_Agentic_retrieval", domain="medical"),
+        'islamic': load_rag_system(collection_name="islamic_texts_Agentic_retrieval", domain="islamic"),
+        'insurance': load_rag_system(collection_name="etiqa_Agentic_retrieval", domain="insurance")
+    }
+except Exception as e:
+    logger.error(f"❌ FAILED to load RAG systems. Check database path and permissions. Error: {e}", exc_info=True)
+    rag_systems = {'medical': None, 'islamic': None, 'insurance': None}
+
+# Store systems and LLM on the app for blueprints
+app.rag_systems = rag_systems
+app.llm = llm
+
+# Register the API blueprint
+app.register_blueprint(api_bp)
+logger.info(f"✅ API Blueprint registered. API endpoints are now available under /api")
+
+# Check initialization status
+logger.info("\n📊 SYSTEM STATUS:")
+for domain, system in rag_systems.items():
+    status = "✅ Ready" if system else "❌ Failed (DB missing?)" 
+    logger.info(f"   {domain}: {status}")
+
+
+# --- FLASK ROUTES ---
+
+@app.route("/")
+def homePage():
+    # Clear all session history when visiting the home page
+    session.pop('medical_history', None)
+    session.pop('islamic_history', None)
+    session.pop('insurance_history', None)
+    session.pop('current_medical_document', None)
+    return render_template("homePage.html")
+
+
+@app.route("/medical", methods=["GET", "POST"])
+def medical_page():
+    # Use session for history and document context
+    if request.method == "GET":
+        # Load all latest data from session (or default to empty if not found)
+        latest_response = session.pop('latest_medical_response', {}) # POP to clear it after one display
+        
+        answer = latest_response.get('answer', "")
+        thoughts = latest_response.get('thoughts', "")
+        validation = latest_response.get('validation', "")
+        source = latest_response.get('source', "")
+
+        # Clear history only when a user first navigates (not on redirect)
+        if not latest_response and 'medical_history' not in session:
+            session.pop('current_medical_document', None)
+        
+        return render_template("medical_page.html", 
+                               history=session.get('medical_history', []),
+                               answer=answer,
+                               thoughts=thoughts,
+                               validation=validation,
+                               source=source)
+    
+    # POST Request Logic
+    answer, thoughts, validation, source = "", "", "", ""
+    history = session.get('medical_history', [])
+    current_medical_document = session.get('current_medical_document', "")
+    
+    
+    try:
+        query=standardize_query(request.form.get("query", ""))
+        has_image = 'image' in request.files and request.files['image'].filename
+        has_document = 'document' in request.files and request.files['document'].filename
+        has_query = request.form.get("query") or request.form.get("question", "")
+            
+        logger.info(f"POST request received: has_image={has_image}, has_document={has_document}, has_query={has_query}")
+            
+        if has_document:
+            # Scenario 3: Query + Document
+            logger.info("Processing Scenario 3: Query + Document with Medical Swarm")
+            file = request.files['document']
+            try:
+                # Store the new document text in the session
+                document_text = file.read().decode("utf-8")
+                session['current_medical_document'] = document_text
+                current_medical_document = document_text # Use the new document for this turn
+            except UnicodeDecodeError:
+                answer = "Error: Could not decode the uploaded document. Please ensure it is a valid text or PDF file."
+                logger.error("Scenario 3: Document decode error")
+                thoughts = traceback.format_exc()
+                  
+            swarm_answer = run_medical_swarm(current_medical_document, query)
+            answer = markdown_bold_to_html(swarm_answer)
+                
+            history.append(HumanMessage(content=f"[Document Uploaded] Query: '{query}'"))
+            history.append(AIMessage(content=swarm_answer))
+            thoughts = "Swarm analysis complete. The process is orchestrated and does not use the ReAct thought process. You can now ask follow-up questions."
+            source= "Medical Swarm"
+            validation = (True, "Swarm output generated.") # Swarm has its own validation logic
+            
+        elif has_image :
+            #Scenario 1 
+            logger.info("Processing Multimodal RAG: Query + Image")
+            # --- Step 1 & 2: Image Setup & Vision Analysis ---
+            file = request.files['image']
+            upload_dir = "Uploads"
+            os.makedirs(upload_dir, exist_ok=True)
+            image_path = os.path.join(upload_dir, file.filename)
+            
+            try:
+                file.save(image_path)
+                file.close()
+            
+                with open(image_path, "rb") as img_file:
+                    img_data = base64.b64encode(img_file.read()).decode("utf-8")
+                
+            
+                vision_prompt = f"Analyze this image and identify the main subject in a single, concise sentence. The user's query is: '{query}'"
+                message = HumanMessage(content=[
+                    {"type": "text", "text": vision_prompt},
+                    {"type": "image_url", "image_url": f"data:image/jpeg;base64,{img_data}"}
+                ])
+                vision_response = llm.invoke([message])
+                visual_prediction = vision_response.content
+                logger.info(f"Vision Prediction: {visual_prediction}")
+
+                # --- Create an Enhanced Query ---
+                enhanced_query = (
+                    f'User Query: "{query}" '
+                    f'Context from an image provided by the LLM: "{visual_prediction}" '
+                    'Based on the user\'s query and the context from LLM, provide a comprehensive answer.'
+                )
+                logger.info(f"Enhanced query : {enhanced_query}")
+            
+                agent = rag_systems['medical']
+                if not agent: raise Exception("Medical RAG system is not loaded.")
+                response_dict = agent.answer(enhanced_query, chat_history=history)
+                answer, thoughts, validation, source = parse_agent_response(response_dict)
+                history.append(HumanMessage(content=query))
+                history.append(AIMessage(content=answer))
+            
+            finally:
+                if os.path.exists(image_path):
+                    try:
+                        os.remove(image_path)
+                        logger.info(f"Successfully deleted temporary image file: {image_path}")
+                    except PermissionError as e:
+                        logger.warning(f"Could not remove {image_path} after processing. "
+                                       f"File may be locked by another process. Error: {e}")
+            
+        elif query:
+            # --- SCENARIO 2: TEXT-ONLY QUERY OR SWARM FOLLOW-UP ---
+            history_for_agent = history
+            if current_medical_document:
+                logger.info("Processing Follow-up Query for Document")
+                history_for_agent = [HumanMessage(content=f"We are discussing this document:\n{current_medical_document}")] + history
+            else:
+                logger.info("Processing Text RAG query for Medical domain")
+            
+            logger.info(f"Original Query: '{query}'")
+            print(f"📚 Using chat history with {len(history)} previous messages to create standalone query")
+            standalone_query = get_standalone_question(query, history_for_agent,llm)
+            logger.info(f"Standalone Query: '{standalone_query}'")
+            
+            agent = rag_systems['medical']
+            if not agent: raise Exception("Medical RAG system is not loaded.")
+            response_dict = agent.answer(standalone_query, chat_history=history_for_agent)
+            answer, thoughts, validation, source = parse_agent_response(response_dict)
+
+            history.append(HumanMessage(content=query))
+            history.append(AIMessage(content=answer))
+
+        else:
+            raise ValueError("No query or file provided.")
+    except Exception as e:
+        logger.error(f"Error on /medical page: {e}", exc_info=True)
+        answer = f"An error occurred: {e}"
+        thoughts = traceback.format_exc()
+    
+    # Save updated history and LATEST RESPONSE DATA back to the session
+    session['medical_history'] = history
+    session['latest_medical_response'] = {
+        'answer': answer, 
+        'thoughts': thoughts, 
+        'validation': validation, 
+        'source': source
+    }
+    session.modified = True
+                             
+    logger.debug(f"Redirecting after saving latest response.")
+    return redirect(url_for('medical_page'))
+
+@app.route("/medical/clear")
+def clear_medical_chat():
+    session.pop('medical_history', None)
+    session.pop('current_medical_document', None)
+    logger.info("Medical chat history cleared.")
+    return redirect(url_for('medical_page'))
+
+@app.route("/islamic", methods=["GET", "POST"])
+def islamic_page():
+    #Use session
+    
+    if request.method == "GET":
+        # Load all latest data from session (or default to empty if not found)
+        latest_response = session.pop('latest_islamic_response', {}) # POP to clear it after one display
+        
+        answer = latest_response.get('answer', "")
+        thoughts = latest_response.get('thoughts', "")
+        validation = latest_response.get('validation', "")
+        source = latest_response.get('source', "")
+        
+        # Clear history only when a user first navigates (no latest_response and no current history)
+        if not latest_response and 'islamic_history' not in session:
+            session.pop('islamic_history', None)
+        
+        return render_template("islamic_page.html", 
+                                history=session.get('islamic_history', []),
+                                answer=answer,
+                                thoughts=thoughts,
+                                validation=validation,
+                                source=source)
+    
+    # POST Request Logic
+    answer, thoughts, validation, source = "", "", "", ""
+    history = session.get('islamic_history', [])
+    
+    # This try/except block wraps the ENTIRE POST logic
+    try:
+        query = standardize_query(request.form.get("query", ""))
+        has_image = 'image' in request.files and request.files['image'].filename
+        
+        final_query = query # Default to the original query
+        
+        if has_image:
+            logger.info("Processing Multimodal RAG query for Islamic domain")
+            
+            file = request.files['image']
+            
+            upload_dir = "Uploads"
+            os.makedirs(upload_dir, exist_ok=True)
+            image_path = os.path.join(upload_dir, file.filename)
+            
+            try:
+                file.save(image_path)
+                file.close() 
+                
+                with open(image_path, "rb") as img_file:
+                    img_base64 = base64.b64encode(img_file.read()).decode("utf-8")
+                
+                vision_prompt = f"Analyze this image's main subject. User's query is: '{query}'"
+                message = HumanMessage(content=[{"type": "text", "text": vision_prompt}, {"type": "image_url", "image_url": f"data:image/jpeg;base64,{img_base64}"}])
+                visual_prediction = llm.invoke([message]).content
+
+                enhanced_query = (
+                    f'User Query: "{query}" '
+                    f'Context from an image provided by the LLM: "{visual_prediction}" '
+                    'Based on the user\'s query and the context from LLM, provide a comprehensive answer.'
+                )
+                logger.info(f"Create enchanced query : {enhanced_query}")
+                
+                final_query = enhanced_query 
+            
+            finally:
+                if os.path.exists(image_path):
+                    try:
+                        os.remove(image_path)
+                        logger.info(f"Successfully cleaned up {image_path}")
+                    except PermissionError as e:
+                        logger.warning(f"Could not remove {image_path} after processing. "
+                                       f"File may be locked. Error: {e}")
+            
+        elif query: # Only run text logic if there's a query and no image
+            logger.info("Processing Text RAG query for Islamic domain")
+            standalone_query = get_standalone_question(query, history,llm)
+            logger.info(f"Original Query: '{query}'")
+            print(f"📚 Using chat history with {len(history)} previous messages to create standalone query")
+            logger.info(f"Standalone Query: '{standalone_query}'")
+            final_query = standalone_query
+            
+        if not final_query: 
+            raise ValueError("No query or file provided.")
+        
+        agent = rag_systems['islamic']
+        if not agent: raise Exception("Islamic RAG system is not loaded.")
+        response_dict = agent.answer(final_query, chat_history=history)
+        answer, thoughts , validation, source = parse_agent_response(response_dict)
+        history.append(HumanMessage(content=query))
+        history.append(AIMessage(content=answer))
+
+    except Exception as e:
+        logger.error(f"Error on /islamic page: {e}", exc_info=True)
+        answer = f"An error occurred: {e}"
+        thoughts = traceback.format_exc()
+            
+    # Save updated history and LATEST RESPONSE DATA back to the session
+    session['islamic_history'] = history
+    session['latest_islamic_response'] = {
+        'answer': answer, 
+        'thoughts': thoughts, 
+        'validation': validation, 
+        'source': source
+    }
+    session.modified = True
+                        
+    logger.debug(f"Redirecting after saving latest response.")
+    return redirect(url_for('islamic_page'))
+
+@app.route("/islamic/clear")
+def clear_islamic_chat():
+    session.pop('islamic_history', None)
+    logger.info("Islamic chat history cleared.")
+    return redirect(url_for('islamic_page'))
+
+@app.route("/insurance", methods=["GET", "POST"])
+def insurance_page():
+    if request.method == "GET" :
+        latest_response = session.pop('latest_insurance_response',{})
+        
+        answer = latest_response.get('answer', "")
+        thoughts = latest_response.get('thoughts', "")
+        validation = latest_response.get('validation', "")
+        source = latest_response.get('source', "")
+        
+        if not latest_response and 'insurance_history' not in session:
+            session.pop('insurance_history', None)
+        
+        return render_template("insurance_page.html", # You will need to create this HTML file
+                                history=session.get('insurance_history', []),
+                                answer=answer,
+                                thoughts=thoughts,
+                                validation=validation,
+                                source=source)
+    
+    # POST Request Logic
+    answer, thoughts, validation, source = "", "", "", ""
+    history = session.get('insurance_history', [])
+    
+    try:
+        query = standardize_query(request.form.get("query", ""))
+        
+        if query:
+            logger.info("Processing Text RAG query for Insurance domain")
+            standalone_query = get_standalone_question(query, history, llm)
+            logger.info(f"Original Query: '{query}'")
+            logger.info(f"Standalone Query: '{standalone_query}'")
+            
+            agent = rag_systems['insurance']
+            if not agent: raise Exception("Insurance RAG system is not loaded.")
+            response_dict = agent.answer(standalone_query, chat_history=history)
+            answer, thoughts, validation, source = parse_agent_response(response_dict)
+            
+            history.append(HumanMessage(content=query))
+            history.append(AIMessage(content=answer))
+        else:
+            raise ValueError("No query provided.")
+
+    except Exception as e:
+        logger.error(f"Error on /insurance page: {e}", exc_info=True)
+        answer = f"An error occurred: {e}"
+        thoughts = traceback.format_exc()
+            
+    session['insurance_history'] = history
+    session['latest_insurance_response'] = {
+        'answer': answer, 
+        'thoughts': thoughts, 
+        'validation': validation, 
+        'source': source
+    }
+    session.modified = True
+                        
+    logger.debug(f"Redirecting after saving latest response.")
+    return redirect(url_for('insurance_page'))
+
+@app.route("/insurance/clear")
+def clear_insurance_chat():
+    session.pop('insurance_history', None)
+    logger.info("Insurance chat history cleared.")
+    return redirect(url_for('insurance_page'))
+
+@app.route("/about", methods=["GET"])
+def about():
+    return render_template("about.html")
+
+@app.route('/metrics/<domain>')
+def get_metrics(domain):
+    """API endpoint to get metrics for a specific domain."""
+    try:
+        if domain == "medical" and rag_systems['medical']:
+            stats = rag_systems['medical'].metrics_tracker.get_stats()
+        elif domain == "islamic" and rag_systems['islamic']:
+            stats = rag_systems['islamic'].metrics_tracker.get_stats()
+        elif domain == "insurance" and rag_systems['insurance']:
+            stats = rag_systems['insurance'].metrics_tracker.get_stats()
+        elif not rag_systems.get(domain):
+            return jsonify({"error": f"{domain} RAG system not loaded"}), 500
+        else:
+            return jsonify({"error": "Invalid domain"}), 400
+        
+        return jsonify(stats)
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/metrics/reset/<domain>', methods=['POST'])
+def reset_metrics(domain):
+    """Reset metrics for a domain (useful for testing)."""
+    try:
+        if domain == "medical" and rag_systems['medical']:
+            rag_systems['medical'].metrics_tracker.reset_metrics()
+        elif domain == "islamic" and rag_systems['islamic']:
+            rag_systems['islamic'].metrics_tracker.reset_metrics()
+        elif domain == "insurance" and rag_systems['insurance']:
+            rag_systems['insurance'].metrics_tracker.reset_metrics()
+        elif not rag_systems.get(domain):
+            return jsonify({"error": f"{domain} RAG system not loaded"}), 500
+        else:
+            return jsonify({"error": "Invalid domain"}), 400
+        
+        return jsonify({"success": True, "message": f"Metrics reset for {domain}"})
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+if __name__ == "__main__":
+    logger.info("Starting Flask app for deployment testing...")
+    # This port 7860 is what Hugging Face Spaces expects by default
+    app.run(host="0.0.0.0", port=7860, debug=False)

doc_qa.py

@@ -0,0 +1,751 @@
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain_classic import hub
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_classic.chains.combine_documents import create_stuff_documents_chain
+from langchain_core.tools import Tool
+from langchain_community.tools.tavily_search import TavilySearchResults
+from langchain_community.retrievers import BM25Retriever
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from langchain_core.output_parsers import JsonOutputParser
+from langchain_classic.agents import AgentExecutor, create_react_agent
+from langchain_core.documents import Document
+from langchain_core.messages import AIMessage, HumanMessage
+from langchain_chroma import Chroma
+from langchain_core.agents import AgentAction
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from flashrank import Ranker, RerankRequest
+from src.metrics_tracker import MetricsTracker
+import logging
+
+
+# Setup logging
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+class ContextRetriever:
+    def __init__(self, retriever):
+        self.retriever = retriever
+
+    def deduplicate_context(self, context_list): 
+        """Deduplicate context entries to avoid repetition."""
+        seen = set()
+        deduped = []
+        for item in context_list:
+            if item not in seen:
+                seen.add(item)
+                deduped.append(item)
+        return "\n".join(deduped) if deduped else "No relevant context found."
+
+    def retrieve(self, query, top_k=5):
+        """
+        Retrieve the top-k relevant contexts from ChromaDB based on the query.
+        
+        Args:
+            query (str): The query or prediction to search for.
+            top_k (int): Number of top results to return (default: 3).
+        
+        Returns:
+            str: Deduplicated context string from the top-k results.
+        """
+        logger.info(f"Retrieving for query: {query}")
+        try:
+            # Perform similarity search using ChromaDB retriever
+            results = self.retriever.invoke(query, k=top_k) 
+            logger.info(f"Retrieved documents: {[doc.metadata.get('source') for doc in results]}")
+            
+            # Extract the page content (context) from each result
+            contexts = [doc.page_content for doc in results]
+            logger.info(f"Context : {contexts}")
+            
+            # Deduplicate the contexts
+            deduped_context = self.deduplicate_context(contexts)
+            logger.info(f"Deduplicated context: {deduped_context}")
+            
+            return deduped_context
+        except Exception as e:
+            logger.error(f"Retrieval error: {str(e)}")
+            return "Retrieval failed due to error."
+
+class LLMComplexityAnalyzer:
+    """
+    Analyzes query complexity using an LLM to make a "managerial" decision
+    on the optimal retrieval strategy.
+    """
+    
+    def __init__(self, domain: str, llm: ChatGoogleGenerativeAI):
+        self.domain = domain
+        self.llm = llm
+        
+        self.system_prompt = (
+            "You are a 'Complexity Analyzer' manager for a RAG (Retrieval-Augmented Generation) system. "
+            "Your domain of expertise is: **{domain}**."
+            "\n"
+            "Your task is to analyze the user's query and determine its complexity. Based on this, "
+            "you will decide how many documents (k) to retrieve. More complex queries require "
+            "more documents to synthesize a good answer."
+            "\n"
+            "Here are the retrieval strategies:"
+            "1.  **simple**: For simple, direct fact-finding queries. (e.g., 'What is takaful?') "
+            "    - Set k = 5"
+            "2.  **moderate**: For queries that require explanation, some comparison, or have multiple parts. "
+            "    (e.g., 'What is the difference between madhab Shafi'i and Maliki on prayer?') "
+            "    - Set k = 10"
+            "3.  **complex**: For deep, nuanced, multi-step, or highly comparative/synthetic queries. "
+            "    (e.g., 'Explain in detail the treatment options for type 2 diabetes, comparing "
+            "    their side effects and suitability for elderly patients.')"
+            "    - Set k = 15"
+            "\n"
+            "Analyze the following query and provide your reasoning."
+            "\n"
+            "**IMPORTANT**: You MUST respond ONLY with a single, valid JSON object. Do not add any "
+            "other text. The JSON object must have these three keys:"
+            "-   `complexity`: (string) Must be one of 'simple', 'moderate', or 'complex'."
+            "-   `k`: (integer) Must be 5, 10, or 15, corresponding to the complexity."
+            "-   `reasoning`: (string) A brief 1-sentence explanation for your decision."
+        )
+        
+        self.prompt_template = ChatPromptTemplate.from_messages([
+            ("system", self.system_prompt.format(domain=self.domain)),
+            ("human", "{query}")
+        ])
+        
+        self.output_parser = JsonOutputParser()
+        
+        # This chain will output a parsed dictionary
+        self.chain = self.prompt_template | self.llm | self.output_parser
+        
+        logger.info(f"🧠 LLMComplexityAnalyzer initialized for '{self.domain}'")
+
+    def analyze(self, query: str) -> dict:
+        """
+        Analyzes query complexity using an LLM and returns the retrieval strategy.
+        """
+        logger.info(f"🧠 LLMComplexityAnalyzer: Analyzing query...")
+        
+        try:
+            # Invoke the chain to get the structured JSON output
+            result = self.chain.invoke({"query": query})
+            
+            # Add a 'score' field for compatibility
+            score_map = {"simple": 2, "moderate": 4, "complex": 6}
+            result['score'] = score_map.get(result.get('complexity'), 0)
+                
+            logger.info(f"🧠 LLM Decision: {result.get('complexity').upper()} (k={result.get('k')})")
+            logger.info(f"   Reasoning: {result.get('reasoning')}")
+            
+            return result
+            
+        except Exception as e:
+            # Fallback in case the LLM fails or returns bad JSON
+            logger.error(f"❌ LLMComplexityAnalyzer failed: {e}. Defaulting to 'moderate' strategy.")
+            return {
+                "complexity": "moderate",
+                "k": 12,
+                "score": 4,
+                "reasoning": "Fallback: LLM analysis or JSON parsing failed."
+            }
+
+
+class SwarmRetriever:
+    """
+    Multi-retriever swarm that executes parallel retrieval strategies.
+    Worker component that takes orders from LLMComplexityAnalyzer.
+    """
+    
+    def __init__(self, chroma_retriever, documents):
+        self.dense_retriever = chroma_retriever  # Semantic search
+        self.bm25_retriever = BM25Retriever.from_documents(documents)  # Keyword search
+        self.bm25_retriever.k = 20  # Set high, will be limited by k parameter
+        logger.info("✅ SwarmRetriever initialized (Dense + BM25 workers)")
+    
+    def retrieve_with_swarm(self, query: str, k: int) -> list:
+        """
+        Execute multi-retriever swarm with parallel workers.
+        """
+        logger.info(f"🐝 Swarm deployment: {2} workers, target k={k}")
+        
+        # Define worker tasks
+        retrieval_tasks = {
+            "dense_semantic": lambda: self.dense_retriever.invoke(query, k=k),
+            "bm25_keyword": lambda: self.bm25_retriever.invoke(query)[:k],
+        }
+        
+        # Execute workers in parallel
+        swarm_results = {}
+        with ThreadPoolExecutor(max_workers=2) as executor:
+            futures = {
+                executor.submit(task): name 
+                for name, task in retrieval_tasks.items()
+            }
+            
+            for future in as_completed(futures):
+                worker_name = futures[future]
+                try:
+                    results = future.result()
+                    swarm_results[worker_name] = results
+                    logger.info(f"  ✅ Worker '{worker_name}': {len(results)} docs")
+                except Exception as e:
+                    logger.error(f"  ❌ Worker '{worker_name}' failed: {e}")
+                    swarm_results[worker_name] = []
+        
+        # Combine and deduplicate documents
+        combined_docs = self._combine_and_deduplicate(swarm_results)
+        
+        return combined_docs
+    
+    def _combine_and_deduplicate(self, swarm_results: dict) -> list:
+        """Combine results from all workers and remove duplicates."""
+        all_docs = []
+        seen_content = set()
+        worker_contributions = {}
+        
+        for worker_name, docs in swarm_results.items():
+            for doc in docs:
+                # Use first 200 chars as hash to detect duplicates
+                content_hash = hash(doc.page_content[:200])
+                
+                if content_hash not in seen_content:
+                    seen_content.add(content_hash)
+                    
+                    # Tag document with source worker
+                    doc.metadata['swarm_worker'] = worker_name
+                    all_docs.append(doc)
+                    
+                    # Track contributions
+                    worker_contributions[worker_name] = \
+                        worker_contributions.get(worker_name, 0) + 1
+        
+        logger.info(f"🐝 Swarm combined: {len(all_docs)} unique docs")
+        logger.info(f"   Worker contributions: {worker_contributions}")
+        
+        return all_docs
+    
+class AgenticQA:
+    def __init__(self, config=None):  
+        logger.info("Initializing AgenticQA...")
+        
+        # Load a small, fast reranker model. This runs locally.
+        try:
+            self.reranker = Ranker(model_name="ms-marco-MiniLM-L-12-v2")
+            logger.info("FlashRank Reranker loaded successfully.")
+        except Exception as e:
+            logger.error(f"Failed to load FlashRank reranker: {e}")
+            self.reranker = None
+        
+        self.contextualize_q_system_prompt = (
+            "Given a chat history and the latest user question which might reference context in the chat history, "
+            "formulate a standalone question which can be understood without the chat history. "
+            "IMPORTANT: DO NOT provide any answers or explanations. ONLY rephrase the question if needed. "
+            "If the question is already clear and standalone, return it exactly as is. "
+            "Output ONLY the reformulated question, nothing else."
+        )
+        
+        self.contextualize_q_prompt = ChatPromptTemplate.from_messages(
+            [("system", self.contextualize_q_system_prompt),
+             MessagesPlaceholder("chat_history"),
+             ("human", "{input}")]
+        )
+        self.qa_system_prompt = (
+            "You are an assistant that answers questions in a specific domain for citizens mainly in Malaysia, "
+            "depending on the context. "
+            "You will receive:\n"
+            "  • domain = {domain}  (either 'medical', 'islamic' , or 'insurance')\n"
+            "  • context = relevant retrieved passages\n"
+            "  • user question\n\n"
+            "If the context does not contain the answer, **YOU MUST SAY 'I do not know'** or 'I cannot find that information in the provided documents.' Do not use your general knowledge.\n\n"
+            "Instructions based on domain:\n"
+            "1. If domain = 'medical' :\n"
+            "   - Answer the question in clear, simple layperson language, "
+            "   - Citing your sources (e.g. article name, section)."
+            "   - Add a medical disclaimer: “I am not a doctor…”.\n"
+            "2. If domain = 'islamic':\n"
+            "   - **ALWAYS present both Shafi'i AND Maliki perspectives** if the question is about fiqh/rulings\n"
+            "   - **Cite specific sources**: Always mention the book name (e.g., 'According to Muwatta Imam Malik...', 'Minhaj al-Talibin states...', 'Umdat al-Salik explains...')\n"
+            "   - **Structure answer as**:\n" 
+            "      - Shafi'i view (from Umdat al-Salik/Minhaj): [ruling with citation]\n"
+            "      - Maliki view (from Muwatta): [ruling with citation]\n"
+            "      - If they agree: mention the consensus\n"
+            "      - If they differ: present both views objectively without favoring one\n"
+            "   - **For hadith questions**: provide the narration text, source (book name, hadith number)\n "
+            "   - - **If ruling has EXCEPTIONS** (like 'except for...', 'unless...'), YOU MUST include them. "
+            "      If context doesn't show exceptions but the ruling seems absolute, indicate this uncertainty.\n"
+            "   - If the context does not contain relevant information from BOTH madhabs, acknowledge which sources you have "
+            "      (e.g., 'Based on Shafi'i sources only...') and suggest consulting additional madhab resources.\n"
+            "   - **Always end with**: 'This is not a fatwa. Consult a local scholar for guidance specific to your situation.'\n"
+            "   - Always include hadith narration or quran verse as evidence (if it exists) in the final response "
+            "   - Keep answers concise but comprehensive enough to show different scholarly views.\n\n"
+            
+            "3. If domain = 'insurance':\n"
+            "   - Your knowledge is STRICTLY limited to Etiqa Takaful (Motor and Car policies).\n"
+            "   - First, try to answer ONLY using the provided <context>.\n"
+            "   - **If the answer is not in the context, YOU MUST SAY 'I do not have information on that specific topic.'** Do not make up an answer.\n"
+            "   - If the user asks about other Etiqa products (e.g., medical, travel), you MUST use the 'EtiqaWebSearch' tool.\n"
+            "   - If the user asks about another insurance company (e.g., 'Prudential', 'Takaful Ikhlas'), state that you can only answer about Etiqa Takaful.\n"
+            "   - If the user asks a general insurance question (e.g., 'What is takaful?', 'What is an excess?'), use the 'GeneralWebSearch' tool.\n"
+                    
+            "4. For ALL domains: If the context does not contain the answer, do not make one up. Be honest.\n\n"
+            "Context:\n"
+            "{context}"
+            )
+
+        self.qa_prompt = ChatPromptTemplate.from_messages(
+            [("system", self.qa_system_prompt),
+             MessagesPlaceholder("chat_history"),
+             ("human", "{input}")]
+        )
+        self.llm = ChatGoogleGenerativeAI(model="gemini-2.5-flash",temperature=0.05) 
+        # --- START: NEW QUERY REFINER ---
+        self.refiner_system_prompt = (
+            "You are an expert search query refiner. Your task is to take a user's question "
+            "and rewrite it to be a perfect, concise search query for a database. "
+            "Remove all conversational fluff, emotion, and filler words. "
+            "Distill the query to its core semantic intent. "
+            "For example:"
+            "- 'Hi, I was wondering if I can touch a dog if I found it is cute?' becomes 'ruling on touching a dog in islam'"
+            "- 'What are the treatments for, like, a common cold?' becomes 'common cold treatment options'"
+            "- 'Tell me about diabetes' becomes 'what is diabetes'"
+            "Output ONLY the refined query, nothing else."
+        )
+        
+        self.refiner_prompt = ChatPromptTemplate.from_messages([
+            ("system", self.refiner_system_prompt),
+            ("human", "{query}")
+        ])
+        
+        self.refiner_chain = self.refiner_prompt | self.llm
+        logger.info("✅ Query Refiner chain initialized.")
+        # --- END: NEW QUERY REFINER ---
+
+        self.react_docstore_prompt = hub.pull("aallali/react_tool_priority")
+        self.answer_validator = AnswerValidatorAgent(self.llm)
+
+        self.retriever = None
+        self.agent_executor = None
+        self.tools = [] # Initialize the attribute
+        self.domain = "general"
+        self.answer_validator = None
+        self.retrieval_agent = None
+
+        if config:
+            logger.info(f"Configuring AgenticQA with provided config: {config}")
+            try:
+                collection_name = config["retriever"]["collection_name"]
+                persist_directory = config["retriever"]["persist_directory"]
+                self.domain = config.get("domain", "general") # Get domain from config
+                
+                # 1. Initialize the embedding function
+                embedding_function = GoogleGenerativeAIEmbeddings(model="models/text-embedding-004")
+
+                # 2. Connect to the persistent ChromaDB
+                db_client = Chroma(
+                    persist_directory=persist_directory,
+                    embedding_function=embedding_function,
+                    collection_name=collection_name
+                )
+
+                # 3. Set the retriever for this instance
+                self.retriever = db_client.as_retriever()
+                logger.info(f"✅ Successfully created retriever for collection '{collection_name}'")
+                # --- START: NEW SWARM INITIALIZATION ---
+                logger.info("Initializing Swarm components...")
+                # Get all documents from Chroma for BM25
+                all_docs_data = db_client.get()
+                docs_for_bm25 = [
+                    Document(page_content=content, metadata=meta)
+                    for content, meta in zip(
+                        all_docs_data['documents'], 
+                        all_docs_data['metadatas']
+                    )
+                ]
+                
+                # Initialize SwarmRetriever (Workers)
+                self.swarm_retriever = SwarmRetriever(self.retriever, docs_for_bm25)
+                
+                # Initialize LLMComplexityAnalyzer (Manager)
+                self.complexity_analyzer = LLMComplexityAnalyzer(self.domain, self.llm)
+                logger.info("✅ Swarm components (Manager + Workers) initialized.")
+                # --- END: NEW SWARM INITIALIZATION ---
+                self.metrics_tracker = MetricsTracker(save_path=f"metrics_{self.domain}.json")
+                logger.info("✅ Metrics tracker initialized")
+                # Initialize validator *after* setting domain
+                self.answer_validator = AnswerValidatorAgent(self.llm, self.domain)
+                 # --- This is the new, simple QA chain that will be used *after* reranking ---
+                self.qa_chain = create_stuff_documents_chain(self.llm, self.qa_prompt)
+
+                self._initialize_agent()
+                
+            except Exception as e:
+                logger.error(f"❌ Error during AgenticQA setup for '{self.domain}': {e}", exc_info=True)
+        else:
+            logger.warning("⚠️ AgenticQA initialized without a config. Retriever will be None.")
+
+    # --- 5. NEW UPGRADED RAG FUNCTION ---
+    # This is our new, smarter "worker" function that includes the reranker.
+    def _run_rag_with_reranking(self, query: str, chat_history: list) -> str:
+        """
+        Enhanced Swarm-RAG pipeline with adaptive retrieval and reranking.
+        
+        Pipeline:
+        1. Contextualize query
+        2. Refine query
+        3. ComplexityAnalyzer (Manager) determines optimal k
+        4. SwarmRetriever (Workers) deploys parallel retrievers with k
+        5. Rerank combined swarm results
+        6. Filter results by threshold
+        7. Generate Answer
+        """
+        logger.info(f"--- 🐝 SWARM RAG (with Reranker) PIPELINE RUNNING for query: '{query}' ---")
+        
+        if not self.reranker or not self.swarm_retriever or not self.complexity_analyzer:
+            logger.error("Swarm components or Reranker not initialized. Cannot perform RAG.")
+            return "Error: RAG components are not available."
+
+        try:
+            # 1. Contextualize query
+            standalone_query = query
+            if chat_history:
+                contextualize_chain = self.contextualize_q_prompt | self.llm
+                response = contextualize_chain.invoke({"chat_history": chat_history, "input": query})
+                standalone_query = response.content
+                logger.info(f"Contextualized query: '{standalone_query}'")
+            
+            # 2 - REFINE QUERY ---
+            logger.info("Refining query for search...")
+            response = self.refiner_chain.invoke({"query": standalone_query})
+            refined_query = response.content.strip()
+            logger.info(f"Refined query: '{refined_query}'")
+            
+            
+            # 3. Complexity analysis
+            analysis = self.complexity_analyzer.analyze(standalone_query)
+            k = analysis['k']
+            self._last_complexity_analysis = analysis
+            logger.info(f"Query complexity: {analysis['complexity'].upper()} | k={k}")
+            
+            # 4. Retrieve with Swarm (Workers)
+            swarm_docs = self.swarm_retriever.retrieve_with_swarm(standalone_query, k=k)
+            
+            if not swarm_docs:
+                logger.warning("Swarm Retriever found no documents.")
+                return "I do not know the answer to that as it is not in my documents."
+            
+            # 5. Format for Reranker
+            passages = [
+                {"id": i, "text": doc.page_content, "meta": doc.metadata} 
+                for i, doc in enumerate(swarm_docs)
+            ]
+            
+            # 6. Rerank
+            logger.info(f"Reranking {len(passages)} swarm-retrieved documents...")
+            rerank_request = RerankRequest(query=standalone_query, passages=passages)
+            reranked_results = self.reranker.rerank(rerank_request)
+            
+            top_score = reranked_results[0]['score'] if reranked_results else 0
+            logger.info(f"Reranking complete. Top score: {top_score:.3f}")
+
+            # 7. Filter 
+            threshold = 0.1
+            if self.domain == "islamic":
+                threshold = 0.05 
+            elif self.domain == "medical":
+                threshold = 0.15
+            else:
+                threshold = 0.10
+                
+            logger.info(f"Using threshold={threshold} for {self.domain} domain")
+            final_docs = []
+            worker_contributions = {}
+            
+            for result in reranked_results:
+                if result['score'] > threshold:
+                    # Re-create the Document object from reranked data
+                    doc = Document(
+                        page_content=result['text'],
+                        metadata=result.get('meta', {})
+                    )
+                    final_docs.append(doc)
+                    
+                    # Track worker contributions in final answer
+                    worker = result.get('meta', {}).get('swarm_worker', 'unknown')
+                    worker_contributions[worker] = \
+                        worker_contributions.get(worker, 0) + 1
+            
+            logger.info(f"Filtered to {len(final_docs)} documents above threshold {threshold}.")
+            logger.info(f"Final doc contributions: {worker_contributions}")
+            
+            self.metrics_tracker.log_worker_contribution(worker_contributions)
+            # 8. Respond
+            if not final_docs:
+                logger.warning("No documents passed the reranker threshold. Returning 'I don't know.'")
+                return "I do not know the answer to that as my document search found no relevant information."
+            
+            # Call the QA chain with the *reranked, filtered* docs
+            response = self.qa_chain.invoke({
+                "context": final_docs,
+                "chat_history": chat_history,
+                "input": query,
+                "domain": self.domain
+            })
+            
+            logger.info("🐝 Swarm RAG pipeline complete. Returning answer.")
+            return response
+
+        except Exception as e:
+            logger.error(f"Error in Swarm RAG pipeline: {e}", exc_info=True)
+            return "An error occurred while processing your request."
+        
+    def _initialize_agent(self):
+        """Build the ReAct agent"""
+        """A helper function to build the agent components."""
+
+        logger.info(f"Initializing agent for domain: '{self.domain}'")
+        self.context_retriever = ContextRetriever(self.retriever)
+        
+        # Store chat_history as instance variable so tools can access it
+        self._current_chat_history = []
+        
+        # We need a RAG chain for the tool
+        # history_aware_retriever = create_history_aware_retriever(self.llm, self.retriever, self.contextualize_q_prompt)
+        # question_answer_chain = create_stuff_documents_chain(self.llm, self.qa_prompt)
+        # rag_chain = create_retrieval_chain(history_aware_retriever, question_answer_chain)
+        
+        def rag_tool_wrapper(query: str) -> str:
+            """Wrapper to pass chat history to RAG pipeline."""
+            return self._run_rag_with_reranking(query, self._current_chat_history)
+        
+        self.tools = [
+            Tool(
+                name="RAG",
+                func=rag_tool_wrapper,
+                description=(f"Use this tool FIRST to search and answer questions about the {self.domain} domain using internal vector database.")
+            )
+            
+        ]
+        
+        # --- DOMAIN-SPECIFIC TOOLS ---
+        if self.domain == "insurance":
+            # Add a specific tool for searching Etiqa's website
+            etiqa_search_tool = TavilySearchResults(max_results=3)
+            etiqa_search_tool.description = "Use this tool to search the Etiqa Takaful website for products NOT in the RAG context (e.g., medical, travel)."
+            # This is a bit of a "hack" to force Tavily to search a specific site.
+            # We modify the function it calls.
+            original_etiqa_func = etiqa_search_tool.invoke
+            def etiqa_site_search(query):
+                return original_etiqa_func(f"site:etiqa.com.my {query}")
+            
+            self.tools.append(Tool(
+                name="EtiqaWebSearch",
+                func=etiqa_site_search,
+                description=etiqa_search_tool.description
+            ))
+            
+            # Add a general web search tool
+            self.tools.append(Tool(
+                name="GeneralWebSearch",
+                func=TavilySearchResults(max_results=2).invoke,
+                description="Use this tool as a fallback for general, non-Etiqa questions (e.g., 'What is takaful?')."
+            ))
+        elif self.domain == "islamic":
+            # Trusted Islamic sources for Malaysia
+            islamic_search = TavilySearchResults(max_results=3)
+            
+            def islamic_trusted_search(query):
+                # Search only trusted Malaysian Islamic authorities
+                sites = "site:muftiwp.gov.my OR site:zulkiflialbakri.com"
+                return islamic_search.invoke(f"{sites} {query}")
+            
+            self.tools.append(Tool(
+            name="TrustedIslamicSearch",
+            func=islamic_trusted_search,
+            description=(
+                "Use this tool if RAG has incomplete or no answer. "
+                "Searches ONLY trusted Malaysian Islamic sources: "
+                "Pejabat Mufti Wilayah Persekutuan (muftiwp.gov.my) and "
+                "Dr Zulkifli Mohamad Al Bakri (zulkiflialbakri.com/category/soal-jawab-agama/). "
+                "These follow Shafi'i madhab which is official in Malaysia."
+            )
+        ))
+        
+            # General fallback (last resort)
+            self.tools.append(Tool(
+                name="GeneralWebSearch",
+                func=TavilySearchResults(max_results=2).invoke,
+                description="Last resort: Use only for general Islamic terms or definitions not found in RAG or trusted sources."
+        ))
+        else:
+            # Medical and Islamic domains only get the general web search fallback
+            self.tools.append(Tool(
+                name="GeneralWebSearch",
+                func=TavilySearchResults(max_results=2).invoke,
+                description="Use this tool as a fallback if the RAG tool finds no relevant information or if the query is about a general topic."
+            ))
+        
+        agent = create_react_agent(llm=self.llm, tools=self.tools, prompt=self.react_docstore_prompt)
+        
+        self.agent_executor = AgentExecutor.from_agent_and_tools(
+            agent=agent,
+            tools=self.tools,
+            handle_parsing_errors=True,
+            verbose=True,
+            return_intermediate_steps=True,
+            max_iterations=5
+        )
+        logger.info(f"✅ Agent Executor(ReAct Agent) created successfully for '{self.domain}'.")
+    
+        
+    def answer(self, query, chat_history=None):
+        """
+        Process a query using the agent and returns a clean dictionary.
+        
+        Args:
+            query (str): User's question
+            chat_history (list): List of previous messages (AIMessage, HumanMessage)
+            
+        Returns:
+            dict: Contains 'answer', 'context', 'validation', 'source', 'thoughts'
+        """
+        if chat_history is None:
+            chat_history = []
+        self._current_chat_history = chat_history
+        if not self.agent_executor:
+            return {"answer": "Error: Agent not initialized.", "context": "", "validation": (False, "Init failed"), "source": "Error"}
+        # START TIMING
+        start_time = self.metrics_tracker.start_query()
+        print(f"\n📝 AGENTIC_QA PROCESSING QUERY: '{query}'")
+        
+        response = self.agent_executor.invoke({
+            "input": query,
+            "chat_history": chat_history,
+            "domain": self.domain, # Pass domain to agent
+            "metadata": {
+                "domain": self.domain
+            }
+        })
+        thoughts= ""
+        
+        final_answer = response.get("output", "Could not generate an answer")
+        
+        tool_used = None
+        if "intermediate_steps" in response:
+            thought_log= []
+            for step in response["intermediate_steps"]:
+                # --- FIX: Unpack the (Action, Observation) tuple first ---
+                action, observation = step
+                
+                if isinstance(action, AgentAction) and action.tool:
+                    tool_used = action.tool #Capture the last tool used
+                
+                # Append Thought, Action, Action Input & Observation 
+                thought_log.append(action.log)
+                thought_log.append(f"\nObservation: {str(observation)}\n---") 
+            
+            thoughts = "\n".join(thought_log)     
+
+        # Assign source based on the LAST tool used
+        if tool_used == "RAG":
+            source = "Etiqa Takaful Database" if self.domain == "insurance" else "Domain Database (RAG)"
+        elif tool_used == "EtiqaWebSearch":
+            source = "Etiqa Website Search"
+        elif tool_used == "TrustedIslamicSearch":
+            source = "Mufti WP & Dr Zul Search"
+        elif tool_used == "GeneralWebSearch":
+            source = "General Web Search"
+        else:
+            source = "Agent Logic"
+
+        logger.info(f"Tool used: {tool_used}, Source determined: {source}")
+        
+        # Retrieve context only if the RAG tool was used
+        # This call is inefficient (it runs a *second* retrieval), but it
+        # respects your architecture and works for logging.
+        context = "No RAG context retrieved."
+        if source.endswith("(RAG)") or source.startswith("Etiqa Takaful Database"):
+            if self.context_retriever:
+                context = self.context_retriever.retrieve(query)
+            else:
+                context = "RAG tool was used, but ContextRetriever not initialized."
+        elif "Web" in source:
+            context = "Web search results were used. See 'Observation' in thoughts log."
+        
+        validation = self.answer_validator.validate(query, final_answer, source=source)
+        # END TIMING
+        response_time = self.metrics_tracker.end_query(start_time)
+        
+        complexity_info = getattr(self, '_last_complexity_analysis', None)
+    
+        # LOG METRICS
+        self.metrics_tracker.log_query(
+            query=query,
+            domain=self.domain,
+            source=source,
+            complexity=complexity_info,
+            validation=validation,
+            response_time=response_time,
+            answer_preview=final_answer
+        )
+        return {"answer": final_answer, "context": context, "validation": validation, "source": source, "thoughts": thoughts,"response_time": response_time,
+        "complexity": complexity_info}
+
+class AnswerValidatorAgent:
+    def __init__(self, llm, domain="general"):
+        self.llm = llm
+        self.domain = domain
+        self.general_prompt = ChatPromptTemplate.from_messages([
+            ("system", (
+                "You are an answer validator. Check if the generated answer is factually correct "
+                "and relevant to the query. Return 'Valid' if the answer is correct and relevant, "
+                "or 'Invalid: [reason]' if not, where [reason] is a brief explanation of the issue."
+            )),
+            ("human", "Query: {query}\nAnswer: {answer}")
+        ])
+        self.medical_prompt = ChatPromptTemplate.from_messages([
+            ("system", (
+                "You are an answer validator. Check if the generated answer is factually correct, "
+                "relevant to the query, and consistent with known medical knowledge. "
+                "Return 'Valid' if the answer is correct and relevant, or 'Invalid: [reason]' if not, "
+                "where [reason] is a brief explanation of the issue. "
+                "**Pay close attention to contradictions.** If an answer gives advice and then "
+                "contradicts it (e.g., 'switch immediately' and then 'always consult your doctor first'), "
+                "it is **Invalid** because it is unsafe and confusing."
+            )),
+            ("human", "Query: {query}\nAnswer: {answer}")
+        ])
+        self.islamic_prompt = ChatPromptTemplate.from_messages([
+            ("system", (
+                "You are an answer validator for Islamic Fiqh or anything related to Islam. Check if the answer correctly addresses "
+                "the query based on the provided sources. The answer should be neutral and present "
+                "the required perspectives (e.g., Shafi'i and Maliki) if available. "
+                "Return 'Valid' if the answer is correct and relevant, or 'Invalid: [reason]' if not."
+            )),
+            ("human", "Query: {query}\nAnswer: {answer}")
+        ])
+
+    def validate(self, query, answer, source="RAG"):
+        if self.domain == "insurance":
+            logger.info("Skipping validation for insurance domain.")
+            return True, "Validation skipped for insurance domain."
+        
+        try:
+            # --- 11. IMPROVED VALIDATOR LOGIC ---
+            # Choose the right prompt based on domain and source
+            prompt = self.general_prompt # Default
+            if source == "RAG" or "Database" in source:
+                if self.domain == "medical":
+                    prompt = self.medical_prompt
+                elif self.domain == "islamic":
+                    prompt = self.islamic_prompt
+            
+            response = self.llm.invoke(prompt.format(query=query, answer=answer))
+            validation = response.content.strip()
+            logger.info(f"AnswerValidator result for query '{query}': {validation}")
+
+            if validation.lower().startswith("valid"):
+                return True, "Answer is valid and relevant."
+            elif validation.lower().startswith("invalid"):
+                reason = validation.split(":", 1)[1].strip() if ":" in validation else "No reason provided."
+                return False, reason
+            else:
+                return False, "Validation response format unexpected."
+        except Exception as e:
+            logger.error(f"AnswerValidator error: {str(e)}")
+            return False, "Validation failed due to error."

docker

@@ -0,0 +1,34 @@
+# Use Python 3.10 (Stable for LangChain & Flashrank)
+FROM python:3.10-slim
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Install system dependencies (needed for compiling packages like flashrank)
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first to cache dependencies
+COPY requirements.txt .
+
+# Install Python dependencies
+# --no-cache-dir keeps the image size smaller
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy all your code (app_1.py, src/, templates/, api.py) into the container
+COPY . .
+
+# Create writable directories for the database and file uploads
+# The 'chmod -R 777' gives the container permission to write here
+RUN mkdir -p /app/chroma_db && chmod -R 777 /app/chroma_db
+RUN mkdir -p /app/Uploads && chmod -R 777 /app/Uploads
+
+# Tell Docker that the container will listen on port 7860
+EXPOSE 7860
+
+# --- The Command to Run Your App ---
+# We use Gunicorn, a production-grade server
+# It looks for the 'app_1.py' file and the 'app' object inside it
+# --timeout 300 gives your RAG system 5 minutes to respond before timing out
+CMD ["gunicorn", "-b", "0.0.0.0:7860", "app_1:app", "--timeout", "300"]

medical_swarm.py

@@ -0,0 +1,149 @@
+import logging
+from dotenv import load_dotenv
+from langchain_google_genai import ChatGoogleGenerativeAI
+import traceback
+import os
+
+#1. SETUP
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+# Load env
+load_dotenv()
+google_api_key=os.getenv("GOOGLE_API_KEY")
+if not google_api_key:
+    raise ValueError("GOOGLE_API_KEY not found in environment variable.Make sure it is in .env file")
+
+# Agent Definition
+NO_SUGGESTIONS_INSTRUCTION= (
+    "This is a finalized medical report.Do not provide suggestions or improvements."
+    "Focus strictly on your assigned task based on the provided data"
+)
+class MedicalAgent:
+    def __init__(self, llm, name:str, role_prompt: str):
+        self.llm = llm
+        self.name = name
+        self.role_prompt= role_prompt
+    
+    def run(self, input_data:str):
+        
+        try:
+            logger.info(f"Agent '{self.name}' is processing")
+            full_prompt=f"{NO_SUGGESTIONS_INSTRUCTION}\n\n{self.role_prompt}\n\n{input_data}"
+            response=self.llm.invoke(full_prompt)
+            logger.info(f"Agent {self.name} finished")
+            return response.content
+        except Exception as e:
+            logger.error(f"Agent {self.name} error: {str(e)}")
+            traceback.print_exc()
+            return f"Error in agent {self.name}: {str(e)}"
+
+# Initialize SWARM
+llm=ChatGoogleGenerativeAI(model="gemini-2.5-pro",temperature=0.1,google_api_key=google_api_key)
+
+#Define specific roles for each agent in the team
+medical_data_extractor = MedicalAgent(
+    llm, "Medical Data Extractor",
+    "You are a specialized medical data extraction expert. Your role is to extract relevant medical information, focusing on key clinical indicators, test results, vital signs, and patient history from the provided text."
+    )
+diagnostic_specialist = MedicalAgent(
+    llm, "Diagnostic Specialist",
+    "You are a senior diagnostic physician. Your role is to analyze the provided symptoms, lab results, and clinical findings to develop a diagnostic assessment based solely on the data."
+    )
+treatment_planner = MedicalAgent(
+    llm, "Treatment Planner",
+    "You are an experienced clinical treatment specialist. Your role is to outline a prescribed treatment plan (pharmacological and non-pharmacological interventions) based on the provided diagnosis and data."
+    )
+specialist_consultant = MedicalAgent(
+    llm, "Specialist Consultant",
+    "You are a medical specialist consultant. Your role is to provide deep insights on the existing diagnosis and treatment plan, highlighting any potential complications or considerations based on the record."
+    )
+patient_care_coordinator = MedicalAgent(
+    llm, "Patient Care Coordinator (Orchestrator)",
+    "You are a patient care coordinator specializing in comprehensive healthcare management. Your primary role is to manage a team of specialist agents and synthesize their findings."
+    )
+
+# ORCHESTRATOR LOGIC
+def run_medical_swarm(document_text: str, initial_query: str,chat_history: list = None):
+    """
+    Orchestrates a swarm of medical agents to analyze document.
+    
+    Args:
+        document_text: The full text of medical record
+        initial_query: The initial question or goal of analysis
+    
+    Returns:
+        A string containing final, synthesized response
+    """
+    logger.info("--- MEDICAL SWARM INITIATED ---")
+    
+    # If there's a history, use it as the starting point. Otherwise, start fresh.
+    workspace = [f"Initial Patient Document:\n{document_text}", f"Initial Goal: '{initial_query}'"]
+    
+    # The Patient Care Coordinator is our orchestrator/manager
+    orchestrator= patient_care_coordinator
+    
+    # Limit the number of collab rounds to prevent infinite loops
+    for i in range(5):
+        logger.info(f"\n-- Swarm Iteration {i+1} --")
+        current_state = "\n\n".join(workspace)
+        
+        # The orchestrator reviews the current state and decide the next action.
+        orchestrator_prompt= f"""
+        You are the Patient Care Coordinator managing a team of specialist AI agents to analyze a medical case.
+        Review the current Case File below and decide the single next best action.
+
+        Your available specialists are:
+        - 'medical_data_extractor': Best for the first step to get raw data from the initial document.
+        - 'diagnostic_specialist': Best for forming a diagnosis after key data has been extracted.
+        - 'treatment_planner': Best for creating a plan after a clear diagnosis is available.
+        - 'specialist_consultant': Best for getting deeper insight on an existing diagnosis or plan.
+
+        Case File (Summary of work so far):
+        ---
+        {current_state}
+        ---
+
+        Based on the file, which specialist should be called next? Or, if you have a clear diagnosis, treatment plan, and specialist insight, is it time to write the final summary for the patient?
+
+        Respond with ONLY one of the following commands:
+        - "CALL: medical_data_extractor"
+        - "CALL: diagnostic_specialist"
+        - "CALL: treatment_planner"
+        - "CALL: specialist_consultant"
+        - "FINISH"
+        """
+        command=orchestrator.run(orchestrator_prompt).strip().upper()       
+        logger.info(f"Orchestrator Command: {command}")
+        
+        if command == "CALL: MEDICAL_DATA_EXTRACTOR":
+            report= medical_data_extractor.run(f"Original Document:\n{document_text}")
+            workspace.append(f"--- Extractor's Report ---\n{report}")
+            
+        elif command == "CALL: DIAGNOSTIC_SPECIALIST":
+            # Other agents get the whole workspace for context
+            report = diagnostic_specialist.run(f"Full Case File for Diagnosis:\n{current_state}")
+            workspace.append(f"--- Diagnostician's Report ---\n{report}")
+            
+        elif command == "CALL: TREATMENT_PLANNER":
+            report = treatment_planner.run(f"Full Case File for Treatment Plan:\n{current_state}")
+            workspace.append(f"--- Treatment Planner's Report ---\n{report}")
+            
+        elif command == "CALL: SPECIALIST_CONSULTANT":
+            report = specialist_consultant.run(f"Full Case File for Specialist Consultation:\n{current_state}")
+            workspace.append(f"--- Specialist Consultant's Report ---\n{report}")
+            
+        elif command == "FINISH":
+            logger.info("Orchestrator has decided the work is complete. Generating final summary.")
+            final_summary_prompt = f"You are the Patient Care Coordinator. Based on the complete case file below, write a comprehensive, patient-facing summary that coordinates all the findings.\n\nFull Case File:\n{current_state}"
+            final_answer = orchestrator.run(final_summary_prompt)
+            return final_answer
+        else:
+            logger.warning(f"Orchestrator gave an unknown command: '{command}'. Ending swarm.")
+            break
+        
+    # Fallback if the loop finishes without a "FINISH" command
+    logger.warning("Swarm reached max iterations. Finalizing with current data.")
+    final_fallback_prompt = f"You are the Patient Care Coordinator. The analysis time has expired. Summarize the findings from the case file below into a cohesive patient-facing report.\n\nFull Case File:\n{current_state}"
+    final_answer = orchestrator.run(final_fallback_prompt)
+    return final_answer        

metrics_tracker.py

@@ -0,0 +1,206 @@
+import time
+from datetime import datetime
+from collections import defaultdict
+import json
+import os
+
+class MetricsTracker:
+    """
+    Tracks system performance metrics across queries.
+    """
+    
+    def __init__(self, save_path="metrics_data.json"):
+        self.save_path = save_path
+        self.metrics = {
+            "total_queries": 0,
+            "rag_success": 0,
+            "web_search_fallback": 0,
+            "trusted_search_used": 0,
+            "general_search_used": 0,
+            "complexity_distribution": {
+                "simple": 0,
+                "moderate": 0,
+                "complex": 0
+            },
+            "domain_usage": {
+                "medical": 0,
+                "islamic": 0,
+                "insurance": 0
+            },
+            "response_times": [],
+            "worker_contributions": {
+                "dense_semantic": 0,
+                "bm25_keyword": 0
+            },
+            "validation_stats": {
+                "valid": 0,
+                "invalid": 0,
+                "skipped": 0
+            },
+            "query_history": []  # Store recent queries for analysis
+        }
+        
+        # Load existing metrics if available
+        self.load_metrics()
+    
+    def start_query(self):
+        """Start timing a query."""
+        return time.time()
+    
+    def end_query(self, start_time):
+        """Calculate and store query response time."""
+        response_time = time.time() - start_time
+        self.metrics["response_times"].append(response_time)
+        return response_time
+    
+    def log_query(self, query, domain, source, complexity=None, 
+                  validation=None, response_time=None, answer_preview=None):
+        """
+        Log a complete query with all its metadata.
+        
+        Args:
+            query (str): User's query
+            domain (str): Domain (medical, islamic, insurance)
+            source (str): Where answer came from (RAG, WebSearch, etc.)
+            complexity (dict): Complexity analysis result
+            validation (tuple): (is_valid, reason)
+            response_time (float): Time taken in seconds
+            answer_preview (str): First 100 chars of answer
+        """
+        self.metrics["total_queries"] += 1
+        
+        # Track domain usage
+        if domain in self.metrics["domain_usage"]:
+            self.metrics["domain_usage"][domain] += 1
+        
+        # Track source usage
+        if "RAG" in source or "Database" in source:
+            self.metrics["rag_success"] += 1
+        elif "Trusted" in source:
+            self.metrics["trusted_search_used"] += 1
+            self.metrics["web_search_fallback"] += 1
+        elif "Etiqa" in source:
+            self.metrics["web_search_fallback"] += 1
+        elif "Web" in source or "Search" in source:
+            self.metrics["general_search_used"] += 1
+            self.metrics["web_search_fallback"] += 1
+        
+        # Track complexity distribution
+        if complexity and "complexity" in complexity:
+            comp_level = complexity["complexity"]
+            if comp_level in self.metrics["complexity_distribution"]:
+                self.metrics["complexity_distribution"][comp_level] += 1
+        
+        # Track validation
+        if validation:
+            is_valid, reason = validation
+            if "skip" in reason.lower():
+                self.metrics["validation_stats"]["skipped"] += 1
+            elif is_valid:
+                self.metrics["validation_stats"]["valid"] += 1
+            else:
+                self.metrics["validation_stats"]["invalid"] += 1
+        
+        # Store query history (last 50 queries)
+        query_record = {
+            "timestamp": datetime.now().isoformat(),
+            "query": query[:100],  # Truncate long queries
+            "domain": domain,
+            "source": source,
+            "complexity": complexity.get("complexity") if complexity else None,
+            "k_used": complexity.get("k") if complexity else None,
+            "response_time": round(response_time, 2) if response_time else None,
+            "validated": is_valid if validation else None,
+            "answer_preview": answer_preview[:100] if answer_preview else None
+        }
+        
+        self.metrics["query_history"].append(query_record)
+        
+        # Keep only last 50 queries
+        if len(self.metrics["query_history"]) > 50:
+            self.metrics["query_history"] = self.metrics["query_history"][-50:]
+        
+        # Auto-save after each query
+        self.save_metrics()
+    
+    def log_worker_contribution(self, worker_stats):
+        """
+        Log which swarm workers contributed to the final answer.
+        
+        Args:
+            worker_stats (dict): e.g., {"dense_semantic": 5, "bm25_keyword": 3}
+        """
+        for worker, count in worker_stats.items():
+            if worker in self.metrics["worker_contributions"]:
+                self.metrics["worker_contributions"][worker] += count
+    
+    def get_stats(self):
+        """Get current statistics."""
+        total = self.metrics["total_queries"]
+        
+        if total == 0:
+            return {
+                "total_queries": 0,
+                "rag_success_rate": 0,
+                "web_search_rate": 0,
+                "avg_response_time": 0,
+                "complexity_distribution": self.metrics["complexity_distribution"],
+                "domain_usage": self.metrics["domain_usage"]
+            }
+        
+        # Calculate averages and percentages
+        avg_response_time = (
+            sum(self.metrics["response_times"]) / len(self.metrics["response_times"])
+            if self.metrics["response_times"] else 0
+        )
+        
+        stats = {
+            "total_queries": total,
+            "rag_success_rate": round((self.metrics["rag_success"] / total) * 100, 1),
+            "web_search_rate": round((self.metrics["web_search_fallback"] / total) * 100, 1),
+            "trusted_search_rate": round((self.metrics["trusted_search_used"] / total) * 100, 1),
+            "general_search_rate": round((self.metrics["general_search_used"] / total) * 100, 1),
+            "avg_response_time": round(avg_response_time, 2),
+            "median_response_time": self._get_median(self.metrics["response_times"]),
+            "complexity_distribution": self.metrics["complexity_distribution"],
+            "domain_usage": self.metrics["domain_usage"],
+            "worker_contributions": self.metrics["worker_contributions"],
+            "validation_stats": self.metrics["validation_stats"],
+            "recent_queries": self.metrics["query_history"][-10:]  # Last 10 queries
+        }
+        
+        return stats
+    
+    def _get_median(self, values):
+        """Calculate median of a list."""
+        if not values:
+            return 0
+        sorted_values = sorted(values)
+        n = len(sorted_values)
+        mid = n // 2
+        if n % 2 == 0:
+            return round((sorted_values[mid-1] + sorted_values[mid]) / 2, 2)
+        return round(sorted_values[mid], 2)
+    
+    def save_metrics(self):
+        """Save metrics to JSON file."""
+        try:
+            with open(self.save_path, 'w') as f:
+                json.dump(self.metrics, f, indent=2)
+        except Exception as e:
+            print(f"Warning: Could not save metrics: {e}")
+    
+    def load_metrics(self):
+        """Load metrics from JSON file if it exists."""
+        if os.path.exists(self.save_path):
+            try:
+                with open(self.save_path, 'r') as f:
+                    self.metrics = json.load(f)
+                print(f"✅ Loaded existing metrics from {self.save_path}")
+            except Exception as e:
+                print(f"Warning: Could not load metrics: {e}")
+    
+    def reset_metrics(self):
+        """Reset all metrics (useful for testing)."""
+        self.__init__(self.save_path)
+        self.save_metrics()