Upload 3 files

app.py

@@ -1,198 +1,147 @@
-#!/usr/bin/env python3
-"""Main application for MANIT RAG Chatbot"""
-from typing import List, Dict
-import gradio as gr
-import numpy as np
-import faiss
-import pickle
-import os
-import time
-from sentence_transformers import SentenceTransformer
-from src.retrieval.semantic_retriever import SemanticRetriever
-from src.generation.response_generator import ResponseGenerator
-from config.settings import config
-
-print(f"--- SYSTEM INFO: This machine has {os.cpu_count()} CPU cores. ---")
-
-
-class MANITChatbot:
-    """Main chatbot class"""
-    
-    def __init__(self):
-        # Load vector store
-        self.embeddings = np.load(os.path.join(config.VECTOR_STORE_PATH, "embeddings.npy"))
-        self.faiss_index = faiss.read_index(os.path.join(config.VECTOR_STORE_PATH, "faiss_index.bin"))
-        
-        with open(os.path.join(config.VECTOR_STORE_PATH, "chunks.pkl"), "rb") as f:
-            self.chunks = pickle.load(f)
-        
-        with open(os.path.join(config.VECTOR_STORE_PATH, "bm25.pkl"), "rb") as f:
-            self.bm25 = pickle.load(f)
-        
-        with open(os.path.join(config.VECTOR_STORE_PATH, "relationships.pkl"), "rb") as f:
-            self.relationships = pickle.load(f)
-        
-        # Initialize models
-        self.embedding_model = SentenceTransformer(config.EMBEDDING_MODEL, device='cpu')
-        
-        # Initialize components
-        self.retriever = SemanticRetriever(
-            embedding_model=self.embedding_model,
-            faiss_index=self.faiss_index,
-            chunks=self.chunks,
-            bm25_index=self.bm25,
-            relationships=self.relationships
-        )
-        
-        self.generator = ResponseGenerator()
-        print("MANIT Chatbot initialized successfully!")
-    
-    def process_query(self, query: str) -> str:
-        """Process user query through full RAG pipeline"""
-        if not query.strip():
-            return "Please enter a question about MANIT Bhopal."
-        
-        start_time = time.time()
-        
-        try:
-            print(f"Processing query: {query}")
-            
-            # Retrieve relevant documents
-            retrieval_start = time.time()
-            retrieved_chunks = self.retriever.retrieve(query)
-            retrieval_time = time.time() - retrieval_start
-            
-            if not retrieved_chunks:
-                return "I couldn't find relevant information about this topic. Please try another question."
-            
-            print(f"Retrieved {len(retrieved_chunks)} chunks in {retrieval_time:.2f}s")
-            
-            # Format context
-            context = self._format_context(retrieved_chunks)
-            
-            # Check if web search is needed
-            web_context = ""
-            if self.generator.needs_web_search(query, context):
-                web_results = self.generator.web_search(query)
-                if web_results:
-                    web_context = "\n\n".join(web_results)
-            
-            # Generate response
-            generation_start = time.time()
-            response = self.generator.generate_response(query, context, web_context)
-            generation_time = time.time() - generation_start
-            
-            total_time = time.time() - start_time
-            print(f"Total processing time: {total_time:.2f}s (Retrieval: {retrieval_time:.2f}s, Generation: {generation_time:.2f}s)")
-            
-            return response
-            
-        except Exception as e:
-            print(f"Error processing query: {e}")
-            return "I encountered an error processing your question. Please try again."
-    def process_query_stream(self, query: str):
-        """Processes a user query and yields the response as a stream."""
-        if not query.strip():
-            yield "Please enter a question about MANIT Bhopal."
-            return
-
-        try:
-            print(f"Processing query: {query}")
-            
-            # 1. Retrieve documents (this part is not streamed)
-            retrieved_chunks = self.retriever.retrieve(query)
-            if not retrieved_chunks:
-                yield "I couldn't find relevant information about this topic. Please try another question."
-                return
-
-            context = self._format_context(retrieved_chunks)
-            
-            web_context = ""
-            if self.generator.needs_web_search(query, context):
-                web_results = self.generator.web_search(query)
-                if web_results:
-                    web_context = "\n\n".join(web_results)
-            
-            # 2. Yield the response from the streaming generator
-            yield from self.generator.generate_response_stream(query, context, web_context)
-            
-        except Exception as e:
-            print(f"Error processing query: {e}")
-            yield "I encountered an error processing your question. Please try again."
-    def _format_context(self, chunks: List[Dict]) -> str:
-        """Format context for the prompt"""
-        context_parts = []
-        
-        for chunk in chunks:
-            source = chunk['metadata']['source']
-            content = chunk['content']
-            context_parts.append(f"Source: {source}\nContent: {content}")
-        
-        return "\n\n---\n\n".join(context_parts)
-
-def create_interface():
-    """Create Gradio interface"""
-    chatbot_instance = MANITChatbot()
-    
-    def chat_fn(message, history):
-        """Function to handle chat interaction and stream the response."""
-        # Add the user's message to the history
-        history.append([message, ""])
-        
-        # Stream the response from the bot
-        response_stream = chatbot_instance.process_query_stream(message)
-        
-        # Loop through the stream and update the chatbot history
-        for chunk in response_stream:
-            history[-1][1] += chunk
-            yield history, ""  # Update the chatbot UI and keep the textbox clear
-            
-    # --- Your existing Gradio UI code ---
-    with gr.Blocks(
-        title="MANIT Bhopal Expert Assistant",
-        theme=gr.themes.Soft(),
-        css=""".gradio-container {max-width: 900px; margin: 0 auto;}"""
-    ) as demo:
-        
-        gr.Markdown("""
-        # 🎓 MANIT Bhopal Expert Assistant
-        *Powered by Advanced RAG Technology*
-        
-        Ask questions about programs, admissions, faculty, facilities, research, and more.
-        """)
-        
-        chatbot_ui = gr.Chatbot(
-            height=500,
-            show_label=False,
-            avatar_images=[None, "👨‍🎓"],
-            show_copy_button=True
-        )
-        
-        with gr.Row():
-            msg = gr.Textbox(
-                label="Your Question",
-                placeholder="Ask about MANIT Bhopal...",
-                scale=8,
-                lines=2
-            )
-            submit = gr.Button("Send", scale=1, variant="primary")
-        
-        gr.Examples(
-            examples=[
-                "Who is the director of MANIT?",
-                "Tell me about history of MANIT",
-                "What research facilities are available at MANIT?"
-            ],
-            inputs=msg,
-            label="Example Questions"
-        )
-        
-        # --- Updated event handlers for streaming ---
-        msg.submit(chat_fn, [msg, chatbot_ui], [chatbot_ui, msg])
-        submit.click(chat_fn, [msg, chatbot_ui], [chatbot_ui, msg])
-    
-    return demo
-
-if __name__ == "__main__":
-    demo = create_interface()
-    demo.launch()
+#!/usr/bin/env python3
+"""Optimized MANIT RAG Chatbot for HuggingFace Spaces"""
+import os
+import gradio as gr
+import time
+import logging
+from typing import List, Dict
+import warnings
+warnings.filterwarnings("ignore")
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Import your components
+from src.retrieval.semantic_retriever import SemanticRetriever
+from src.generation.response_generator import ResponseGenerator
+from config.settings import config
+
+class OptimizedMANITChatbot:
+    """HF Space optimized chatbot"""
+    
+    def __init__(self):
+        self.initialized = False
+        self.initialization_status = "Starting..."
+        self.setup_components()
+    
+    def setup_components(self):
+        """Initialize with progress tracking"""
+        try:
+            self.initialization_status = "Loading vector store..."
+            # Load your pre-built vector store components here
+            # (Same as your current implementation)
+            
+            self.initialization_status = "Warming up models..."
+            # Warm up embedding model
+            self.embedding_model.encode(["warmup"], show_progress_bar=False)
+            
+            self.initialization_status = "Ready!"
+            self.initialized = True
+            logger.info("MANIT Chatbot initialized successfully")
+            
+        except Exception as e:
+            logger.error(f"Initialization failed: {e}")
+            self.initialization_status = f"Error: {str(e)}"
+    
+    def process_query_stream(self, query: str):
+        """Stream response with error handling"""
+        if not self.initialized:
+            yield "System is still initializing. Please wait..."
+            return
+        
+        if not query.strip():
+            yield "Please enter a question about MANIT Bhopal."
+            return
+        
+        try:
+            # Your existing streaming logic
+            yield from self.generator.generate_response_stream(query, context, web_context)
+            
+        except Exception as e:
+            logger.error(f"Query processing error: {e}")
+            yield "I encountered an error. Please try rephrasing your question."
+
+def create_hf_interface():
+    """Create HF Space optimized interface"""
+    
+    # Initialize chatbot
+    chatbot_instance = OptimizedMANITChatbot()
+    
+    def chat_fn(message, history):
+        """Chat function with initialization check"""
+        if not chatbot_instance.initialized:
+            return history + [[message, f"⚠️ {chatbot_instance.initialization_status}"]], ""
+        
+        # Add user message
+        history.append([message, ""])
+        
+        # Stream bot response
+        for chunk in chatbot_instance.process_query_stream(message):
+            history[-1][1] += chunk
+            yield history, ""
+    
+    # Custom CSS for better mobile experience
+    custom_css = """
+    .gradio-container {max-width: 900px !important; margin: 0 auto !important;}
+    .message.user {background-color: #e3f2fd !important;}
+    .message.bot {background-color: #f5f5f5 !important;}
+    """
+    
+    with gr.Blocks(
+        title="MANIT Bhopal Expert Assistant",
+        theme=gr.themes.Soft(),
+        css=custom_css
+    ) as demo:
+        
+        gr.HTML("""
+        <div style="text-align: center; margin-bottom: 20px;">
+            <h1>🎓 MANIT Bhopal Expert Assistant</h1>
+            <p><em>Powered by Advanced RAG Technology</em></p>
+            <p>Ask questions about programs, admissions, faculty, facilities, research, and more.</p>
+        </div>
+        """)
+        
+        chatbot_ui = gr.Chatbot(
+            height=500,
+            show_label=False,
+            avatar_images=[None, "🎓"],
+            show_copy_button=True,
+            placeholder="Hi! I'm your MANIT Bhopal assistant. Ask me anything!"
+        )
+        
+        with gr.Row():
+            msg = gr.Textbox(
+                label="Your Question",
+                placeholder="Ask about MANIT Bhopal...",
+                scale=8,
+                lines=2,
+                max_lines=4
+            )
+            submit = gr.Button("Send", scale=1, variant="primary")
+        
+        gr.Examples(
+            examples=[
+                "Who is the director of MANIT?",
+                "What are the dispensary timings?",
+                "Tell me about the computer science department",
+                "What research facilities are available?",
+                "How do I apply for admission?"
+            ],
+            inputs=msg,
+            label="Example Questions"
+        )
+        
+        # Event handlers
+        msg.submit(chat_fn, [msg, chatbot_ui], [chatbot_ui, msg])
+        submit.click(chat_fn, [msg, chatbot_ui], [chatbot_ui, msg])
+    
+    return demo
+
+if __name__ == "__main__":
+    demo = create_hf_interface()
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False
+    )

build_vector_store.py

@@ -1,31 +1,48 @@
 #!/usr/bin/env python3
-"""Build the vector store from raw text files"""
+"""Rebuild vector store with optimized processing"""
 
 import os
 import sys
-from src.preprocessing.advanced_processor import AdvancedTextProcessor
+from pathlib import Path
+
+# Add the project root to Python path
+project_root = Path(__file__).parent
+sys.path.insert(0, str(project_root))
+
+from src.preprocessing.advanced_processor import OptimizedTextProcessor
 from config.settings import config
 
 def main():
-    print("Building MANIT RAG Vector Store...")
+    print("Rebuilding MANIT RAG Vector Store with optimized chunking...")
     
     # Check if raw texts exist
     if not os.path.exists(config.RAW_TEXT_PATH):
         print(f"Error: Raw text path {config.RAW_TEXT_PATH} does not exist")
         sys.exit(1)
     
-    # Process texts and build vector store
-    processor = AdvancedTextProcessor()
+    # Process with optimized processor
+    processor = OptimizedTextProcessor()
     chunks = processor.process_directory()
     
     if not chunks:
         print("No chunks were processed. Check your input files.")
         sys.exit(1)
     
-    print(f"Processed {len(chunks)} chunks from text files")
+    print(f"Processed {len(chunks)} optimized chunks")
     processor.build_vector_store(chunks)
     
-    print("Vector store built successfully!")
+    print("✅ Optimized vector store built successfully!")
+    print(f"📊 Total chunks: {len(chunks)}")
+    
+    # Display chunk type distribution
+    chunk_types = {}
+    for chunk in chunks:
+        chunk_type = chunk['metadata'].get('chunk_type', 'unknown')
+        chunk_types[chunk_type] = chunk_types.get(chunk_type, 0) + 1
+    
+    print("\n📈 Chunk distribution:")
+    for chunk_type, count in sorted(chunk_types.items()):
+        print(f"  - {chunk_type}: {count} chunks")
 
 if __name__ == "__main__":
     main()

evaluate.py

@@ -29,7 +29,7 @@ def evaluate_performance():
         "what is the name of person who registered the design for a paver block",
         "What are the objective for intellectual property rights cell at manit",
         "Tell me about mentorship program at MANIT",
-        "What are the recent events at manti"
+        "What are the recent events at manit"
     ]
     
     results = []