uploading pdfs...10

.gitignore

@@ -50,9 +50,4 @@ Thumbs.db
 
 # Documents (optional - you may want to track these)
 documents/
-# documents1/
-
-# documents/*.docx
-# documents/*.txt
-
 documents/*.pdf

Dockerfile

@@ -22,7 +22,7 @@ COPY backend/ ./backend/
 
 # Copy processed documents and vector data
 # Ensure documents and vectorstore folders exist (even if empty)
-RUN mkdir -p vectorstore
+RUN mkdir -p vectorstore documents
 
 # Copy vectorstore folder if it exists in the build context
 # Note: If vectorstore/ doesn't exist in repo, ensure an empty vectorstore/ folder exists
@@ -39,6 +39,7 @@ COPY frontend/build/ ./frontend/build/
 
 # Copy main app entry point
 COPY app.py .
+COPY files_upload.py ./files_upload.py
 
 
 # Expose port (Hugging Face Spaces uses 7860)

app.py

@@ -3,6 +3,8 @@ Hugging Face Spaces entry point
 This file serves both the FastAPI backend and React frontend
 """
 import os
+import subprocess
+import sys
 from contextlib import asynccontextmanager
 from pathlib import Path
 from fastapi import FastAPI, Request
@@ -12,6 +14,33 @@ from fastapi.middleware.cors import CORSMiddleware
 from dotenv import load_dotenv
 from backend.main import app as backend_app, initialize_rag_system
 
+
+def run_files_upload_script(script_path: Path) -> None:
+    """Run the files_upload.py helper to ensure documents are available."""
+    if not script_path.exists():
+        print(f"[root_app] Upload script not found at {script_path}")
+        return
+
+    print(f"[root_app] Running upload script: {script_path}")
+    try:
+        completed = subprocess.run(
+            [sys.executable, str(script_path)],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        if completed.stdout:
+            print(f"[root_app] upload script output:\n{completed.stdout}")
+        if completed.stderr:
+            print(f"[root_app] upload script warnings:\n{completed.stderr}")
+    except subprocess.CalledProcessError as exc:
+        print(f"[root_app] WARNING: upload script failed with code {exc.returncode}")
+        if exc.stdout:
+            print(f"[root_app] upload script stdout:\n{exc.stdout}")
+        if exc.stderr:
+            print(f"[root_app] upload script stderr:\n{exc.stderr}")
+        raise
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     """Lifespan context manager for FastAPI startup and shutdown"""
@@ -28,6 +57,7 @@ async def lifespan(app: FastAPI):
         project_root = Path(__file__).parent
         documents_dir = project_root / "documents"
         processed_json = project_root / "processed_documents.json"
+        upload_script = project_root / "files_upload.py"
         
         # Load environment variables from .env file (if it exists)
         env_path = project_root / ".env"
@@ -63,7 +93,10 @@ async def lifespan(app: FastAPI):
         print(f"[root_app] PROJECT_ROOT: {project_root}")
         print(f"[root_app] DOCUMENTS_DIR: {documents_dir} (exists: {documents_dir.exists()})")
         print(f"[root_app] PROCESSED_JSON: {processed_json} (exists: {processed_json.exists()})")
-        
+
+        # Run the upload script before initializing the RAG system
+        run_files_upload_script(upload_script)
+
         initialize_rag_system()
         print("[root_app] RAG system initialization completed")
     except Exception as e:

backend/main.py

@@ -107,9 +107,9 @@ def initialize_rag_system():
         print("[RAG Init] Starting initialization (import-time)")
         
         # Ensure documents folder exists
-        if not DOCUMENTS_DIR.exists():
-            DOCUMENTS_DIR.mkdir(parents=True, exist_ok=True)
-            print(f"[RAG Init] Created documents folder at {DOCUMENTS_DIR}")
+        # if not DOCUMENTS_DIR.exists():
+        #     DOCUMENTS_DIR.mkdir(parents=True, exist_ok=True)
+        #     print(f"[RAG Init] Created documents folder at {DOCUMENTS_DIR}")
         
         rag_system = RAGSystem()
 

files_upload.py

@@ -0,0 +1,42 @@
+from huggingface_hub import HfApi
+from pathlib import Path
+import os
+from dotenv import load_dotenv
+
+# Load environment variables from .env if present
+load_dotenv()
+
+# Get token from environment variable (more secure)
+token = os.getenv("HF_TOKEN")
+if not token:
+    raise ValueError("HF_TOKEN environment variable not set. Set it with: export HF_TOKEN='your_token_here'")
+
+# Initialize API with token
+api = HfApi(token=token)
+repo_id = "AldawsariNLP/Saudi-Law-AI-Assistant"
+
+# Upload all PDFs from local documents folder
+local_docs = Path("documents")
+pdf_files = list(local_docs.glob("*.pdf"))
+
+if not pdf_files:
+    print("No PDF files found in documents/ folder; skipping upload.")
+    exit(0)
+
+print(f"Found {len(pdf_files)} PDF file(s) to upload")
+for pdf_file in pdf_files:
+    print(f"Uploading {pdf_file.name}...")
+    try:
+        api.upload_file(
+            path_or_fileobj=str(pdf_file),
+            path_in_repo=f"documents/{pdf_file.name}",
+            repo_id=repo_id,
+            repo_type="space",
+            token=token,  # Also pass token here for safety
+        )
+        print(f"✓ Successfully uploaded {pdf_file.name}")
+    except Exception as e:
+        print(f"✗ Failed to upload {pdf_file.name}: {e}")
+        raise
+
+print("Upload complete!")

pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
     "langchain==0.1.16",
     "langchain-community==0.0.36",
     "openai>=1.50.0",
+    "huggingface_hub>=0.25.0",
     "httpx==0.27.2",
     "requests==2.31.0",
     "faiss-cpu==1.7.4",