completed full flow

.Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.11-slim
+
+# Avoid Hugging Face cache permission errors
+ENV HF_HOME=/tmp/hf-cache
+ENV HF_DATASETS_CACHE=/tmp/hf-cache
+
+# Install necessary system dependencies
+RUN apt-get update && apt-get install -y \
+    git \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /code
+
+# Copy application and requirements
+COPY ./app /code/app
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Expose the FastAPI port
+EXPOSE 7860
+
+# Run the FastAPI app
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,13 @@
+---
+title: Auro Chatbot Backend
+emoji: 🔥
+colorFrom: yellow
+colorTo: yellow
+sdk: docker
+app_port: 7860
+base_path: /
+pinned: false
+---
+
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app/chatbot/demo_routes.py

@@ -1,10 +1,10 @@
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
 from app.chatbot.demo_rag import get_response
-from app.config import demo_chatbot_configs
+from app.config import demo_chatbot_configs, demo_form_submissions
 
 router = APIRouter()
-chatbot_sessions = {}  # chatbot_id -> loaded config/session
+chatbot_sessions = {}  # submission_id -> loaded config/session
 
 class ChatInput(BaseModel):
     question: str
@@ -13,22 +13,24 @@ class ChatInput(BaseModel):
     email: str
 
 
-@router.post("/demo/{chatbot_id}")
-async def demo_chat(chatbot_id: str, input: ChatInput):
-    # Lazy-load chatbot config
-    print(f"got question: {input.question} for chatbot_id: {chatbot_id} and session_id: {input.session_id}")
-    if chatbot_id not in chatbot_sessions:
-        rag_config = demo_chatbot_configs.find_one({"chatbot_id": chatbot_id})
+@router.post("/demo/{submission_id}")
+async def demo_chat(submission_id: str, input: ChatInput):
+    # Lazy-load chatbot config by submission_id
+    print(f"Got question: {input.question} for submission_id: {submission_id} and session_id: {input.session_id}")
+
+    if submission_id not in chatbot_sessions:
+        # Fetch RAG config using submission_id
+        rag_config = demo_chatbot_configs.find_one({"submission_id": submission_id})
         if not rag_config:
             raise HTTPException(status_code=404, detail="Chatbot not found")
-        chatbot_sessions[chatbot_id] = rag_config
-    
-    rag_config = chatbot_sessions[chatbot_id]
-    print(rag_config)
+        chatbot_sessions[submission_id] = rag_config
+
+    rag_config = chatbot_sessions[submission_id]
+    print(f"Loaded RAG config for submission_id {submission_id}: {rag_config}")
 
     config = {
-        'configurable': {
-            'thread_id': input.session_id
+        "configurable": {
+            "thread_id": input.session_id
         }
     }
 
@@ -40,4 +42,26 @@ async def demo_chat(chatbot_id: str, input: ChatInput):
         rag_config=rag_config,
         config=config
     )
-    return {"answer": response['response']}
+
+    return {"answer": response.get("response", "")}
+
+
+@router.get("/demo/status/{submission_id}")
+async def demo_chatbot_status(submission_id: str):
+    """
+    Returns the per-stage progress of a demo chatbot based on the submission_id.
+    """
+    config = demo_form_submissions.find_one({"submission_id": submission_id})
+    if not config:
+        raise HTTPException(status_code=404, detail="Demo submission not found")
+
+    return {
+        "submission_id": submission_id,
+        "stages": config.get("stages", {}),  # only true/false now
+        "company_id": config.get("company_id"),
+        "website_url": config.get("website_url"),
+        "chatbot_id": config.get("chatbot_id")
+    }
+
+
+

app/ingestion/demo_form_fetch_store.py

@@ -18,6 +18,14 @@ def store_demo_chatbot(ingest_data: ChatbotIngest):
         data_dict['company_id'] = company_id
         data_dict['chatbot_id'] = chatbot_id
 
+        #Add status field
+        data_dict['stages'] = {
+            'form_data_extracted': True,
+            'website_scraped': False,
+            "content_embedded": False,
+            "rag_ready": False
+        }
+
         # Insert into MongoDB
         demo_form_submissions.insert_one(data_dict)
         print(f"Chatbot {chatbot_id} ingested successfully for company {ingest_data.company_name}")

app/ingestion/models.py

@@ -3,6 +3,7 @@ from typing import List, Optional
 from datetime import datetime, timezone
 
 class ChatbotIngest(BaseModel):
+    submission_id: str
     company_name: str
     chatbot_name: Optional[str] = None
     website_url: HttpUrl

app/ingestion/rag_setup.py

@@ -178,6 +178,7 @@ def store_demo_rag_config(chatbot_id, company_id, ingest: ChatbotIngest) -> None
     Stores the RAG configuration prompt for the demo chatbot in MongoDB.
     """
     demo_rag_dict = {
+        "submission_id": ingest.submission_id,
         "chatbot_id": chatbot_id,
         "company_id": company_id,
         "company_name": ingest.company_name,
@@ -187,7 +188,7 @@ def store_demo_rag_config(chatbot_id, company_id, ingest: ChatbotIngest) -> None
                 "name": "all",
                 "collection": f"chatbot_{chatbot_id}",
                 "top_k": 25,
-                "filter_score": 50
+                "filter_score": 0.8
             }
         ]
     }

app/ingestion/routes.py

@@ -10,6 +10,7 @@ logger.setLevel(logging.INFO)
 
 router = APIRouter()
 
+FRONTEND_BASE_URL = "http://localhost:8000"
 
 @router.post("/ingest")
 async def ingest_chatbot_webhook(payload: dict,background_tasks: BackgroundTasks):
@@ -32,7 +33,7 @@ async def ingest_chatbot_webhook(payload: dict,background_tasks: BackgroundTasks
 
         return {
             "chatbot_id": chatbot_id,
-            "demo_url": f"/chatbot/demo/{chatbot_id}",
+            "demo_url": f"{FRONTEND_BASE_URL}/chatbot/demo/{chatbot_id}",
             "message": "Demo chatbot ingestion succesfull"
         }
     except Exception as e:

app/ingestion/utils.py

@@ -11,6 +11,9 @@ def extract_tally_payload(payload: dict):
     # Map key -> field
     key_map = {f["key"]: f for f in fields}
 
+    # Submission ID
+    submission_id = payload.get("data", {}).get("submissionId")
+
     # Company info
     company_name = key_map.get("question_kyp5Dd", {}).get("value")
     chatbot_name = key_map.get("question_vAp47X", {}).get("value") or None
@@ -46,6 +49,7 @@ def extract_tally_payload(payload: dict):
     submitted_at = datetime.fromisoformat(submitted_at_str.replace("Z", "+00:00"))
 
     return {
+        "submission_id": submission_id,
         "company_name": company_name,
         "chatbot_name": chatbot_name,
         "website_url": website_url,

app/ingestion/workers.py

@@ -1,22 +1,54 @@
-# workers.py
 from app.ingestion.demo_form_fetch_store import get_chatbot_config
 from app.ingestion.models import ChatbotIngest
 from app.ingestion.rag_setup import scrape_website, chunk_and_embed, store_demo_rag_config
+from app.config import demo_form_submissions
 
-
-def build_rag_for_chatbot(chatbot_id: str) -> None:
-    """
-    Full synchronous RAG build pipeline for a demo chatbot.
-    """
-
+def build_rag_for_chatbot(chatbot_id: str):
     config = get_chatbot_config(chatbot_id)
     if not config or not config.get("website_url"):
+        demo_form_submissions.update_one(
+            {"chatbot_id": chatbot_id},
+            {"$set": {"stages.rag_ready": False, "error": "Missing website URL"}}
+        )
         raise ValueError("Chatbot config missing website_url")
 
-    pages = scrape_website(config["website_url"])
+    try:
+        # Stage: Website scraping
+        demo_form_submissions.update_one(
+            {"chatbot_id": chatbot_id},
+            {"$set": {"stages.website_scraped": False}}
+        )
+        pages = scrape_website(config["website_url"])
+        demo_form_submissions.update_one(
+            {"chatbot_id": chatbot_id},
+            {"$set": {"stages.website_scraped": True}}
+        )
+
+        # Stage: Chunking & embedding
+        demo_form_submissions.update_one(
+            {"chatbot_id": chatbot_id},
+            {"$set": {"stages.content_embedded": False}}
+        )
+        chunk_and_embed(chatbot_id=chatbot_id, pages=pages)
+        demo_form_submissions.update_one(
+            {"chatbot_id": chatbot_id},
+            {"$set": {"stages.content_embedded": True}}
+        )
+
+        # Stage: RAG ready
+        store_demo_rag_config(
+            chatbot_id=chatbot_id,
+            company_id=config['company_id'],
+            ingest=ChatbotIngest(**config)
+        )
+        demo_form_submissions.update_one(
+            {"chatbot_id": chatbot_id},
+            {"$set": {"stages.rag_ready": True}}
+        )
 
-    chunk_and_embed(
-        chatbot_id=chatbot_id,
-        pages=pages,
-    )
-    store_demo_rag_config(chatbot_id=chatbot_id, company_id=config['company_id'], ingest=ChatbotIngest(**config))
+    except Exception as e:
+        demo_form_submissions.update_one(
+            {"chatbot_id": chatbot_id},
+            {"$set": {"stages.rag_ready": False, "error": str(e)}}
+        )
+        raise