Initial commit

README.md

@@ -0,0 +1,39 @@
+# 🌐 RAGBot 
+
+A cloud-native, multi-tenant AI chatbot that instantly learns the content of any website and answers questions in real-time. Built for the Cloud Computing final project.
+
+## 🚀 Key Features
+
+* **Universal Ingestion:** Crawls and indexes any provided URL (e.g., University sites, NGOs, Government portals) on-demand.
+* **SaaS Architecture:** Uses **Multi-Tenancy** via Vector Database Namespacing to isolate customer data within a single index.
+* **RAG Pipeline:** Combines **Semantic Search** (Pinecone) with **LLM Generation** (Gemini 2.0 Flash) for hallucination-free answers.
+* **Client-Server Model:** Decoupled FastAPI Backend and Chrome Extension Frontend.
+* **Asynchronous Processing:** Background workers handle heavy scraping tasks without blocking the UI.
+
+## 🛠️ Tech Stack
+
+### **Backend (Cloud Engine)**
+* **Framework:** FastAPI (Python)
+* **Vector Database:** Pinecone (Serverless AWS)
+* **LLM:** Google Gemini 2.0 Flash
+* **Crawler:** Trafilatura (Sitemap & Content Discovery)
+* **Embeddings:** Sentence-Transformers (`all-MiniLM-L6-v2`)
+
+### **Frontend (Client)**
+* **Interface:** Google Chrome Extension (Manifest V3)
+* **Interaction:** Real-time Polling & Dynamic UI
+
+## 📂 Project Structure
+
+```bash
+├── server.py           # Main FastAPI entry point
+├── crawler.py          # Logic for sitemap parsing & scraping
+├── vectorstore.py      # Pinecone batching & management
+├── config.py           # Environment & Logging setup
+├── utils.py            # Security & URL validation
+├── schemas.py          # Pydantic data models
+├── extension/          # Chrome Extension Source Code
+│   ├── manifest.json
+│   ├── popup.html
+│   └── popup.js
+└── requirements.txt    # Python Dependencies

api/config.py

@@ -0,0 +1,43 @@
+import os
+import sys
+import logging
+from dotenv import load_dotenv
+from vectorstore import VectorStoreManager
+import google.generativeai as genai
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[logging.StreamHandler(sys.stdout)],
+)
+
+logger = logging.getLogger(__name__)
+
+load_dotenv()
+
+GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", "")
+PINECONE_API_KEY = os.environ.get("PINECONE_API_KEY", "")
+INDEX_NAME = "production-rag-bot"
+
+
+def log_critical(api_key: str):
+    logger.critical(f"CRITICAL: {api_key} is missing from .env")
+    sys.exit(1)
+
+
+if not GOOGLE_API_KEY:
+    log_critical("PINECONE_API_KEY")
+
+if not PINECONE_API_KEY:
+    log_critical("PINECONE_API_KEY")
+
+
+try:
+    vs = VectorStoreManager(api_key=PINECONE_API_KEY, index_name=INDEX_NAME)
+
+    genai.configure(api_key=GOOGLE_API_KEY)  # type: ignore
+    flash = genai.GenerativeModel("gemini-2.0-flash")  # type: ignore
+
+except Exception as e:
+    logger.critical(f"Startup Failure: {e}")
+    sys.exit(1)

api/crawler.py

@@ -0,0 +1,90 @@
+import time
+import trafilatura
+from trafilatura.sitemaps import sitemap_search
+from utils import logger, is_valid_url
+
+
+def smart_chunk(text: str, chunk_size=500) -> list[str]:
+    """
+    Respects semantic boundaries.
+    Splits by Paragraphs (\n\n) -> Sentences (. ) -> Characters.
+    """
+    if not text:
+        return []
+
+    chunks = []
+    # split by paragraphs first (strongest delimiter)
+    paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
+
+    current_chunk = ""
+
+    for para in paragraphs:
+        # If adding this paragraph exceeds size, push current chunk and start new
+        if len(current_chunk) + len(para) > chunk_size:
+            if current_chunk:
+                chunks.append(current_chunk.strip())
+
+            # If the paragraph itself is massive, we must split it by sentence
+            if len(para) > chunk_size:
+                sentences = para.split(". ")
+                current_chunk = ""
+                for sent in sentences:
+                    if len(current_chunk) + len(sent) > chunk_size:
+                        chunks.append(current_chunk.strip())
+                        current_chunk = sent + ". "
+                    else:
+                        current_chunk += sent + ". "
+            else:
+                current_chunk = para + "\n"  # Start new chunk with this paragraph
+        else:
+            current_chunk += para + "\n"
+
+    if current_chunk:
+        chunks.append(current_chunk.strip())
+
+    return chunks
+
+
+def crawl_website(base_url: str, limit: int = 15):
+    """
+    Generator function that yields processed chunks one by one.
+    """
+    logger.info(f"Starting crawl for {base_url}")
+
+    # 1. Discovery
+    urls = sitemap_search(base_url)
+    if not urls:
+        logger.warning("No sitemap found. Fallback to base URL.")
+        urls = [base_url]
+
+    # 2. Filtering
+    valid_urls = [u for u in urls if is_valid_url(u, base_url)]
+    logger.info(f"Found {len(urls)} URLs, {len(valid_urls)} valid.")
+
+    # 3. Crawl Loop
+    count = 0
+    for link in valid_urls:
+        if count >= limit:
+            break
+
+        try:
+            # Etiquette: Sleep 0.5s between requests
+            time.sleep(0.5)
+
+            downloaded = trafilatura.fetch_url(link)
+            if not downloaded:
+                continue
+
+            text = trafilatura.extract(downloaded, include_comments=False)
+            if not text:
+                continue
+
+            chunks = smart_chunk(text)
+
+            # Yield result to main app
+            yield link, chunks
+            count += 1
+
+        except Exception as e:
+            logger.error(f"Failed to crawl {link}: {e}")
+            continue

api/schemas.py

@@ -0,0 +1,16 @@
+from typing_extensions import Annotated
+from pydantic import BaseModel, Field, StringConstraints, HttpUrl
+
+
+class ChatRequest(BaseModel):
+    message: Annotated[
+        str,
+        StringConstraints(
+            strip_whitespace=True, strict=True, min_length=1, max_length=4000
+        ),
+    ]
+    url: Annotated[HttpUrl, Field(description="A valid HTTP/HTTPS URL only.")]
+
+
+class IngestRequest(BaseModel):
+    url: Annotated[HttpUrl, Field(description="A valid HTTP/HTTPS URL only.")]

api/server.py

@@ -0,0 +1,132 @@
+from fastapi import FastAPI, BackgroundTasks, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+
+from utils import get_namespace_id, logger
+from crawler import crawl_website
+from schemas import ChatRequest, IngestRequest
+from config import vs, flash
+
+app = FastAPI(title="RAG Chatbot API", version="1.0")
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=False,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+def background_ingest_task(url: str, namespace_id: str):
+    logger.info(f"Background crawl STARTED for: {url}")
+    try:
+        # Limit set to 25 to balance speed/completeness
+        crawler_gen = crawl_website(url, limit=25)
+
+        buffer = []
+        for source_url, chunks in crawler_gen:
+            for chunk in chunks:
+                buffer.append({"text": chunk, "source": source_url})
+
+                if len(buffer) >= 50:
+                    vs.batch_upsert(buffer, namespace_id)
+                    buffer = []
+
+        if buffer:
+            vs.batch_upsert(buffer, namespace_id)
+
+        logger.info(f"Background crawl COMPLETED for {namespace_id}")
+    except Exception as e:
+        logger.error(f"Background task failed for {url}: {e}")
+
+
+@app.get("/")
+def check_health():
+    return {"status": "online", "system": "RAG-Chatbot v2.0"}
+
+
+@app.post("/check")
+def check_endpoint(req: IngestRequest):
+    url_str = str(req.url)
+    namespace_id = get_namespace_id(url_str)
+
+    try:
+        stats = vs.index.describe_index_stats()
+        exists = namespace_id in stats.namespaces
+
+        count = 0
+        if exists:
+            count = stats.namespaces[namespace_id].vector_count
+
+        return {
+            "exists": exists and count > 0,
+            "namespace": namespace_id,
+            "vector_count": count,
+        }
+    except Exception as e:
+        logger.error(f"Check failed: {e}")
+        return {"exists": False, "error": str(e)}
+
+
+@app.post("/ingest")
+def ingest_endpoint(req: IngestRequest, background_tasks: BackgroundTasks):
+    url_str = str(req.url)
+    namespace_id = get_namespace_id(url_str)
+
+    background_tasks.add_task(background_ingest_task, url_str, namespace_id)
+    logger.info(f"Background ingest dispatched: {url_str} -> {namespace_id}")
+
+    return {
+        "status": "processing",
+        "message": "Ingestion started in background.",
+        "namespace": namespace_id,
+    }
+
+
+@app.post("/chat")
+def chat_endpoint(req: ChatRequest):
+    url_str = str(req.url)
+    namespace_id = get_namespace_id(url_str)
+
+    # 1. Retrieval
+    results = vs.query_namespace(req.message, namespace_id)
+
+    contexts = []
+    sources = set()
+
+    if results and results.matches:
+        for match in results.matches:
+            if match.metadata:
+                text = match.metadata.get("text", "")
+                src = match.metadata.get("source", None)
+                if text:
+                    contexts.append(text)
+                if src:
+                    sources.add(src)
+
+    if not contexts:
+        return {
+            "answer": "I haven't learned this website yet. Please click 'Train' first!",
+            "sources": [],
+        }
+
+    # 2. Prompting
+    context_blob = "\n\n".join(contexts[:5])
+    prompt = f"""
+    You are a helpful assistant for the website: {url_str}.
+    Use ONLY the context provided below to answer the user's question.
+    If the answer is not in the context, say "I don't have that information."
+
+    CONTEXT:
+    {context_blob}
+
+    USER QUESTION:
+    {req.message}
+    """
+
+    try:
+        response = flash.generate_content(prompt)
+        return {"answer": response.text, "sources": list(sources)}
+    except Exception as e:
+        logger.error(f"LLM Error: {e}")
+        raise HTTPException(status_code=500, detail="AI Service Error")

api/utils.py

@@ -0,0 +1,60 @@
+import hashlib
+import re
+import logging
+from urllib.parse import urlparse
+
+logger = logging.getLogger(__name__)
+
+
+def get_namespace_id(url: str) -> str:
+    """
+    Generates a collision-resistant namespace ID.
+    Format: clean_name_MD5hash
+    """
+    clean_name = re.sub(r"https?://(www\.)?", "", url)
+    clean_name = re.sub(r"[^a-zA-Z0-9]", "_", clean_name)
+    clean_name = clean_name.strip("_")[:30]
+
+    url_hash = hashlib.md5(url.encode("utf-8")).hexdigest()[:6]
+
+    return f"{clean_name}_{url_hash}"
+
+
+def is_valid_url(url: str, base_domain: str) -> bool:
+    """
+    Enforces strict crawl scope.
+    """
+    try:
+        parsed = urlparse(url)
+        base_parsed = urlparse(base_domain)
+
+        # 1. Scheme Check
+        if parsed.scheme not in ["http", "https"]:
+            return False
+
+        # 2. Strict Domain Check (Ends with pattern to prevent 'google.com.evil.com')
+        # We allow subdomains (e.g. portal.batstateu.edu.ph)
+        if not parsed.netloc.endswith(base_parsed.netloc):
+            return False
+
+        # 3. File Extension Check
+        ignored_exts = [
+            ".pdf",
+            ".jpg",
+            ".png",
+            ".gif",
+            ".css",
+            ".js",
+            ".docx",
+            ".xlsx",
+            ".xml",
+            ".zip",
+            ".rar",
+            ".mp4",
+        ]
+        if any(parsed.path.lower().endswith(ext) for ext in ignored_exts):
+            return False
+
+        return True
+    except Exception:
+        return False

api/vectorstore.py

@@ -0,0 +1,70 @@
+from typing import Any
+import uuid
+from pinecone import Pinecone, ServerlessSpec
+from sentence_transformers import SentenceTransformer
+from utils import logger
+
+
+class VectorStoreManager:
+    # Add ': str' to api_key and index_name
+    def __init__(self, api_key: str, index_name: str, region: str = "us-east-1"):
+        self.pc = Pinecone(api_key=api_key)
+        self.index_name = index_name
+        self.embed_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+        self.dimension = 384
+
+        # ... rest of the code is fine
+
+        # Ensure Index Exists
+        if self.index_name not in [i.name for i in self.pc.list_indexes()]:
+            logger.info(f"Creating index {self.index_name}...")
+            self.pc.create_index(
+                name=self.index_name,
+                dimension=self.dimension,
+                metric="cosine",
+                spec=ServerlessSpec(cloud="aws", region=region),
+            )
+        self.index = self.pc.Index(self.index_name)
+
+    def batch_upsert(self, data_buffer, namespace, batch_size=100):
+        """
+        Upserts vectors in chunks to avoid timeout.
+        data_buffer: list of (text, source_url)
+        """
+        if not data_buffer:
+            return
+
+        # 1. Batch Embed (Much faster than 1-by-1)
+        texts = [item["text"] for item in data_buffer]
+        embeddings = self.embed_model.encode(texts)
+
+        # 2. Prepare Vectors
+        vectors = []
+        for i, (item, vector) in enumerate(zip(data_buffer, embeddings)):
+            vector_id = str(uuid.uuid4())
+            metadata = {"text": item["text"], "source": item["source"]}
+            vectors.append((vector_id, vector.tolist(), metadata))
+
+        # 3. Batch Upload to Pinecone
+        total_vectors = len(vectors)
+        for i in range(0, total_vectors, batch_size):
+            batch = vectors[i : i + batch_size]
+            try:
+                self.index.upsert(vectors=batch, namespace=namespace)
+                logger.info(f"Upserted batch {i} to {i + len(batch)} into {namespace}")
+            except Exception as e:
+                logger.error(f"Upsert failed for batch {i}: {e}")
+
+    def query_namespace(self, query_text, namespace, top_k=5) -> Any:
+        query_vector = self.embed_model.encode(query_text).tolist()
+        try:
+            results = self.index.query(
+                vector=query_vector,
+                top_k=top_k,
+                include_metadata=True,
+                namespace=namespace,
+            )
+            return results
+        except Exception as e:
+            logger.error(f"Query failed: {e}")
+            return None

requirements.txt

@@ -0,0 +1,101 @@
+altair==5.5.0
+annotated-doc==0.0.4
+annotated-types==0.7.0
+anyio==4.11.0
+attrs==25.4.0
+babel==2.17.0
+beautifulsoup4==4.14.2
+blinker==1.9.0
+cachetools==6.2.2
+certifi==2025.11.12
+charset-normalizer==3.4.4
+click==8.3.1
+courlan==1.3.2
+dateparser==1.2.2
+fastapi==0.122.0
+filelock==3.20.0
+fsspec==2025.10.0
+gitdb==4.0.12
+GitPython==3.1.45
+google-ai-generativelanguage==0.6.15
+google-api-core==2.28.1
+google-api-python-client==2.187.0
+google-auth==2.43.0
+google-auth-httplib2==0.2.1
+google-generativeai==0.8.5
+googleapis-common-protos==1.72.0
+grpcio==1.76.0
+grpcio-status==1.71.2
+h11==0.16.0
+hf-xet==1.2.0
+htmldate==1.9.4
+httplib2==0.31.0
+huggingface-hub==0.36.0
+idna==3.11
+Jinja2==3.1.6
+joblib==1.5.2
+jsonschema==4.25.1
+jsonschema-specifications==2025.9.1
+jusText==3.0.2
+lxml==6.0.2
+lxml_html_clean==0.4.3
+MarkupSafe==3.0.3
+mpmath==1.3.0
+narwhals==2.12.0
+networkx==3.6
+numpy==2.3.5
+orjson==3.11.4
+packaging==24.2
+pandas==2.3.3
+pillow==12.0.0
+pinecone==8.0.0
+pinecone-client==6.0.0
+pinecone-plugin-assistant==3.0.1
+pinecone-plugin-interface==0.0.7
+proto-plus==1.26.1
+protobuf==5.29.5
+pyarrow==21.0.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pydantic==2.12.4
+pydantic_core==2.41.5
+pydeck==0.9.1
+pyparsing==3.2.5
+python-dateutil==2.9.0.post0
+python-dotenv==1.2.1
+pytz==2025.2
+PyYAML==6.0.3
+referencing==0.37.0
+regex==2025.11.3
+requests==2.32.5
+rpds-py==0.29.0
+rsa==4.9.1
+safetensors==0.7.0
+scikit-learn==1.7.2
+scipy==1.16.3
+sentence-transformers==5.1.2
+setuptools==80.9.0
+six==1.17.0
+smmap==5.0.2
+sniffio==1.3.1
+soupsieve==2.8
+starlette==0.50.0
+streamlit==1.51.0
+sympy==1.14.0
+tenacity==9.1.2
+threadpoolctl==3.6.0
+tld==0.13.1
+tokenizers==0.22.1
+toml==0.10.2
+torch==2.9.1
+tornado==6.5.2
+tqdm==4.67.1
+trafilatura==2.0.0
+transformers==4.57.2
+typing-inspection==0.4.2
+typing_extensions==4.15.0
+tzdata==2025.2
+tzlocal==5.3.1
+uritemplate==4.2.0
+urllib3==2.5.0
+uvicorn==0.38.0