Upload 14 files

.env

@@ -0,0 +1,117 @@
+# ============================================================
+#  RAG System — Environment Configuration
+# ============================================================
+
+# --- API Credentials (for /webhook/search — used by n8n) ---
+API_USERNAME=admin
+API_PASSWORD=12345
+
+# --- Admin Dashboard Credentials (fallback if users.csv missing) ---
+FLASK_ADMIN_USERNAME=admin
+FLASK_ADMIN_PASSWORD=1234
+
+# ============================================================
+#  RAG — Storage & Sources
+# ============================================================
+
+# Absolute or relative path to the folder containing source documents
+SOURCES_DIR=sources
+
+# Absolute or relative path where the FAISS index will be stored
+RAG_STORAGE_DIR=faiss_storage
+
+# ============================================================
+#  RAG — Embedding Model
+# ============================================================
+
+RAG_EMBEDDING_MODEL=BAAI/bge-small-en
+
+# Set to True to use CUDA GPU for embeddings (requires CUDA)
+RAG_EMBEDDING_GPU=False
+
+# Load existing FAISS index from disk on startup (True/False)
+RAG_LOAD_INDEX=True
+
+# ============================================================
+#  RAG — Chunking
+# ============================================================
+
+RAG_CHUNK_SIZE=2000
+RAG_CHUNK_OVERLAP=150
+
+# ============================================================
+#  RAG — Retrieval
+# ============================================================
+
+# How many docs to fetch from FAISS before reranking
+RAG_INITIAL_FETCH_K=20
+
+# How many docs to return after reranking (final result count)
+RAG_RERANKER_K=5
+
+# Max files to process in a single incremental index update
+RAG_MAX_FILES_FOR_INCREMENTAL=50
+
+# ============================================================
+#  CSV Specific Parameters
+# ============================================================
+
+# Maximum number of results to return from CSV sources
+RAG_CSV_MAX_RESULTS=50
+
+# Threshold for CSV match confidence 
+# (If reranker is enabled, higher is better e.g., 0.0 to 1.0)
+# (If reranker is disabled/FAISS L2, lower is better e.g., < 1.0)
+RAG_CSV_CONFIDENCE_THRESHOLD=0.1
+
+# ============================================================
+#  RAG — Reranker Model
+# ============================================================
+
+RAG_RERANKER_MODEL=jinaai/jina-reranker-v2-base-multilingual
+
+# Set to False to disable reranking entirely
+RAG_RERANKER_ENABLED=False
+
+# ============================================================
+#  URL Source (General URL / Rentry)
+# ============================================================
+
+# Toggle fetching from external URL completely on/off
+URL_FETCH_ENABLED=False
+
+# General URL to scrape (replacing RENTRY_URL)
+EXTERNAL_URL="https://rentry.co/ada"
+
+# How often (in minutes) to auto-refresh from the URL (0 = disabled)
+URL_UPDATE_PERIOD_MINUTES=60
+
+# ============================================================
+#  Google Drive — Source Documents Folder
+# ============================================================
+
+# Set to True to download source docs from a GDrive folder on startup
+GDRIVE_SOURCES_ENABLED=False
+GDRIVE_FOLDER_URL=
+
+# ============================================================
+#  Google Drive — Pre-built FAISS Index (ZIP)
+# ============================================================
+
+# Set to True to download a pre-built FAISS index ZIP from GDrive on startup
+GDRIVE_INDEX_ENABLED=False
+GDRIVE_INDEX_URL=
+
+# ============================================================
+#  Google Drive — Users CSV
+# ============================================================
+
+# Set to True to download users.csv from GDrive on startup
+GDRIVE_USERS_CSV_ENABLED=False
+GDRIVE_USERS_CSV_URL=
+
+# ============================================================
+#  Logging
+# ============================================================
+
+RAG_DETAILED_LOGGING=True

Dockerfile

@@ -0,0 +1,51 @@
+# Use Python 3.11 to support Pandas 3.x and newer libraries
+FROM python:3.11-slim
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Install system dependencies
+# libgl1 and libglib2.0-0 are often needed for CV/PDF libraries
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libgl1 \
+    libglib2.0-0 \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy the requirements file
+COPY requirements.txt requirements.txt
+
+# Install Python packages with timeout increase
+RUN pip install --no-cache-dir --timeout=1000 -r requirements.txt
+
+# Copy application code
+COPY . /app
+
+# Create a non-root user (Security Best Practice)
+RUN useradd -m -u 1000 user
+
+# Change ownership of the app directory and the temp directories
+RUN chown -R user:user /app
+
+# Create temp directories for HuggingFace/Torch cache and set permissions
+RUN mkdir -p /tmp/transformers_cache /tmp/hf_home /tmp/torch_home && \
+    chown -R user:user /tmp/transformers_cache /tmp/hf_home /tmp/torch_home
+
+# Switch to the non-root user
+USER user
+
+# Expose the port (Standard for HF Spaces)
+EXPOSE 7860
+
+# Set environment variables
+ENV FLASK_HOST=0.0.0.0
+ENV FLASK_PORT=7860
+ENV FLASK_DEBUG=False
+
+# CRITICAL: Set HF-specific env vars to writable directories
+ENV TRANSFORMERS_CACHE=/tmp/transformers_cache
+ENV HF_HOME=/tmp/hf_home
+ENV TORCH_HOME=/tmp/torch_home
+
+# Command to run the app
+CMD ["python", "app.py"]

app.py

@@ -0,0 +1,339 @@
+from flask import Flask, request, jsonify, Response, render_template
+from flask_cors import CORS
+import os
+import logging
+import functools
+import pandas as pd
+import threading
+import time
+import tempfile
+import shutil
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+# Custom Imports
+from rag_system import initialize_and_get_rag_system
+from config import (
+    API_USERNAME, API_PASSWORD, RAG_SOURCES_DIR, RAG_STORAGE_PARENT_DIR,
+    GDRIVE_INDEX_ENABLED, GDRIVE_INDEX_ID_OR_URL,
+    GDRIVE_USERS_CSV_ENABLED, GDRIVE_USERS_CSV_ID_OR_URL,
+    ADMIN_USERNAME, ADMIN_PASSWORD, RAG_RERANKER_K,
+    EXTERNAL_URL, URL_UPDATE_PERIOD_MINUTES, URL_FETCH_ENABLED,
+    RAG_CSV_MAX_RESULTS, RAG_CSV_CONFIDENCE_THRESHOLD
+)
+from utils import download_and_unzip_gdrive_file, download_gdrive_file, fetch_and_clean_url
+
+# Logging Setup
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Flask Init
+app = Flask(__name__, static_folder='static', template_folder='templates')
+CORS(app)
+
+# Global State
+rag_system = None
+user_df = None
+_APP_BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# --- Helper: Load Users ---
+def load_users_from_csv():
+    global user_df
+    assets_folder = os.path.join(_APP_BASE_DIR, 'assets')
+    os.makedirs(assets_folder, exist_ok=True)
+    users_csv_path = os.path.join(assets_folder, 'users.csv')
+    
+    try:
+        if os.path.exists(users_csv_path):
+            user_df = pd.read_csv(users_csv_path)
+            # Normalize email
+            if 'email' in user_df.columns:
+                user_df['email'] = user_df['email'].str.lower().str.strip()
+            logger.info(f"Loaded {len(user_df)} users from CSV.")
+        else:
+            logger.warning("users.csv not found in assets folder.")
+            user_df = None
+    except Exception as e:
+        logger.error(f"Failed to load users.csv: {e}")
+        user_df = None
+
+# --- Helper: Auth Decorators ---
+def require_api_auth(f):
+    """Protects the N8N Webhook endpoint"""
+    @functools.wraps(f)
+    def decorated(*args, **kwargs):
+        auth = request.authorization
+        if not auth or auth.username != API_USERNAME or auth.password != API_PASSWORD:
+            return Response('Unauthorized', 401, {'WWW-Authenticate': 'Basic realm="API Login Required"'})
+        return f(*args, **kwargs)
+    return decorated
+
+def require_admin_auth(f):
+    """Protects Admin Rebuild/Update endpoints"""
+    @functools.wraps(f)
+    def decorated(*args, **kwargs):
+        auth = request.authorization
+        if not auth:
+            return jsonify({"error": "Unauthorized"}), 401
+
+        if user_df is not None:
+            user_email = auth.username.lower().strip()
+            user_record = user_df[user_df['email'] == user_email]
+            if not user_record.empty:
+                user_data = user_record.iloc[0]
+                if str(user_data['password']) == auth.password and user_data['role'] == 'admin':
+                    return f(*args, **kwargs)
+
+        if auth.username == ADMIN_USERNAME and auth.password == ADMIN_PASSWORD:
+            return f(*args, **kwargs)
+
+        return jsonify({"error": "Unauthorized"}), 401
+    return decorated
+
+# --- URL Zero-Downtime Updater ---
+def trigger_url_update():
+    global rag_system
+    if not URL_FETCH_ENABLED or not EXTERNAL_URL: 
+        return {"error": "External URL fetching is disabled or not configured"}
+    
+    logger.info(f"[URL_UPDATE] Starting zero-downtime fetch from {EXTERNAL_URL}")
+    
+    # 1. Create temporary staging folders
+    temp_staging_sources = tempfile.mkdtemp(prefix="rag_sources_temp_")
+    temp_index = tempfile.mkdtemp(prefix="rag_index_temp_")
+    
+    try:
+        # 2. COMBINE SOURCES: Copy existing GDrive/Local sources to staging first
+        if os.path.exists(RAG_SOURCES_DIR):
+            shutil.copytree(RAG_SOURCES_DIR, temp_staging_sources, dirs_exist_ok=True)
+            
+        # 3. Fetch URL data — saved to <app_root>/tmp/ for persistence & inspection
+        tmp_dir = os.path.join(_APP_BASE_DIR, 'tmp')
+        os.makedirs(tmp_dir, exist_ok=True)
+        url_out_path = os.path.join(tmp_dir, "url_data.txt")
+        success = fetch_and_clean_url(EXTERNAL_URL, url_out_path)
+        
+        if not success:
+            return {"error": "Failed to fetch or parse the URL."}
+
+        # Copy from tmp/ into staging so it gets indexed alongside other sources
+        shutil.copy2(url_out_path, os.path.join(temp_staging_sources, "url_data.txt"))
+        
+        # 4. Build a brand new RAG instance isolated in the temp directories
+        new_rag = initialize_and_get_rag_system(
+            force_rebuild=True, 
+            source_dir_override=temp_staging_sources,
+            storage_dir_override=temp_index
+        )
+        
+        if new_rag is None:
+            raise Exception("Failed to build new RAG index from parsed text.")
+            
+        # 5. Atomic Swap (Now incoming requests hit the new DB immediately)
+        rag_system = new_rag
+        
+        # 6. Backup/Replace persistent INDEX directory ONLY
+        os.makedirs(RAG_STORAGE_PARENT_DIR, exist_ok=True)
+        shutil.copytree(temp_index, RAG_STORAGE_PARENT_DIR, dirs_exist_ok=True)
+        
+        rag_system.index_storage_dir = RAG_STORAGE_PARENT_DIR
+        
+        logger.info("[URL_UPDATE] Success! RAG database updated combining Local, GDrive, and URL sources.")
+        return {"status": "success", "message": "Database successfully updated using combined sources."}
+        
+    except Exception as e:
+        logger.error(f"[URL_UPDATE] Error during update: {e}", exc_info=True)
+        return {"error": str(e)}
+        
+    finally:
+        shutil.rmtree(temp_staging_sources, ignore_errors=True)
+        shutil.rmtree(temp_index, ignore_errors=True)
+
+def url_periodic_loop():
+    if not URL_FETCH_ENABLED or not EXTERNAL_URL or URL_UPDATE_PERIOD_MINUTES <= 0:
+        logger.info("Periodic URL updates disabled.")
+        return
+        
+    logger.info(f"[URL_UPDATE] Background thread started for: {EXTERNAL_URL}")
+    trigger_url_update()
+    
+    while True:
+        time.sleep(URL_UPDATE_PERIOD_MINUTES * 60)
+        logger.info(f"[URL_UPDATE] Triggering scheduled periodic update...")
+        trigger_url_update()
+
+# --- Startup Logic ---
+def run_startup_tasks():
+    global rag_system
+    logger.info("--- Executing Startup Tasks ---")
+    
+    if GDRIVE_USERS_CSV_ENABLED and GDRIVE_USERS_CSV_ID_OR_URL:
+        target = os.path.join(_APP_BASE_DIR, 'assets', 'users.csv')
+        download_gdrive_file(GDRIVE_USERS_CSV_ID_OR_URL, target)
+    
+    load_users_from_csv()
+
+    if GDRIVE_INDEX_ENABLED and GDRIVE_INDEX_ID_OR_URL:
+        download_and_unzip_gdrive_file(GDRIVE_INDEX_ID_OR_URL, os.getcwd())
+    
+    rag_system = initialize_and_get_rag_system()
+    
+    if URL_FETCH_ENABLED and EXTERNAL_URL:
+        threading.Thread(target=url_periodic_loop, daemon=True).start()
+        
+    logger.info("--- Startup Tasks Complete ---")
+
+with app.app_context():
+    run_startup_tasks()
+
+# ===========================
+#       API ROUTES
+# ===========================
+
+@app.route('/webhook/search', methods=['POST'])
+@require_api_auth
+def search_knowledgebase_api():
+    if not rag_system:
+        return jsonify({"error": "RAG not initialized. Check server logs."}), 503
+
+    data = request.json or {}
+    query = data.get('query')
+    if not query:
+        return jsonify({"error": "Query field is required"}), 400
+
+    top_k = data.get('final_k', RAG_RERANKER_K)
+    use_reranker = data.get('use_reranker', True)
+    
+    # 1. NEW: Extract the 'cleaned' parameter (defaults to False)
+    cleaned = data.get('cleaned', False)
+
+    if rag_system.retriever:
+        if not use_reranker:
+            rag_system.retriever.reranker = None
+        elif use_reranker and rag_system.reranker:
+             rag_system.retriever.reranker = rag_system.reranker
+
+    try:
+        raw_results = rag_system.search_knowledge_base(query, top_k=top_k)
+        
+        # Apply CSV limitations and thresholds
+        final_results = []
+        csv_count = 0
+        
+        for res in raw_results:
+            is_csv = res["metadata"].get("source_type") == "csv" or res["metadata"].get("source_document_name", "").endswith(".csv")
+            
+            if is_csv:
+                score = res["score"]
+                passed_threshold = False
+                
+                # Check confidence limit depending on method used (reranker: higher is better | FAISS L2: lower is better)
+                if rag_system.reranker:
+                    confidence = score
+                else:
+                    # Convert FAISS L2 Distance into a 0-1 Confidence Score
+                    confidence = 1 / (1 + score)
+                    res["score"] = confidence # Update the result so the API shows the neat confidence score
+
+                passed_threshold = confidence >= RAG_CSV_CONFIDENCE_THRESHOLD
+                    
+                if passed_threshold and csv_count < RAG_CSV_MAX_RESULTS:
+                    final_results.append(res)
+                    csv_count += 1
+            else:
+                final_results.append(res)
+
+        # 2. NEW: If cleaned is True, strip out 'metadata' and 'score'
+        if cleaned:
+            final_results = [{"content": r["content"]} for r in final_results]
+
+        return jsonify({"results": final_results, "count": len(final_results), "status": "success"})
+    except Exception as e:
+        logger.error(f"Search API Error: {e}")
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/user-login', methods=['POST'])
+def user_login():
+    if user_df is None:
+        return jsonify({"error": "User database not available."}), 503
+        
+    data = request.json
+    email = data.get('email', '').lower().strip()
+    password = data.get('password')
+    
+    if not email or not password:
+        return jsonify({"error": "Email and password required"}), 400
+
+    user_record = user_df[user_df['email'] == email]
+    if not user_record.empty:
+        u_data = user_record.iloc[0]
+        if str(u_data['password']) == str(password):
+            resp = u_data.to_dict()
+            if 'password' in resp: 
+                del resp['password']
+            return jsonify(resp), 200
+            
+    return jsonify({"error": "Invalid credentials"}), 401
+
+@app.route('/')
+def index_route():
+    return render_template('chat-bot.html')
+
+@app.route('/admin/login', methods=['POST'])
+@require_admin_auth
+def admin_login():
+    return jsonify({"status": "success", "message": "Authenticated"}), 200
+
+@app.route('/admin/update_faiss_index', methods=['POST'])
+@require_admin_auth
+def update_faiss_index():
+    if not rag_system: 
+        return jsonify({"error": "RAG system not initialized"}), 503
+    
+    data = request.json or {}
+    max_files = data.get('max_new_files')
+    
+    try:
+        result = rag_system.update_index_with_new_files(RAG_SOURCES_DIR, max_files)
+        return jsonify(result), 200
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/admin/rebuild_index', methods=['POST'])
+@require_admin_auth
+def rebuild_index():
+    global rag_system
+    try:
+        if URL_FETCH_ENABLED and EXTERNAL_URL:
+            result = trigger_url_update()
+            if "error" in result:
+                return jsonify(result), 500
+            return jsonify({"status": "Index rebuilt successfully using combined local & URL sources"}), 200
+        else:
+            rag_system = initialize_and_get_rag_system(force_rebuild=True)
+            return jsonify({"status": "Index rebuilt successfully"}), 200
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+# Retained specific endpoint name to ensure the frontend doesn't break
+@app.route('/admin/fetch_rentry', methods=['POST'])
+@require_admin_auth
+def api_fetch_url():
+    result = trigger_url_update()
+    if "error" in result:
+        return jsonify(result), 500
+    return jsonify(result), 200
+
+@app.route('/status', methods=['GET'])
+def status_route():
+    return jsonify({
+        "status": "online",
+        "rag_initialized": rag_system is not None,
+        "users_loaded": user_df is not None
+    })
+
+if __name__ == '__main__':
+    port = int(os.environ.get("PORT", 7860)) 
+    app.run(host='0.0.0.0', port=port)

assets/users.csv

@@ -0,0 +1,12 @@
+sl,name,email,password,role
+1,Sifat Hossain Fahim,fahim@ge-bd.com,WorldTour1234!,admin
+2,Sakib Ahmed,sakib.ahmed@ge-bd.com,12345678!,admin
+3,Rezwanul Islam,rezwanul@ge-bd.com,marstour1234!,admin
+4,Sarwar Jahan,sarwar.piel@ge-bd.com,password123,user
+5,Rezaul Kabir,rezaul.kabir@ge-bd.com,securepass,user
+6,Test,test@test.com,12345678!,user
+7,Sadiquzzaman,sadiquzzaman@ge-bd.com,wqeqw1234,user
+8,Sadman,sadman@ge-bd.com,1234fvb,user
+9,Pavel,pavel@ge-bd.com,12314rdf,user
+10,Sajib,sajib.hossain@ge-bd.com,1234rge,user
+11,Abdur Rahim,arahim@ge-bd.com,23ree4rt,user

chunker.py

@@ -0,0 +1,135 @@
+import os
+import logging
+import json
+import argparse
+import csv
+from typing import List, Dict, Optional
+
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+from utils import extract_text_from_file, FAISS_RAG_SUPPORTED_EXTENSIONS
+
+# --- Logging Setup ---
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+def process_sources_and_create_chunks(
+    sources_dir: str,
+    output_file: str,
+    chunk_size: int = 1000,
+    chunk_overlap: int = 150,
+    text_output_dir: Optional[str] = None
+) -> None:
+    if not os.path.isdir(sources_dir):
+        logger.error(f"Source directory not found: '{sources_dir}'")
+        raise FileNotFoundError(f"Source directory not found: '{sources_dir}'")
+
+    logger.info(f"Starting chunking process. Sources: '{sources_dir}', Output: '{output_file}'")
+    
+    if text_output_dir:
+        os.makedirs(text_output_dir, exist_ok=True)
+        logger.info(f"Will save raw extracted text to: '{text_output_dir}'")
+
+    all_chunks_for_json: List[Dict] = []
+    processed_files_count = 0
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+
+    for filename in os.listdir(sources_dir):
+        file_path = os.path.join(sources_dir, filename)
+        if not os.path.isfile(file_path):
+            continue
+
+        file_ext = filename.split('.')[-1].lower()
+        if file_ext not in FAISS_RAG_SUPPORTED_EXTENSIONS:
+            logger.debug(f"Skipping unsupported file: {filename}")
+            continue
+
+        logger.info(f"Processing source file: {filename}")
+        
+        # CSV Handling natively row by row
+        if file_ext == 'csv':
+            try:
+                with open(file_path, mode='r', encoding='utf-8-sig') as f:
+                    reader = csv.DictReader(f)
+                    for i, row in enumerate(reader):
+                        row_text = "\n".join([f"{k}: {v}" for k, v in row.items() if k and v and str(v).strip()])
+                        chunk_data = {
+                            "page_content": row_text,
+                            "metadata": {
+                                "source_document_name": filename,
+                                "chunk_index": i,
+                                "full_location": f"{filename}, Row {i+1}",
+                                "source_type": "csv"
+                            }
+                        }
+                        all_chunks_for_json.append(chunk_data)
+                processed_files_count += 1
+            except Exception as e:
+                logger.error(f"Error processing CSV {filename}: {e}")
+        else:
+            text_content = FAISS_RAG_SUPPORTED_EXTENSIONS[file_ext](file_path)
+
+            if text_content and text_content != "CSV_HANDLED_NATIVELY":
+                if text_output_dir:
+                    try:
+                        text_output_path = os.path.join(text_output_dir, f"{filename}.txt")
+                        with open(text_output_path, 'w', encoding='utf-8') as f_text:
+                            f_text.write(text_content)
+                    except Exception as e_text_save:
+                        logger.error(f"Could not save extracted text for '{filename}': {e_text_save}")
+
+                chunks = text_splitter.split_text(text_content)
+                for i, chunk_text in enumerate(chunks):
+                    chunk_data = {
+                        "page_content": chunk_text,
+                        "metadata": {
+                            "source_document_name": filename,
+                            "chunk_index": i,
+                            "full_location": f"{filename}, Chunk {i+1}"
+                        }
+                    }
+                    all_chunks_for_json.append(chunk_data)
+
+                processed_files_count += 1
+
+    if not all_chunks_for_json:
+        logger.warning(f"No processable documents found in '{sources_dir}'.")
+
+    output_dir = os.path.dirname(output_file)
+    os.makedirs(output_dir, exist_ok=True)
+
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(all_chunks_for_json, f, indent=2)
+
+    logger.info(f"Chunking complete. Processed {processed_files_count} files. Total chunks: {len(all_chunks_for_json)}")
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--sources-dir', type=str, required=True)
+    parser.add_argument('--output-file', type=str, required=True)
+    parser.add_argument('--text-output-dir', type=str, default=None)
+    parser.add_argument('--chunk-size', type=int, default=1000)
+    parser.add_argument('--chunk-overlap', type=int, default=150)
+    args = parser.parse_args()
+
+    try:
+        process_sources_and_create_chunks(
+            sources_dir=args.sources_dir,
+            output_file=args.output_file,
+            chunk_size=args.chunk_size,
+            chunk_overlap=args.chunk_overlap,
+            text_output_dir=args.text_output_dir
+        )
+    except Exception as e:
+        logger.critical(f"Chunking failed: {e}", exc_info=True)
+        exit(1)
+
+if __name__ == "__main__":
+    main()

config.py

@@ -0,0 +1,83 @@
+import os
+import logging
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# --- Logging Setup ---
+logger = logging.getLogger(__name__)
+if not logger.handlers:
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    )
+
+_MODULE_BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# API Authentication for n8n (Basic Auth)
+API_USERNAME = os.getenv("API_USERNAME", "admin")
+API_PASSWORD = os.getenv("API_PASSWORD", "password")
+
+# Admin fallback credentials for dashboard (used when users.csv is missing or unavailable)
+ADMIN_USERNAME = os.getenv('FLASK_ADMIN_USERNAME', 'admin')
+ADMIN_PASSWORD = os.getenv('FLASK_ADMIN_PASSWORD', '1234')
+
+# URL Fetcher configs (Generalized from Rentry)
+URL_FETCH_ENABLED = os.getenv("URL_FETCH_ENABLED", "False").lower() == "true"
+EXTERNAL_URL = os.getenv("EXTERNAL_URL", os.getenv("RENTRY_URL", ""))
+URL_UPDATE_PERIOD_MINUTES = int(os.getenv("URL_UPDATE_PERIOD_MINUTES", os.getenv("RENTRY_UPDATE_PERIOD_MINUTES", "60")))
+
+# CSV Configuration
+RAG_CSV_MAX_RESULTS = int(os.getenv("RAG_CSV_MAX_RESULTS", "5"))
+RAG_CSV_CONFIDENCE_THRESHOLD = float(os.getenv("RAG_CSV_CONFIDENCE_THRESHOLD", "0.5"))
+
+RAG_FAISS_INDEX_SUBDIR_NAME = "faiss_index"
+RAG_STORAGE_PARENT_DIR = os.getenv("RAG_STORAGE_DIR", os.path.join(_MODULE_BASE_DIR, "faiss_storage"))
+RAG_SOURCES_DIR = os.getenv("SOURCES_DIR", os.path.join(_MODULE_BASE_DIR, "sources"))
+RAG_CHUNKED_SOURCES_FILENAME = "pre_chunked_sources.json"
+
+os.makedirs(RAG_SOURCES_DIR, exist_ok=True)
+os.makedirs(RAG_STORAGE_PARENT_DIR, exist_ok=True)
+
+# Embedding and model configuration
+RAG_EMBEDDING_MODEL_NAME = os.getenv("RAG_EMBEDDING_MODEL", "BAAI/bge-small-en")
+RAG_EMBEDDING_USE_GPU = os.getenv("RAG_EMBEDDING_GPU", "False").lower() == "true"
+RAG_LOAD_INDEX_ON_STARTUP = os.getenv("RAG_LOAD_INDEX", "True").lower() == "true"
+
+# Retrieval Settings
+RAG_INITIAL_FETCH_K = int(os.getenv("RAG_INITIAL_FETCH_K", 20))
+RAG_RERANKER_K = int(os.getenv("RAG_RERANKER_K", 5))
+RAG_MAX_FILES_FOR_INCREMENTAL = int(os.getenv("RAG_MAX_FILES_FOR_INCREMENTAL", "50"))
+
+# Chunk configuration
+RAG_CHUNK_SIZE = int(os.getenv("RAG_CHUNK_SIZE", 2000))
+RAG_CHUNK_OVERLAP = int(os.getenv("RAG_CHUNK_OVERLAP", 150))
+
+# Reranker configuration
+RAG_RERANKER_MODEL_NAME = os.getenv("RAG_RERANKER_MODEL", "jinaai/jina-reranker-v2-base-multilingual")
+RAG_RERANKER_ENABLED = os.getenv("RAG_RERANKER_ENABLED", "True").lower() == "true"
+
+# GDrive configuration for RAG sources
+GDRIVE_SOURCES_ENABLED = os.getenv("GDRIVE_SOURCES_ENABLED", "False").lower() == "true"
+GDRIVE_FOLDER_ID_OR_URL = os.getenv("GDRIVE_FOLDER_URL")
+
+# GDrive configuration for downloading a pre-built FAISS index
+GDRIVE_INDEX_ENABLED = os.getenv("GDRIVE_INDEX_ENABLED", "False").lower() == "true"
+GDRIVE_INDEX_ID_OR_URL = os.getenv("GDRIVE_INDEX_URL")
+
+# GDrive configuration for downloading users.csv
+GDRIVE_USERS_CSV_ENABLED = os.getenv("GDRIVE_USERS_CSV_ENABLED", "False").lower() == "true"
+GDRIVE_USERS_CSV_ID_OR_URL = os.getenv("GDRIVE_USERS_CSV_URL")
+
+RAG_DETAILED_LOGGING = os.getenv("RAG_DETAILED_LOGGING", "True").lower() == "true"
+
+logger.info(f"RAG Config Loaded - Chunk Size: {RAG_CHUNK_SIZE}, Chunk Overlap: {RAG_CHUNK_OVERLAP}")
+logger.info(f"Embedding Model: {RAG_EMBEDDING_MODEL_NAME}")
+logger.info(f"Reranker Model: {RAG_RERANKER_MODEL_NAME}")
+logger.info(f"Retrieval Pipeline: Initial Fetch K={RAG_INITIAL_FETCH_K}, Reranker Final K={RAG_RERANKER_K}")
+logger.info(f"CSV Filters: Max Results={RAG_CSV_MAX_RESULTS}, Threshold={RAG_CSV_CONFIDENCE_THRESHOLD}")
+logger.info(f"URL Fetching: {'ENABLED' if URL_FETCH_ENABLED else 'DISABLED'}")
+logger.info(f"Detailed Logging: {'ENABLED' if RAG_DETAILED_LOGGING else 'DISABLED'}")
+logger.info(f"GDrive Sources Download: {'ENABLED' if GDRIVE_SOURCES_ENABLED else 'DISABLED'}")
+logger.info(f"GDrive Pre-built Index Download: {'ENABLED' if GDRIVE_INDEX_ENABLED else 'DISABLED'}")
+logger.info(f"GDrive users.csv Download: {'ENABLED' if GDRIVE_USERS_CSV_ENABLED else 'DISABLED'}")

note.txt

@@ -0,0 +1,7 @@
+https://aibot8800-rag-brain.hf.space
+
+
+
+pip install "huggingface-hub>=0.34.0,<1.0"
+pip install -U transformers sentence-transformers
+pip install langchain-huggingface

postman.json

@@ -0,0 +1,153 @@
+{
+	"info": {
+		"_postman_id": "b8f9e9a1-5c8e-4a8e-9b8e-1f8e9a1f8e9a",
+		"name": "edmond_cad_refund",
+		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
+	},
+	"item": [
+		{
+			"name": "N8N - Search Knowledgebase",
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "password",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "admin",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"query\": \"how to get a refund for electronics?\",\n    \"use_reranker\": true,\n    \"final_k\": 5,\n    \"persona\": [\"standard\"],\n    \"tier\": [\"gold\"]\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "{{base_url}}/webhook/search",
+					"host": [
+						"{{base_url}}"
+					],
+					"path": [
+						"webhook",
+						"search"
+					]
+				},
+				"description": "Main endpoint used by N8N to retrieve context chunks."
+			},
+			"response": []
+		},
+		{
+			"name": "Admin - Rebuild Index",
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "1234",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "admin",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"url": {
+					"raw": "{{base_url}}/admin/rebuild_index",
+					"host": [
+						"{{base_url}}"
+					],
+					"path": [
+						"admin",
+						"rebuild_index"
+					]
+				},
+				"description": "Completely deletes and rebuilds the FAISS index from sources."
+			},
+			"response": []
+		},
+		{
+			"name": "Admin - Update Index (Incremental)",
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "1234",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "admin",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"max_new_files\": 50\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "{{base_url}}/admin/update_faiss_index",
+					"host": [
+						"{{base_url}}"
+					],
+					"path": [
+						"admin",
+						"update_faiss_index"
+					]
+				},
+				"description": "Adds only new files to the existing index."
+			},
+			"response": []
+		},
+		{
+			"name": "Public - Status",
+			"request": {
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "{{base_url}}/status",
+					"host": [
+						"{{base_url}}"
+					],
+					"path": [
+						"status"
+					]
+				}
+			},
+			"response": []
+		}
+	],
+	"variable": [
+		{
+			"key": "base_url",
+			"value": "http://localhost:5000",
+			"type": "string"
+		}
+	]
+}

rag_components.py

@@ -0,0 +1,357 @@
+import os
+import logging
+import json
+import time
+import csv
+from typing import List, Dict, Optional, Any
+
+import torch
+from sentence_transformers import CrossEncoder
+
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+
+from langchain_core.documents import Document
+from langchain_core.retrievers import BaseRetriever
+from langchain_core.callbacks import CallbackManagerForRetrieverRun
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+from config import (
+    RAG_RERANKER_MODEL_NAME, RAG_DETAILED_LOGGING,
+    RAG_CHUNK_SIZE, RAG_CHUNK_OVERLAP, RAG_CHUNKED_SOURCES_FILENAME,
+    RAG_FAISS_INDEX_SUBDIR_NAME, RAG_INITIAL_FETCH_K, RAG_RERANKER_K,
+    RAG_MAX_FILES_FOR_INCREMENTAL
+)
+from utils import FAISS_RAG_SUPPORTED_EXTENSIONS
+
+logger = logging.getLogger(__name__)
+
+
+class DocumentReranker:
+    def __init__(self, model_name: str = RAG_RERANKER_MODEL_NAME):
+        self.logger = logging.getLogger(__name__ + ".DocumentReranker")
+        self.model_name = model_name
+        self.model = None
+        
+        try:
+            self.logger.info(f"[RERANKER_INIT] Loading reranker model: {self.model_name}")
+            start_time = time.time()
+            self.model = CrossEncoder(model_name, trust_remote_code=True)
+            load_time = time.time() - start_time
+            self.logger.info(f"[RERANKER_INIT] Reranker model '{self.model_name}' loaded successfully in {load_time:.2f}s")
+        except Exception as e:
+            self.logger.error(f"[RERANKER_INIT] Failed to load reranker model '{self.model_name}': {e}", exc_info=True)
+            raise RuntimeError(f"Could not initialize reranker model: {e}") from e
+
+    def rerank_documents(self, query: str, documents: List[Document], top_k: int) -> List[Document]:
+        if not documents or not self.model:
+            return documents[:top_k] if documents else []
+
+        try:
+            start_time = time.time()
+            doc_pairs = [[query, doc.page_content] for doc in documents]
+            scores = self.model.predict(doc_pairs)
+            rerank_time = time.time() - start_time
+            self.logger.info(f"[RERANKER] Computed relevance scores in {rerank_time:.3f}s")
+            
+            doc_score_pairs = list(zip(documents, scores))
+            doc_score_pairs.sort(key=lambda x: x[1], reverse=True)
+            
+            reranked_docs = []
+            for doc, score in doc_score_pairs[:top_k]:
+                doc.metadata["reranker_score"] = float(score)
+                reranked_docs.append(doc)
+            
+            return reranked_docs
+        except Exception as e:
+            self.logger.error(f"[RERANKER] Error during reranking: {e}", exc_info=True)
+            return documents[:top_k] if documents else []
+
+
+class FAISSRetrieverWithScore(BaseRetriever):
+    vectorstore: FAISS
+    reranker: Optional[DocumentReranker] = None
+    initial_fetch_k: int = RAG_INITIAL_FETCH_K
+    final_k: int = RAG_RERANKER_K
+
+    def _get_relevant_documents(
+        self, query: str, *, run_manager: CallbackManagerForRetrieverRun
+    ) -> List[Document]:
+        
+        start_time = time.time()
+        num_to_fetch = self.initial_fetch_k if self.reranker else self.final_k
+        
+        logger.info(f"[RETRIEVER] Fetching {num_to_fetch} docs (Rerank={self.reranker is not None})")
+
+        docs_and_scores = self.vectorstore.similarity_search_with_score(query, k=num_to_fetch)
+        
+        relevant_docs = []
+        for doc, score in docs_and_scores:
+            doc.metadata["retrieval_score"] = float(score)
+            relevant_docs.append(doc)
+        
+        if self.reranker and relevant_docs:
+            relevant_docs = self.reranker.rerank_documents(query, relevant_docs, top_k=self.final_k)
+        
+        total_time = time.time() - start_time
+        logger.info(f"[RETRIEVER] Completed in {total_time:.3f}s. Returned {len(relevant_docs)} docs.")
+        return relevant_docs
+
+
+class KnowledgeRAG:
+    def __init__(
+        self,
+        index_storage_dir: str,
+        embedding_model_name: str,
+        use_gpu_for_embeddings: bool,
+        chunk_size: int = RAG_CHUNK_SIZE,
+        chunk_overlap: int = RAG_CHUNK_OVERLAP,
+        reranker_model_name: Optional[str] = None,
+        enable_reranker: bool = True,
+    ):
+        self.logger = logging.getLogger(__name__ + ".KnowledgeRAG")
+        self.logger.info(f"[RAG_INIT] Initializing KnowledgeRAG system")
+        
+        self.index_storage_dir = index_storage_dir
+        os.makedirs(self.index_storage_dir, exist_ok=True)
+
+        self.embedding_model_name = embedding_model_name
+        self.use_gpu_for_embeddings = use_gpu_for_embeddings
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.reranker_model_name = reranker_model_name or RAG_RERANKER_MODEL_NAME
+        self.enable_reranker = enable_reranker
+        self.reranker = None
+
+        device = "cpu"
+        if self.use_gpu_for_embeddings:
+            if torch.cuda.is_available():
+                self.logger.info(f"[RAG_INIT] CUDA available. Requesting GPU.")
+                device = "cuda"
+            else:
+                self.logger.warning("[RAG_INIT] CUDA not available. Fallback to CPU.")
+        
+        self.embeddings = HuggingFaceEmbeddings(
+            model_name=self.embedding_model_name,
+            model_kwargs={"device": device},
+            encode_kwargs={"normalize_embeddings": True}
+        )
+
+        if self.enable_reranker:
+            try:
+                self.reranker = DocumentReranker(self.reranker_model_name)
+            except Exception as e:
+                self.logger.warning(f"[RAG_INIT] Reranker Init Failed: {e}")
+                self.reranker = None
+
+        self.vector_store: Optional[FAISS] = None
+        self.retriever: Optional[FAISSRetrieverWithScore] = None
+        self.processed_source_files: List[str] = []
+
+    def _save_chunk_config(self):
+        faiss_path = os.path.join(self.index_storage_dir, RAG_FAISS_INDEX_SUBDIR_NAME)
+        config_file = os.path.join(faiss_path, "chunk_config.json")
+        with open(config_file, 'w') as f:
+            json.dump({"chunk_size": self.chunk_size, "chunk_overlap": self.chunk_overlap}, f)
+
+    def _load_chunk_config(self) -> Optional[dict]:
+        faiss_path = os.path.join(self.index_storage_dir, RAG_FAISS_INDEX_SUBDIR_NAME)
+        config_file = os.path.join(faiss_path, "chunk_config.json")
+        if os.path.exists(config_file):
+            with open(config_file, 'r') as f:
+                return json.load(f)
+        return None
+
+    def chunk_config_has_changed(self) -> bool:
+        saved = self._load_chunk_config()
+        if saved is None:
+            return False  
+        changed = saved.get("chunk_size") != self.chunk_size or saved.get("chunk_overlap") != self.chunk_overlap
+        if changed:
+            self.logger.warning(
+                f"[CONFIG_CHANGE] Chunk config mismatch! "
+                f"Saved=(size={saved.get('chunk_size')}, overlap={saved.get('chunk_overlap')}) "
+                f"Current=(size={self.chunk_size}, overlap={self.chunk_overlap}). "
+                f"Index will be rebuilt."
+            )
+        return changed
+
+    def build_index_from_source_files(self, source_folder_path: str):
+        self.logger.info(f"[INDEX_BUILD] Building from: {source_folder_path}")
+        if not os.path.isdir(source_folder_path):
+            raise FileNotFoundError(f"Source folder not found: '{source_folder_path}'.")
+
+        all_docs = []
+        processed_files = []
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap)
+
+        pre_chunked_path = os.path.join(self.index_storage_dir, RAG_CHUNKED_SOURCES_FILENAME)
+        if os.path.exists(pre_chunked_path):
+            try:
+                with open(pre_chunked_path, 'r', encoding='utf-8') as f:
+                    chunk_data_list = json.load(f)
+                for chunk in chunk_data_list:
+                    doc = Document(page_content=chunk.get("page_content", ""), metadata=chunk.get("metadata", {}))
+                    all_docs.append(doc)
+                    if 'source_document_name' in doc.metadata:
+                        processed_files.append(doc.metadata['source_document_name'])
+                processed_files = sorted(list(set(processed_files)))
+            except Exception as e:
+                self.logger.error(f"[INDEX_BUILD] JSON load failed: {e}")
+
+        if not all_docs:
+            for filename in os.listdir(source_folder_path):
+                file_path = os.path.join(source_folder_path, filename)
+                if not os.path.isfile(file_path): continue
+                file_ext = filename.split('.')[-1].lower()
+                
+                if file_ext in FAISS_RAG_SUPPORTED_EXTENSIONS:
+                    # Specific handler for CSV formatting
+                    if file_ext == 'csv':
+                        try:
+                            with open(file_path, mode='r', encoding='utf-8-sig') as f:
+                                reader = csv.DictReader(f)
+                                for i, row in enumerate(reader):
+                                    row_text = "\n".join([f"{k}: {v}" for k, v in row.items() if k and v and str(v).strip()])
+                                    meta = {"source_document_name": filename, "chunk_index": i, "source_type": "csv"}
+                                    all_docs.append(Document(page_content=row_text, metadata=meta))
+                            processed_files.append(filename)
+                        except Exception as e:
+                            self.logger.error(f"[INDEX_BUILD] Error processing CSV {filename}: {e}")
+                    else:
+                        text_content = FAISS_RAG_SUPPORTED_EXTENSIONS[file_ext](file_path)
+                        if text_content and text_content != "CSV_HANDLED_NATIVELY":
+                            chunks = text_splitter.split_text(text_content)
+                            for i, chunk_text in enumerate(chunks):
+                                meta = {"source_document_name": filename, "chunk_index": i}
+                                all_docs.append(Document(page_content=chunk_text, metadata=meta))
+                            processed_files.append(filename)
+
+        if not all_docs:
+            raise ValueError("No documents to index.")
+
+        self.processed_source_files = processed_files
+        self.logger.info(f"[INDEX_BUILD] Creating FAISS index with {len(all_docs)} chunks.")
+        
+        self.vector_store = FAISS.from_documents(all_docs, self.embeddings)
+        faiss_path = os.path.join(self.index_storage_dir, RAG_FAISS_INDEX_SUBDIR_NAME)
+        self.vector_store.save_local(faiss_path)
+        self._save_chunk_config()
+        
+        self.retriever = FAISSRetrieverWithScore(
+            vectorstore=self.vector_store,
+            reranker=self.reranker,
+            initial_fetch_k=RAG_INITIAL_FETCH_K,
+            final_k=RAG_RERANKER_K
+        )
+
+    def load_index_from_disk(self):
+        faiss_path = os.path.join(self.index_storage_dir, RAG_FAISS_INDEX_SUBDIR_NAME)
+        if not os.path.exists(faiss_path):
+            raise FileNotFoundError("Index not found.")
+
+        self.vector_store = FAISS.load_local(
+            folder_path=faiss_path,
+            embeddings=self.embeddings,
+            allow_dangerous_deserialization=True
+        )
+        self.retriever = FAISSRetrieverWithScore(
+            vectorstore=self.vector_store,
+            reranker=self.reranker,
+            initial_fetch_k=RAG_INITIAL_FETCH_K,
+            final_k=RAG_RERANKER_K
+        )
+        
+        meta_file = os.path.join(faiss_path, "processed_files.json")
+        if os.path.exists(meta_file):
+            with open(meta_file, 'r') as f:
+                self.processed_source_files = json.load(f)
+        else:
+            self.processed_source_files = ["Loaded from disk (unknown sources)"]
+            
+        self.logger.info("[INDEX_LOAD] Success.")
+
+    def update_index_with_new_files(self, source_folder_path: str, max_files_to_process: Optional[int] = None) -> Dict[str, Any]:
+        self.logger.info(f"[INDEX_UPDATE] Checking for new files in: {source_folder_path}")
+        
+        if not self.vector_store:
+            raise RuntimeError("Cannot update: no index loaded.")
+        
+        processed_set = set(self.processed_source_files)
+        all_new_files = []
+        for filename in sorted(os.listdir(source_folder_path)):
+            if filename not in processed_set:
+                file_ext = filename.split('.')[-1].lower()
+                if file_ext in FAISS_RAG_SUPPORTED_EXTENSIONS:
+                    all_new_files.append(filename)
+
+        if not all_new_files:
+            return {"status": "success", "message": "No new files found.", "files_added": []}
+            
+        limit = max_files_to_process if max_files_to_process is not None else RAG_MAX_FILES_FOR_INCREMENTAL
+        files_to_process = all_new_files[:limit]
+        
+        new_docs = []
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap)
+        
+        for filename in files_to_process:
+            file_path = os.path.join(source_folder_path, filename)
+            file_ext = filename.split('.')[-1].lower()
+
+            if file_ext == 'csv':
+                try:
+                    with open(file_path, mode='r', encoding='utf-8-sig') as f:
+                        reader = csv.DictReader(f)
+                        for i, row in enumerate(reader):
+                            row_text = "\n".join([f"{k}: {v}" for k, v in row.items() if k and v and str(v).strip()])
+                            meta = {"source_document_name": filename, "chunk_index": i, "source_type": "csv"}
+                            new_docs.append(Document(page_content=row_text, metadata=meta))
+                except Exception as e:
+                    self.logger.error(f"[INDEX_UPDATE] Error processing CSV {filename}: {e}")
+            else:
+                text_content = FAISS_RAG_SUPPORTED_EXTENSIONS[file_ext](file_path)
+                if text_content and text_content != "CSV_HANDLED_NATIVELY":
+                    chunks = text_splitter.split_text(text_content)
+                    for i, chunk_text in enumerate(chunks):
+                        meta = {"source_document_name": filename, "chunk_index": i}
+                        new_docs.append(Document(page_content=chunk_text, metadata=meta))
+        
+        if not new_docs:
+            return {"status": "warning", "message": "New files found but no text extracted.", "files_added": []}
+            
+        self.vector_store.add_documents(new_docs)
+        
+        faiss_path = os.path.join(self.index_storage_dir, RAG_FAISS_INDEX_SUBDIR_NAME)
+        self.vector_store.save_local(faiss_path)
+        
+        self.processed_source_files.extend(files_to_process)
+        with open(os.path.join(faiss_path, "processed_files.json"), 'w') as f:
+            json.dump(sorted(self.processed_source_files), f)
+
+        return {
+            "status": "success",
+            "message": f"Added {len(files_to_process)} files.",
+            "files_added": files_to_process,
+            "remaining": len(all_new_files) - len(files_to_process)
+        }
+
+    def search_knowledge_base(self, query: str, top_k: Optional[int] = None) -> List[Dict[str, Any]]:
+        if not self.retriever:
+            raise RuntimeError("Retriever not initialized.")
+        
+        original_k = self.retriever.final_k
+        if top_k:
+            self.retriever.final_k = top_k
+
+        try:
+            docs = self.retriever.invoke(query)
+            results = []
+            for doc in docs:
+                results.append({
+                    "content": doc.page_content,
+                    "metadata": doc.metadata,
+                    "score": doc.metadata.get("reranker_score") or doc.metadata.get("retrieval_score")
+                })
+            return results
+        finally:
+            self.retriever.final_k = original_k

rag_system.py

@@ -0,0 +1,70 @@
+import os
+import logging
+import shutil
+from typing import Optional
+
+from rag_components import KnowledgeRAG
+from utils import download_and_unzip_gdrive_folder
+from config import (
+    GDRIVE_SOURCES_ENABLED, GDRIVE_FOLDER_ID_OR_URL, RAG_SOURCES_DIR,
+    RAG_STORAGE_PARENT_DIR, RAG_FAISS_INDEX_SUBDIR_NAME, RAG_LOAD_INDEX_ON_STARTUP,
+    RAG_EMBEDDING_MODEL_NAME, RAG_EMBEDDING_USE_GPU, 
+    RAG_CHUNK_SIZE, RAG_CHUNK_OVERLAP,
+    RAG_RERANKER_MODEL_NAME, RAG_RERANKER_ENABLED
+)
+
+logger = logging.getLogger(__name__)
+
+def initialize_and_get_rag_system(force_rebuild: bool = False, source_dir_override: Optional[str] = None, storage_dir_override: Optional[str] = None) -> Optional[KnowledgeRAG]:
+    
+    logger.info("[RAG_SYSTEM_INIT] Initializing...")
+    source_dir_to_use = source_dir_override if source_dir_override else RAG_SOURCES_DIR
+    storage_dir_to_use = storage_dir_override if storage_dir_override else RAG_STORAGE_PARENT_DIR
+    
+    # GDrive Logic 
+    if GDRIVE_SOURCES_ENABLED and not source_dir_override and GDRIVE_FOLDER_ID_OR_URL:
+        logger.info("[RAG_SYSTEM_INIT] Downloading sources from GDrive...")
+        if os.path.exists(RAG_SOURCES_DIR):
+            shutil.rmtree(RAG_SOURCES_DIR)
+        download_and_unzip_gdrive_folder(GDRIVE_FOLDER_ID_OR_URL, RAG_SOURCES_DIR)
+
+    faiss_index_path = os.path.join(storage_dir_to_use, RAG_FAISS_INDEX_SUBDIR_NAME)
+
+    if force_rebuild and os.path.exists(faiss_index_path):
+        logger.info("[RAG_SYSTEM_INIT] Force rebuild: deleting old index.")
+        shutil.rmtree(faiss_index_path)
+
+    try:
+        rag = KnowledgeRAG(
+            index_storage_dir=storage_dir_to_use, 
+            embedding_model_name=RAG_EMBEDDING_MODEL_NAME,
+            use_gpu_for_embeddings=RAG_EMBEDDING_USE_GPU,
+            chunk_size=RAG_CHUNK_SIZE,
+            chunk_overlap=RAG_CHUNK_OVERLAP,
+            reranker_model_name=RAG_RERANKER_MODEL_NAME,
+            enable_reranker=RAG_RERANKER_ENABLED,
+        )
+
+        loaded = False
+        if RAG_LOAD_INDEX_ON_STARTUP and not force_rebuild:
+            if rag.chunk_config_has_changed():
+                logger.warning("[RAG_SYSTEM_INIT] Chunk config changed — forcing index rebuild.")
+            else:
+                try:
+                    rag.load_index_from_disk()
+                    loaded = True
+                except Exception as e:
+                    logger.warning(f"[RAG_SYSTEM_INIT] Load failed ({e}). Building new.")
+
+        if not loaded:
+            if not os.path.exists(source_dir_to_use) or not os.listdir(source_dir_to_use):
+                logger.warning("[RAG_SYSTEM_INIT] No sources found. System empty.")
+            else:
+                rag.build_index_from_source_files(source_dir_to_use)
+
+        logger.info("[RAG_SYSTEM_INIT] Complete.")
+        return rag
+
+    except Exception as e:
+        logger.critical(f"[RAG_SYSTEM_INIT] FATAL: {e}", exc_info=True)
+        return None

requirements.txt

@@ -0,0 +1,16 @@
+Flask==3.1.2
+Flask_Cors==5.0.0
+gdown==5.2.1
+langchain==1.2.10
+langchain_community==0.4.1
+langchain_huggingface==1.2.0
+pandas==3.0.0
+pypdf==6.7.0
+python-dotenv==1.2.1
+python_docx==1.1.2
+sentence_transformers==3.4.0
+torch==2.9.0
+langchain_core
+langchain_text_splitters
+faiss-cpu
+langchain-huggingface

sources/vehicle.csv

@@ -0,0 +1,262 @@
+Stock Number,Make,Model,Mileage,Registration,Human Description,Status,Location,Fuel,Voi Post Checked,Price
+"428,611",Volkswagen,tiguan,26036,GK72OLN,"Grey 1.4 litre Semi Auto Hybrid Electric Volkswagen tiguan life at 26,036 miles",In Stock,Cannock,Hybrid Electric,TRUE,"19,500"
+"424,986",Volkswagen,tiguan,66303,VE70LDJ,"Black 1.5 litre Semi Auto Petrol Volkswagen tiguan elegance at 66,303 miles",In Stock,Leeds,Petrol,TRUE,"17,700"
+"434,913",Volkswagen,tiguan,14529,DA21BHF,"Grey 1.5 litre Semi Auto Petrol Volkswagen tiguan life at 14,529 miles",In Stock,Camberley,Petrol,TRUE,"20,200"
+"417,721",Vauxhall,corsa,6597,LR73MYB,"Grey 1.2 litre Manual Petrol Vauxhall corsa design at 6,597 miles",In Stock,Sheffield,Petrol,TRUE,"10,500"
+"426,301",Land Rover,discovery sport,46282,LL21DDX,"Blue 1.5 litre Automatic Hybrid Electric Land Rover discovery sport hse at 46,282 miles",In Stock,Leeds,Hybrid Electric,TRUE,"22,600"
+"428,073",Ds,ds,42236,LB71OMC,"Black 1.2 litre Automatic Petrol Ds ds 3 crossback puretech performance line s/s eat8 at 42,236 miles",In Stock,Wimbledon,Petrol,TRUE,"11,800"
+"406,980",Audi,a4,7434,WR73DVW,"Black 2 litre Semi Auto Diesel Audi a4 avant quattro s line black edition mhev at 7,434 miles",In Stock,Leeds,Diesel,TRUE,"31,000"
+"434,265",Bmw,5 series,41102,MJ70PYP,"Grey 2 litre Automatic Electric Diesel Bmw 5 series m sport mhev at 41,102 miles",In Stock,Cannock,Electric Diesel,TRUE,"19,300"
+"422,417",Mg,mg,21471,ST72MHL,"Silver 1.5 litre Manual Petrol Mg mg hs excite at 21,471 miles",In Stock,Norwich,Petrol,TRUE,"12,279"
+"429,882",Nissan,qashqai,69582,LE72CPV,"Grey 1.5 litre Automatic Hybrid Electric Nissan qashqai e-power tekna at 69,582 miles",In Stock,Fengate,Hybrid Electric,TRUE,"17,414"
+"422,146",Volkswagen,tiguan,58117,LM19UNY,"Black 2 litre Manual Diesel Volkswagen tiguan allspace sel at 58,117 miles",In Stock,Fengate,Diesel,TRUE,"17,759"
+"425,521",Volkswagen,polo,32622,YH72EWR,"Blue 1 litre Semi Auto Petrol Volkswagen polo r-line at 32,622 miles",In Stock,Wimbledon,Petrol,TRUE,"16,500"
+"434,451",Volkswagen,polo,28183,SD72HNP,"Grey 1 litre Semi Auto Petrol Volkswagen polo r-line at 28,183 miles",In Stock,Wimbledon,Petrol,TRUE,"16,700"
+"418,382",Mini,hatch,27294,HN73XTE,"Blue/Black 1.5 litre Automatic Petrol Mini hatch cooper exclusive at 27,294 miles",In Stock,Leeds,Petrol,TRUE,"17,800"
+"415,694",Mercedes,cla,37995,KT23HMV,"Grey 1.3 litre Automatic Hybrid Electric Mercedes cla e amg line premium plus night edition at 37,995 miles",Reserved,Cannock,Hybrid Electric,TRUE,"21,100"
+"415,694",Mercedes,cla,37995,KT23HMV,"Grey 1.3 litre Automatic Hybrid Electric Mercedes cla e amg line premium plus night edition at 37,995 miles",Reserved,Cannock,Hybrid Electric,TRUE,"21,100"
+"423,612",Bmw,3 series,16729,LS69JXC,"Black 2 litre Automatic Petrol Bmw 3 series m sport at 16,729 miles",In Stock,Blue bell hill sales,Petrol,TRUE,"22,000"
+"422,684",Mercedes,cla,32207,WA72XRP,"Grey 1.3 litre Automatic Petrol Mercedes cla amg line premium plus night edition at 32,207 miles",Reserved,Sheffield,Petrol,TRUE,"23,300"
+"414,969",Mini,hatch,10644,HN72ZZO,"Black 0 litre Automatic Electric Mini hatch cooper s level 2 at 10,644 miles",In Stock,Camberley,Electric,TRUE,"12,473"
+"431,158",Polestar,polestar,20353,OW72LKK,"Black 0 litre Automatic Electric Polestar polestar 2 base at 20,353 miles",In Stock,Leeds,Electric,TRUE,"17,500"
+"437,101",Toyota,chr,32559,YE72LKF,"Grey 1.8 litre Cvt Hybrid Electric Toyota chr icon at 32,559 miles",In Stock,Sheffield,Hybrid Electric,TRUE,"17,000"
+"421,665",Audi,q2,21699,LT72KJF,"Grey 1.5 litre Semi Auto Petrol Audi q2 sport at 21,699 miles",In Stock,Camberley,Petrol,TRUE,"19,268"
+"428,189",Toyota,corolla,27409,BK72ZZX,"White 1.8 litre Cvt Hybrid Electric Toyota corolla icon tech at 27,409 miles",In Stock,Cannock,Hybrid Electric,TRUE,"18,400"
+"436,789",Audi,a1,39387,SA21WXX,"White 1 litre Semi Auto Petrol Audi a1 sportback sport at 39,387 miles",In Stock,Wimbledon,Petrol,TRUE,"15,200"
+"424,743",Audi,a1,23997,YP71XRT,"Yellow 1 litre Semi Auto Petrol Audi a1 sportback technik at 23,997 miles",In Stock,Blue bell hill sales,Petrol,TRUE,"16,100"
+"433,092",Vauxhall,corsa,6798,DN23UFU,"White 1.2 litre Automatic Petrol Vauxhall corsa gs at 6,798 miles",In Stock,Camberley,Petrol,TRUE,"13,935"
+"424,869",Hyundai,kona,46528,CN70CGY,"White 0 litre Automatic Electric Hyundai kona premium se at 46,528 miles",In Stock,Camberley,Electric,TRUE,"11,800"
+"417,721",Vauxhall,corsa,6597,LR73MYB,"Grey 1.2 litre Manual Petrol Vauxhall corsa design at 6,597 miles",In Stock,Sheffield,Petrol,TRUE,"10,500"
+"417,056",Skoda,kamiq,18462,DG72LOH,"Grey 1 litre Manual Petrol Skoda kamiq se drive tsi at 18,462 miles",In Stock,Cannock,Petrol,TRUE,"12,700"
+"435,085",Tesla,model,57721,YE71XMB,"White 0 litre Automatic Electric Tesla model 3 standard range plus at 57,721 miles",In Stock,Leeds,Electric,TRUE,"14,000"
+"406,286",Audi,a3,34668,HJ72MKG,"Black 1.5 litre Manual Petrol Audi a3 sportback s line mhev at 34,668 miles",In Stock,Blue bell hill sales,Petrol,TRUE,"17,500"
+"436,805",Citroen,grand,28183,LF69SKX,"White 1.2 litre Manual Petrol Citroen grand c4 spacetourer puretech flair s/s at 28,183 miles",In Stock,Fengate,Petrol,TRUE,"12,747"
+"428,265",Hyundai,kona,18880,MK73VCJ,"Grey 0 litre Automatic Electric Hyundai kona ultimate at 18,880 miles",In Stock,Fengate,Electric,TRUE,"17,200"
+"431,595",Peugeot,5008,42709,ST70SZX,"Black 1.2 litre Automatic Petrol Peugeot 5008 puretech s/s allure at 42,709 miles",In Stock,Fengate,Petrol,TRUE,"16,600"
+"419,093",Mg,mg,5607,YS25TEO,"Grey 1.5 litre Automatic Petrol Mg mg hs se dct at 5,607 miles",In Stock,Cannock,Petrol,TRUE,"18,300"
+"431,500",Mercedes,eqc,19751,WN71MWL,"Black 0 litre Automatic Electric Mercedes eqc 4matic amg line premium at 19,751 miles",In Stock,Leeds,Electric,TRUE,"25,300"
+"437,637",Mercedes,a-class,59136,HS71AYV,"White 1.3 litre Automatic Hybrid Electric Mercedes a-class e amg line edition premium at 59,136 miles",In Stock,Camberley,Hybrid Electric,TRUE,"17,200"
+"435,649",Mercedes,a-class,59210,BU70KMZ,"White 1.3 litre Automatic Hybrid Electric Mercedes a-class e amg line premium at 59,210 miles",In Stock,Sheffield,Hybrid Electric,TRUE,"15,500"
+"426,466",Tesla,model,45292,MW72WLE,"Grey 0 litre Automatic Electric Tesla model 3 model 3 at 45,292 miles",In Stock,Cannock,Electric,TRUE,"16,700"
+"437,692",Mg,mg,15493,PK24RUJ,"Grey 0 litre Automatic Electric Mg mg 4 xpower at 15,493 miles",In Stock,Sheffield,Electric,TRUE,"18,700"
+"428,618",Hyundai,ioniq,25750,BL72ETK,"Black 0 litre Automatic Electric Hyundai ioniq 5 premium at 25,750 miles",In Stock,Leeds,Electric,TRUE,"15,900"
+"434,887",Skoda,karoq,28146,DE22MWZ,"Grey 1.5 litre Semi Auto Petrol Skoda karoq se l tsi dsg at 28,146 miles",In Stock,Cannock,Petrol,TRUE,"19,900"
+"432,660",Vauxhall,astra,12295,LS74ZZB,"Red 1.2 litre Automatic Petrol Vauxhall astra gs at 12,295 miles",In Stock,Enfield,Petrol,TRUE,"16,900"
+"433,308",Peugeot,208,23400,DA22KME,"Grey 1.2 litre Automatic Petrol Peugeot 208 puretech gt s/s at 23,400 miles",In Stock,Leeds,Petrol,TRUE,"13,400"
+"425,987",Mercedes,cla,33568,WV22OHF,"White 1.3 litre Automatic Hybrid Electric Mercedes cla e amg line premium at 33,568 miles",In Stock,Sheffield,Hybrid Electric,TRUE,"19,500"
+"420,970",Land Rover,range rover evoque,89584,DT21HBC,"Grey 1.5 litre Automatic Hybrid Electric Land Rover range rover evoque hse at 89,584 miles",In Stock,Leeds,Hybrid Electric,TRUE,"18,500"
+"413,067",Lexus,ux,29066,HF22YFR,"Black 2 litre Cvt Hybrid Electric Lexus ux 250h at 29,066 miles",In Stock,Camberley,Hybrid Electric,TRUE,"18,300"
+"424,353",Lexus,ux,30213,FN71YBT,"Silver 2 litre Cvt Hybrid Electric Lexus ux 250h at 30,213 miles",In Stock,Sheffield,Hybrid Electric,TRUE,"17,900"
+"424,141",Tesla,model,26352,BK72JMX,"Grey 0 litre Automatic Electric Tesla model y long range awd at 26,352 miles",In Stock,Enfield,Electric,TRUE,"23,600"
+"432,935",Tesla,model,47247,LA72FYH,"Black 0 litre Automatic Electric Tesla model y long range awd at 47,247 miles",In Stock,Blue bell hill sales,Electric,TRUE,"21,800"
+"427,271",Audi,q3,55095,BT23CFF,"Blue 1.4 litre Semi Auto Hybrid Electric Audi q3 e technik at 55,095 miles",In Stock,Blue bell hill sales,Hybrid Electric,TRUE,"19,100"
+"429,006",Skoda,enyaq,31121,SD72OER,"Black 0 litre Automatic Electric Skoda enyaq iv 60 at 31,121 miles",In Stock,Leeds,Electric,TRUE,"16,000"
+"427,278",Polestar,polestar,62460,OE72VML,"Silver 0 litre Automatic Electric Polestar polestar 2 base at 62,460 miles",In Stock,Camberley,Electric,TRUE,"15,200"
+"429,046",Kia,niro,51915,ML22XAX,"Blue 0 litre Automatic Electric Kia niro 2 at 51,915 miles",In Stock,Sheffield,Electric,TRUE,"12,400"
+"427,368",Kia,niro,71221,CV71VZX,"Black 0 litre Automatic Electric Kia niro 4 plus at 71,221 miles",In Stock,Leeds,Electric,TRUE,"12,700"
+"430,171",Audi,q3,15596,KY73MXB,"Blue 1.4 litre Semi Auto Hybrid Electric Audi q3 e technik at 15,596 miles",In Stock,Fengate,Hybrid Electric,TRUE,"24,200"
+"426,807",Toyota,yaris,25606,FL70HWJ,"Blue 1.5 litre Cvt Hybrid Electric Toyota yaris design fhev at 25,606 miles",In Stock,Sheffield,Hybrid Electric,TRUE,"14,300"
+"432,503",Volkswagen,id4,26512,YC72CSZ,"Grey 0 litre Automatic Electric Volkswagen id4 life at 26,512 miles",In Stock,Leeds,Electric,TRUE,"16,100"
+"433,391",Volkswagen,id4,37517,ND22UAZ,"Black 0 litre Automatic Electric Volkswagen id4 life at 37,517 miles",In Stock,Cannock,Electric,TRUE,"15,000"
+"432,663",Mercedes,eqc,24994,RX72EVM,"Blue 0 litre Automatic Electric Mercedes eqc 4matic amg line at 24,994 miles",In Stock,Cannock,Electric,TRUE,"23,300"
+"429,592",Mercedes,gle-class,58814,BC19WGD,"Silver 2 litre Automatic Diesel Mercedes gle-class 4matic amg line premium at 58,814 miles",In Stock,Wimbledon,Diesel,TRUE,"29,500"
+"414,421",Mercedes,gle-class,60875,YG69ECR,"Blue 2 litre Automatic Diesel Mercedes gle-class 4matic amg line premium at 60,875 miles",Reserved,BBH MMC,Diesel,TRUE,"28,700"
+"431,695",Hyundai,bayon,27177,OE71AXO,"Bronze 1 litre Manual Hybrid Electric Hyundai bayon t-gdi se connect mhev at 27,177 miles",In Stock,Sheffield,Hybrid Electric,TRUE,"11,100"
+"431,695",Hyundai,bayon,27177,OE71AXO,"Bronze 1 litre Manual Hybrid Electric Hyundai bayon t-gdi se connect mhev at 27,177 miles",In Stock,Sheffield,Hybrid Electric,TRUE,"11,100"
+"408,106",Bmw,1 series,39234,NL69ANX,"White 2 litre Automatic Diesel Bmw 1 series m sport at 39,234 miles",In Stock,Blue bell hill sales,Diesel,TRUE,"16,100"
+"420,797",Bmw,ix3,15944,GF72XST,"Black 0 litre Automatic Electric Bmw ix3 sport pro at 15,944 miles",In Stock,Enfield,Electric,TRUE,"29,500"
+"428,319",Renault,zoe,41533,MT72LHU,"White 0 litre Automatic Electric Renault zoe s edition at 41,533 miles",In Stock,Camberley,Electric,TRUE,"9,200"
+"434,331",Nissan,leaf,36989,YP72GKC,"Grey 0 litre Automatic Electric Nissan leaf tekna at 36,989 miles",In Stock,Camberley,Electric,TRUE,"9,500"
+"423,282",Renault,zoe,30629,MT72LHJ,"White 0 litre Automatic Electric Renault zoe s edition at 30,629 miles",In Stock,Camberley,Electric,TRUE,"9,200"
+"417,597",Peugeot,208,25189,MK72XFC,"Grey 0 litre Automatic Electric Peugeot 208 gt premium at 25,189 miles",Reserved,On site (west malling),Electric,TRUE,"12,300"
+"435,356",Peugeot,208,28412,CP72DEU,"Yellow 0 litre Automatic Electric Peugeot 208 gt at 28,412 miles",In Stock,On site (west malling),Electric,TRUE,"12,300"
+"421,495",Hyundai,i10,13517,GL23RYF,"Blue 1.2 litre Automatic Petrol Hyundai i10 mpi se connect at 13,517 miles",In Stock,Enfield,Petrol,TRUE,"13,000"
+"434,227",Peugeot,208,15677,BL23AWX,"Red 0 litre Automatic Electric Peugeot 208 allure premium plus at 15,677 miles",In Stock,Wimbledon,Electric,TRUE,"12,400"
+"429,976",Bmw,3 series,64079,AF70HTV,"Black 2 litre Automatic Hybrid Electric Bmw 3 series m sport at 64,079 miles",In Stock,Wimbledon,Hybrid Electric,TRUE,"16,800"
+"436,263",Volvo,xc40,31542,GF71BNE,"Black 0 litre Automatic Electric Volvo xc40 recharge plus twin awd at 31,542 miles",In Stock,On site (west malling),Electric,TRUE,"21,800"
+"421,271",Mg,mg,68853,WU72YRD,"Black 1.5 litre Semi Auto Petrol Mg mg hs excite dct at 68,853 miles",In Stock,Fengate,Petrol,TRUE,"11,302"
+"410,642",Mg,mg,9650,EJ73FGK,"White 1 litre Automatic Petrol Mg mg zs exclusive t-gdi at 9,650 miles",In Stock,Fengate,Petrol,TRUE,"13,890"
+"429,671",Mg,mg,28871,YE72OOW,"Red 1.5 litre Manual Petrol Mg mg hs excite at 28,871 miles",In Stock,Leeds,Petrol,TRUE,"12,000"
+"407,014",Hyundai,ioniq,31030,VF21WZX,"White 1.6 litre Semi Auto Hybrid Electric Hyundai ioniq premium se at 31,030 miles",In Stock,Camberley,Hybrid Electric,TRUE,"16,464"
+"429,542",Ford,focus,27445,YM71SZL,"Blue 1 litre Manual Petrol Ford focus st-line x edition mhev at 27,445 miles",In Stock,Leeds,Petrol,TRUE,"13,600"
+"424,525",Ford,fiesta,6728,GL72RJY,"Grey 1 litre Automatic Petrol Ford fiesta titanium x mhev at 6,728 miles",In Stock,Leeds,Petrol,TRUE,"15,000"
+"419,966",Vauxhall,astra,14404,VK73ZTL,"Yellow/Black 1.5 litre Automatic Diesel Vauxhall astra ultimate at 14,404 miles",In Stock,Leeds,Diesel,TRUE,"18,300"
+"436,321",Citroen,c4,7101,AV22OBT,"Blue 1.2 litre Automatic Petrol Citroen c4 puretech sense plus s/s eat8 at 7,101 miles",In Stock,Wimbledon,Petrol,TRUE,"14,000"
+"435,895",Peugeot,5008,57228,MF22NWT,"White 1.6 litre Automatic Petrol Peugeot 5008 puretech s/s gt at 57,228 miles",In Stock,Enfield,Petrol,TRUE,"19,500"
+"427,069",Mercedes,a-class,40877,KT68XSX,"White 2 litre Automatic Petrol Mercedes a-class amg line premium plus at 40,877 miles",Deposit Taken,Leeds,Petrol,TRUE,"17,300"
+"419,827",Citroen,c3,22543,LB72FKJ,"Beige 1.2 litre Manual Petrol Citroen c3 puretech c-series edition s/s at 22,543 miles",In Stock,Enfield,Petrol,TRUE,"9,300"
+"435,029",Vauxhall,corsa,17071,DL73HNT,"Black 1.2 litre Automatic Petrol Vauxhall corsa gs at 17,071 miles",In Stock,Enfield,Petrol,TRUE,"13,400"
+"419,827",Citroen,c3,22543,LB72FKJ,"Beige 1.2 litre Manual Petrol Citroen c3 puretech c-series edition s/s at 22,543 miles",In Stock,Enfield,Petrol,TRUE,"9,300"
+"391,462",Toyota,chr,17725,BMZ6931,"Black 1.8 litre Cvt Hybrid Electric Toyota chr icon at 17,725 miles",In Stock,Cannock,Hybrid Electric,TRUE,"17,300"
+"420,612",Volkswagen,up,20913,YE72XSM,"Black 0 litre Cvt Electric Volkswagen up e-up at 20,913 miles",In Stock,Blue bell hill sales,Electric,TRUE,"10,000"
+"434,213",Bmw,x2,23001,KR72RNE,"Black 2 litre Manual Diesel Bmw x2 sport at 23,001 miles",In Stock,Leeds,Diesel,TRUE,"19,100"
+"429,954",Volkswagen,tiguan,25528,YC23KFX,"Black 1.5 litre Semi Auto Petrol Volkswagen tiguan r-line at 25,528 miles",In Stock,Wimbledon,Petrol,TRUE,"24,800"
+"428,070",Volkswagen,id4,29917,VK72ZSJ,"Grey 0 litre Automatic Electric Volkswagen id4 life at 29,917 miles",In Stock,Leeds,Electric,TRUE,"15,200"
+"430,367",Volkswagen,id4,17211,YF72PNV,"Black 0 litre Automatic Electric Volkswagen id4 life at 17,211 miles",In Stock,Leeds,Electric,TRUE,"17,300"
+"433,393",Bmw,ix3,43841,AO71XPN,"Blue 0 litre Automatic Electric Bmw ix3 edition pro at 43,841 miles",In Stock,Sheffield,Electric,TRUE,"20,900"
+"431,595",Peugeot,5008,42709,ST70SZX,"Black 1.2 litre Automatic Petrol Peugeot 5008 puretech s/s allure at 42,709 miles",In Stock,Fengate,Petrol,TRUE,"16,600"
+"428,914",Volkswagen,tiguan,70589,DE72UHU,"White 1.5 litre Semi Auto Petrol Volkswagen tiguan allspace life at 70,589 miles",In Stock,Blue bell hill sales,Petrol,TRUE,"17,000"
+"400,750",Bmw,3 series,73255,YG71WVL,"Grey 3 litre Automatic Electric Diesel Bmw 3 series xdrive mhev at 73,255 miles",In Stock,Cannock,Electric Diesel,TRUE,"24,200"
+"434,830",Mazda,mx-30,7990,NG72BTF,"Grey 0 litre Automatic Electric Mazda mx-30 gt sport tech at 7,990 miles",In Stock,Enfield,Electric,TRUE,"12,500"
+"436,889",Mercedes,a-class,18900,BX71VOF,"Black 1.3 litre Automatic Petrol Mercedes a-class amg line at 18,900 miles",In Stock,Sheffield,Petrol,TRUE,"18,500"
+"435,098",Bmw,1 series,56931,YG20YUY,"Grey 1.5 litre Manual Petrol Bmw 1 series m sport at 56,931 miles",In Stock,Cannock,Petrol,TRUE,"14,900"
+"425,808",Volvo,xc40,45932,YC72EOJ,"Grey 1.5 litre Automatic Hybrid Electric Volvo xc40 recharge t4 core at 45,932 miles",In Stock,Enfield,Hybrid Electric,TRUE,"18,600"
+"429,263",Polestar,polestar,2850,OW25CEU,"Silver 0 litre Automatic Electric Polestar polestar 4 base at 2,850 miles",In Stock,Enfield,Electric,TRUE,"34,800"
+"427,311",Volvo,xc40,75344,YY72OXG,"Blue 0 litre Automatic Electric Volvo xc40 recharge ultimate at 75,344 miles",In Stock,Wimbledon,Electric,TRUE,"18,600"
+"436,710",Ford,ecosport,24115,FV72XLD,"Silver 1 litre Manual Petrol Ford ecosport active at 24,115 miles",In Stock,On site (west malling),Petrol,TRUE,"11,300"
+"438,257",Toyota,rav-4,13877,YD72NVO,"Grey 2.5 litre Cvt Hybrid Electric Toyota rav-4 vvt-i icon at 13,877 miles",In Stock,On site (west malling),Hybrid Electric,TRUE,"26,100"
+"419,858",Toyota,rav-4,38002,YD72TKC,"Grey/Black 2.5 litre Automatic Hybrid Electric Toyota rav-4 vvt-i dynamic phev at 38,002 miles",In Stock,Enfield,Hybrid Electric,TRUE,"26,500"
+"422,517",Toyota,rav-4,15242,LD72GYJ,"Blue 2.5 litre Cvt Hybrid Electric Toyota rav-4 vvt-i excel at 15,242 miles",In Stock,Enfield,Hybrid Electric,TRUE,"29,000"
+"432,146",Polestar,polestar,29115,OW72LSZ,"Silver 0 litre Automatic Electric Polestar polestar 2 plus at 29,115 miles",In Stock,Sheffield,Electric,TRUE,"20,200"
+"414,972",Audi,a4,40747,BG21SDU,"Grey 2 litre Semi Auto Diesel Audi a4 avant sport edition mhev at 40,747 miles",In Stock,Norwich,Diesel,TRUE,"18,700"
+"434,315",Nissan,ariya,43947,SH72DPK,"Green 0 litre Automatic Electric Nissan ariya evolve at 43,947 miles",In Stock,On site (west malling),Electric,TRUE,"19,800"
+"435,253",Kia,sportage,11532,EN24HCA,"Black 1.6 litre Semi Auto Hybrid Electric Kia sportage gt-line at 11,532 miles",In Stock,On site (west malling),Hybrid Electric,TRUE,"24,600"
+"427,475",Mg,mg,13527,FD72MVG,"Red 1.5 litre Manual Petrol Mg mg hs excite at 13,527 miles",In Stock,Norwich,Petrol,TRUE,"12,874"
+"426,946",Nissan,leaf,5197,YS72FSG,"Grey 0 litre Automatic Electric Nissan leaf acenta at 5,197 miles",In Stock,Enfield,Electric,TRUE,"9,300"
+"430,046",Hyundai,ioniq,6205,BK72CKC,"Blue 0 litre Automatic Electric Hyundai ioniq premium at 6,205 miles",In Stock,Enfield,Electric,TRUE,"12,700"
+"428,541",Land Rover,range rover evoque,46584,YM21BGZ,"Black 2 litre Automatic Diesel Land Rover range rover evoque s mhev at 46,584 miles",Reserved,Cannock,Diesel,TRUE,"20,500"
+"437,158",Jaecoo,7,3648,YH75RJV,"Black 1.5 litre Automatic Hybrid Electric Jaecoo 7 luxury at 3,648 miles",In Stock,Sheffield,Hybrid Electric,TRUE,"28,400"
+"431,927",Hyundai,bayon,14497,AVI4307,"White 1 litre Automatic Hybrid Electric Hyundai bayon t-gdi premium dct mhev at 14,497 miles",In Stock,Enfield,Hybrid Electric,TRUE,"14,900"
+"434,369",Volkswagen,passat,53021,RJ21XLM,"White 1.4 litre Semi Auto Hybrid Electric Volkswagen passat gte dsg at 53,021 miles",In Stock,Camberley,Hybrid Electric,TRUE,"14,553"
+"419,439",Honda,civic,39251,KJ68HHM,"White 1 litre Manual Petrol Honda civic vtec sport line at 39,251 miles",In Stock,Leeds,Petrol,TRUE,"11,300"
+"423,587",Honda,cr-v,15024,FG72VNE,"Grey 2 litre Cvt Hybrid Electric Honda cr-v i-mmd se at 15,024 miles",In Stock,On site (west malling),Hybrid Electric,TRUE,"21,800"
+"417,336",Kia,ceed,16627,RF74GFY,"Red 1.5 litre Manual Petrol Kia ceed gt-line at 16,627 miles",In Stock,Wimbledon,Petrol,TRUE,"16,300"
+"433,857",Lexus,ux,34891,CA22BXC,"Blue 0 litre Automatic Electric Lexus ux 300e at 34,891 miles",In Stock,Sheffield,Electric,TRUE,"13,400"
+"422,334",Volvo,xc60,26980,DA72FKF,"White 2 litre Automatic Hybrid Electric Volvo xc60 recharge t6 plus awd at 26,980 miles",In Stock,Camberley,Hybrid Electric,TRUE,"30,600"
+"424,836",Audi,q3,43347,AK70FKS,"Blue 1.5 litre Semi Auto Petrol Audi q3 s line edition 1 mhev at 43,347 miles",In Stock,Blue bell hill sales,Petrol,TRUE,"22,800"
+"427,097",Bmw,i4,50448,HN72XKS,"Green 0 litre Automatic Electric Bmw i4  at 50,448 miles",In Stock,Leeds,Electric,TRUE,"27,500"
+"431,391",Mg,mg,10468,WA71VPM,"Blue 1 litre Automatic Petrol Mg mg zs excite t-gdi at 10,468 miles",In Stock,Cannock,Petrol,TRUE,"12,700"
+"430,942",Volkswagen,id4,33878,VN72DWO,"Grey 0 litre Automatic Electric Volkswagen id4 life at 33,878 miles",In Stock,Sheffield,Electric,TRUE,"15,000"
+"433,852",Jaguar,f-pace,45232,BV72LTJ,"White 2 litre Automatic Diesel Jaguar f-pace r-dynamic black mhev at 45,232 miles",In Stock,Sheffield,Diesel,TRUE,"26,700"
+"416,622",Volkswagen,id4,20906,VE72UJG,"Blue 0 litre Automatic Electric Volkswagen id4 life edition at 20,906 miles",In Stock,Camberley,Electric,TRUE,"16,100"
+"415,271",Volkswagen,id4,28823,VN72ZDJ,"Grey 0 litre Automatic Electric Volkswagen id4 life at 28,823 miles",In Stock,Camberley,Electric,TRUE,"14,764"
+"420,157",Volkswagen,id4,6890,ND74CWW,"White 0 litre Automatic Electric Volkswagen id4 match pro 4motion at 6,890 miles",In Stock,Camberley,Electric,TRUE,"25,700"
+"424,814",Kia,niro,35474,DP73KZR,"Blue 0 litre Automatic Electric Kia niro 2 at 35,474 miles",In Stock,Enfield,Electric,TRUE,"16,500"
+"420,304",Kia,niro,27186,NJ72LNO,"Red 0 litre Automatic Electric Kia niro 2 at 27,186 miles",In Stock,Enfield,Electric,TRUE,"15,000"
+"421,950",Kia,ev6,48027,MJ71ZWE,"Black 0 litre Automatic Electric Kia ev6 gt-line at 48,027 miles",In Stock,Enfield,Electric,TRUE,"17,800"
+"418,761",Volkswagen,golf,49795,WM21FNV,"White 1.4 litre Semi Auto Hybrid Electric Volkswagen golf gte dsg at 49,795 miles",In Stock,Wimbledon,Hybrid Electric,TRUE,"16,100"
+"429,561",Audi,a5,79959,WV72ULJ,"Black 2 litre Semi Auto Diesel Audi a5 sportback s line mhev at 79,959 miles",In Stock,Leeds,Diesel,TRUE,"19,500"
+"424,056",Mazda,cx-5,35286,PO72NBZ,"White 2 litre Manual Petrol Mazda cx-5 sport edition at 35,286 miles",In Stock,On site (west malling),Petrol,TRUE,"17,200"
+"430,693",Hyundai,tucson,38060,OE72FBO,"Grey 1.6 litre Semi Auto Hybrid Electric Hyundai tucson t-gdi se connect dct mhev at 38,060 miles",In Stock,Cannock,Hybrid Electric,TRUE,"17,700"
+"435,584",Volvo,xc40,51698,YY71KYH,"Silver 0 litre Automatic Electric Volvo xc40 recharge pro twin awd at 51,698 miles",In Stock,Cannock,Electric,TRUE,"19,900"
+"435,584",Volvo,xc40,51698,YY71KYH,"Silver 0 litre Automatic Electric Volvo xc40 recharge pro twin awd at 51,698 miles",In Stock,Cannock,Electric,TRUE,"19,900"
+"426,776",Mercedes,eqc,40305,WJ71VXF,"Grey 0 litre Automatic Electric Mercedes eqc 4matic amg line premium plus at 40,305 miles",In Stock,Blue bell hill sales,Electric,TRUE,"24,600"
+"435,577",Volvo,xc40,54691,YM71HCE,"Black 1.5 litre Automatic Hybrid Electric Volvo xc40 recharge t5 r-design pro at 54,691 miles",In Stock,Cannock,Hybrid Electric,TRUE,"20,600"
+"425,975",Polestar,polestar,50455,OW23KCX,"Silver 0 litre Automatic Electric Polestar polestar 2 base at 50,455 miles",In Stock,Cannock,Electric,TRUE,"16,300"
+"435,303",Mazda,cx-5,43693,SY72WTO,"Blue 2 litre Manual Petrol Mazda cx-5 sport edition at 43,693 miles",In Stock,Leeds,Petrol,TRUE,"17,000"
+"437,137",Peugeot,3008,59000,AP72OFA,"Grey 1.6 litre Automatic Hybrid Electric Peugeot 3008 s/s allure premium plus at 59,000 miles",In Stock,Cannock,Hybrid Electric,TRUE,"13,800"
+"410,052",Polestar,polestar,40706,OU71YMV,"Blue 0 litre Automatic Electric Polestar polestar 2 base at 40,706 miles",In Stock,Blue bell hill sales,Electric,TRUE,"16,700"
+"430,454",Volkswagen,tiguan,31183,VN72OEA,"Black 1.5 litre Manual Petrol Volkswagen tiguan allspace life at 31,183 miles",In Stock,Camberley,Petrol,TRUE,"19,801"
+"430,456",Volkswagen,tiguan,27693,DV72ZNU,"Red 1.5 litre Semi Auto Petrol Volkswagen tiguan allspace life at 27,693 miles",In Stock,Camberley,Petrol,TRUE,"20,500"
+"428,914",Volkswagen,tiguan,70589,DE72UHU,"White 1.5 litre Semi Auto Petrol Volkswagen tiguan allspace life at 70,589 miles",In Stock,Blue bell hill sales,Petrol,TRUE,"17,000"
+"433,789",Seat,tarraco,51271,SO70AEW,"White 1.5 litre Manual Petrol Seat tarraco tsi evo xcellence at 51,271 miles",In Stock,Wimbledon,Petrol,TRUE,"16,300"
+"424,032",Peugeot,208,48525,WU20RFZ,"White 0 litre Automatic Electric Peugeot 208 allure at 48,525 miles",In Stock,Cannock,Electric,TRUE,"9,300"
+"435,824",Volkswagen,t-roc,64732,FE21XLK,"Grey 1.5 litre Semi Auto Petrol Volkswagen t-roc r-line at 64,732 miles",In Stock,Blue bell hill sales,Petrol,TRUE,"16,800"
+"423,996",Volkswagen,id4,25376,VN72PHV,"Grey 0 litre Automatic Electric Volkswagen id4 life at 25,376 miles",In Stock,Sheffield,Electric,TRUE,"14,800"
+"423,996",Volkswagen,id4,25376,VN72PHV,"Grey 0 litre Automatic Electric Volkswagen id4 life at 25,376 miles",In Stock,Sheffield,Electric,TRUE,"14,800"
+"405,627",Polestar,polestar,46107,OV23LZN,"Grey 0 litre Automatic Electric Polestar polestar 2 base at 46,107 miles",In Stock,Camberley,Electric,TRUE,"16,200"
+"430,352",Hyundai,kona,67143,CN70CHZ,"White 0 litre Automatic Electric Hyundai kona premium se at 67,143 miles",In Stock,On site (west malling),Electric,TRUE,"10,500"
+"422,524",Tesla,model,32048,LA72ENK,"White 0 litre Automatic Electric Tesla model y long range awd at 32,048 miles",In Stock,Blue bell hill sales,Electric,TRUE,"22,300"
+"429,225",Volkswagen,golf,26720,MM73NWJ,"Grey 1.5 litre Manual Petrol Volkswagen golf r-line at 26,720 miles",In Stock,Blue bell hill sales,Petrol,TRUE,"19,500"
+"436,585",Toyota,corolla,53906,FY71YTE,"Grey 1.8 litre Cvt Hybrid Electric Toyota corolla design at 53,906 miles",In Stock,Cannock,Hybrid Electric,TRUE,"17,800"
+"425,060",Kia,ceed,7785,MH73ATK,"Grey 1.5 litre Manual Petrol Kia ceed 2 isg at 7,785 miles",In Stock,On site (west malling),Petrol,TRUE,"15,500"
+"434,391",Ford,ka+,66501,AO69TXE,"Silver 1.2 litre Manual Petrol Ford ka+ active at 66,501 miles",Reserved,Norwich,Petrol,TRUE,"6,747"
+"432,976",Vauxhall,crossland,46755,HJ18GMY,"Black 1.2 litre Manual Petrol Vauxhall crossland x se ecotec s/s at 46,755 miles",In Stock,Norwich,Petrol,TRUE,"7,000"
+"431,320",Renault,zoe,69801,YC21BWT,"Black 0 litre Automatic Electric Renault zoe gt line at 69,801 miles",In Stock,Leeds,Electric,TRUE,"7,100"
+"431,320",Renault,zoe,69801,YC21BWT,"Black 0 litre Automatic Electric Renault zoe gt line at 69,801 miles",In Stock,Leeds,Electric,TRUE,"7,100"
+"436,491",Ford,ecosport,71498,YV19DYM,"Blue 1 litre Manual Petrol Ford ecosport zetec at 71,498 miles",In Stock,Norwich,Petrol,TRUE,"7,109"
+"431,975",Hyundai,kona,65160,DS71HWT,"Grey 0 litre Automatic Electric Hyundai kona premium at 65,160 miles",In Stock,Wimbledon,Electric,TRUE,"11,500"
+"416,558",Mercedes,c-class,39484,YB18UEL,"Black 2.1 litre Automatic Diesel Mercedes c-class 4matic amg line premium plus at 39,484 miles",In Stock,Sheffield,Diesel,TRUE,"17,100"
+"430,352",Hyundai,kona,67143,CN70CHZ,"White 0 litre Automatic Electric Hyundai kona premium se at 67,143 miles",In Stock,On site (west malling),Electric,TRUE,"10,500"
+"427,035",Hyundai,kona,49543,CN70CGO,"White 0 litre Automatic Electric Hyundai kona premium se at 49,543 miles",In Stock,Enfield,Electric,TRUE,"11,700"
+"431,314",Nissan,qashqai,30426,YB71FJP,"Blue 1.3 litre Cvt Hybrid Electric Nissan qashqai dig-t tekna dct mhev at 30,426 miles",In Stock,Leeds,Hybrid Electric,TRUE,"17,100"
+"429,722",Nissan,qashqai,46244,DL73ZRX,"Grey 1.3 litre Cvt Hybrid Electric Nissan qashqai dig-t n-connecta dct mhev at 46,244 miles",In Stock,Cannock,Hybrid Electric,TRUE,"15,600"
+"417,290",Volvo,xc40,22430,DF21GUE,"Silver 2 litre Automatic Hybrid Electric Volvo xc40 b4 r-design pro mhev at 22,430 miles",In Stock,Norwich,Hybrid Electric,TRUE,"23,778"
+"414,558",Volkswagen,t-roc,35579,PL23JTO,"Blue 1 litre Manual Petrol Volkswagen t-roc life at 35,579 miles",In Stock,Norwich,Petrol,TRUE,"15,200"
+"430,354",Bmw,x3,59821,FL23DLX,"Grey 2 litre Automatic Hybrid Electric Bmw x3 m sport at 59,821 miles",In Stock,Blue bell hill sales,Hybrid Electric,TRUE,"29,500"
+"421,976",Tesla,model,14776,LB72SXC,"White 0 litre Automatic Electric Tesla model 3 model 3 at 14,776 miles",In Stock,Blue bell hill sales,Electric,TRUE,"18,300"
+"420,814",Tesla,model,31833,GF71WGU,"White 0 litre Automatic Electric Tesla model 3 standard range plus at 31,833 miles",In Stock,Blue bell hill sales,Electric,TRUE,"15,300"
+"432,303",Vauxhall,mokka,31466,LY72DWU,"Grey 1.2 litre Automatic Petrol Vauxhall mokka gs line at 31,466 miles",In Stock,Leeds,Petrol,TRUE,"14,075"
+"436,326",Vauxhall,mokka,14462,LF72KJH,"Black 1.2 litre Automatic Petrol Vauxhall mokka ultimate at 14,462 miles",In Stock,Leeds,Petrol,TRUE,"15,700"
+"435,853",Bmw,1 series,37601,YK68ZWR,"Black 2 litre Automatic Petrol Bmw 1 series m sport shadow edition at 37,601 miles",In Stock,Norwich,Petrol,TRUE,"15,627"
+"434,842",Seat,ateca,56834,YR20TCY,"Brown 1.6 litre Semi Auto Diesel Seat ateca tdi xcellence lux dsg at 56,834 miles",In Stock,Sheffield,Diesel,TRUE,"14,200"
+"428,141",Toyota,aygo,38467,WH73HFK,"Grey 1 litre Automatic Petrol Toyota aygo x vvt-i undercover at 38,467 miles",Reserved,Enfield,Petrol,TRUE,"13,900"
+"437,563",Bmw,ix3,31678,NU72KWM,"Grey 0 litre Automatic Electric Bmw ix3 sport at 31,678 miles",In Stock,Fengate,Electric,TRUE,"23,800"
+"424,440",Ford,puma,42007,DS72UGR,"White 1 litre Manual Petrol Ford puma st-line vignale mhev at 42,007 miles",In Stock,Enfield,Petrol,TRUE,"13,800"
+"430,625",Audi,q3,19464,SH22FYP,"White 1.5 litre Semi Auto Petrol Audi q3 technik mhev at 19,464 miles",In Stock,Sheffield,Petrol,TRUE,"21,200"
+"434,260",Peugeot,3008,31050,VE71MYK,"Blue 1.2 litre Automatic Petrol Peugeot 3008 puretech s/s allure premium at 31,050 miles",In Stock,Sheffield,Petrol,TRUE,"15,000"
+"415,768",Mg,mg,21332,VK73HPJ,"Red 0 litre Automatic Electric Mg mg 4 se at 21,332 miles",In Stock,Wimbledon,Electric,TRUE,"12,300"
+"435,311",Nissan,juke,18034,KY72CMV,"Silver 1.6 litre Cvt Hybrid Electric Nissan juke n-connecta at 18,034 miles",In Stock,Camberley,Hybrid Electric,TRUE,"14,800"
+"437,522",Mercedes,cla,54927,RO20WUX,"White 1.3 litre Semi Auto Petrol Mercedes cla amg line at 54,927 miles",In Stock,Sheffield,Petrol,TRUE,"17,200"
+"431,798",Tesla,model,51704,BD72WXG,"Grey 0 litre Automatic Electric Tesla model y long range awd at 51,704 miles",In Stock,Leeds,Electric,TRUE,"21,300"
+"433,916",Mg,mg,54038,FP72VPW,"Blue 1.5 litre Manual Petrol Mg mg zs excite vti-tech at 54,038 miles",In Stock,Sheffield,Petrol,TRUE,"9,900"
+"428,998",Ford,ecosport,17541,YS70FRX,"Silver 1 litre Manual Petrol Ford ecosport st-line at 17,541 miles",In Stock,Sheffield,Petrol,TRUE,"11,100"
+"425,060",Kia,ceed,7785,MH73ATK,"Grey 1.5 litre Manual Petrol Kia ceed 2 isg at 7,785 miles",In Stock,On site (west malling),Petrol,TRUE,"15,500"
+"433,633",Kia,sportage,39085,FL70KFZ,"Red 1.6 litre Manual Petrol Kia sportage 2 isg at 39,085 miles",In Stock,Camberley,Petrol,TRUE,"14,008"
+"427,577",Skoda,octavia,33416,BL71RYB,"White 1.4 litre Semi Auto Hybrid Electric Skoda octavia se l tsi dsg at 33,416 miles",In Stock,On site (west malling),Hybrid Electric,TRUE,"17,100"
+"427,577",Skoda,octavia,33416,BL71RYB,"White 1.4 litre Semi Auto Hybrid Electric Skoda octavia se l tsi dsg at 33,416 miles",In Stock,On site (west malling),Hybrid Electric,TRUE,"17,100"
+"431,312",Toyota,corolla,50533,YP70TVM,"Black 1.8 litre Cvt Hybrid Electric Toyota corolla design at 50,533 miles",In Stock,On site (west malling),Hybrid Electric,TRUE,"15,400"
+"421,793",Tesla,model,36421,LA72EPZ,"White 0 litre Automatic Electric Tesla model 3 model 3 at 36,421 miles",In Stock,Camberley,Electric,TRUE,"17,100"
+"436,373",Citroen,c5,42255,MM69AYO,"Red 1.6 litre Automatic Petrol Citroen c5 aircross puretech flair plus s/s eat8 at 42,255 miles",In Stock,On site (west malling),Petrol,TRUE,"13,000"
+"423,720",Vauxhall,corsa,21653,VE20PXC,"Red 1.2 litre Manual Petrol Vauxhall corsa se at 21,653 miles",In Stock,On site (west malling),Petrol,TRUE,"9,300"
+"423,686",Hyundai,ioniq,59659,FL71YSY,"Grey 0 litre Automatic Electric Hyundai ioniq 5 ultimate at 59,659 miles",In Stock,Sheffield,Electric,TRUE,"15,600"
+"435,301",Hyundai,bayon,54746,WM72LJC,"Turquoise 1 litre Automatic Hybrid Electric Hyundai bayon t-gdi premium dct mhev at 54,746 miles",In Stock,Camberley,Hybrid Electric,TRUE,"12,044"
+"435,301",Hyundai,bayon,54746,WM72LJC,"Turquoise 1 litre Automatic Hybrid Electric Hyundai bayon t-gdi premium dct mhev at 54,746 miles",In Stock,Camberley,Hybrid Electric,TRUE,"12,044"
+"436,373",Citroen,c5,42255,MM69AYO,"Red 1.6 litre Automatic Petrol Citroen c5 aircross puretech flair plus s/s eat8 at 42,255 miles",In Stock,On site (west malling),Petrol,TRUE,"13,000"
+"437,184",Mazda,cx-30,59945,RO71OSB,"Red 2 litre Manual Petrol Mazda cx-30 gt sport mhev at 59,945 miles",In Stock,Camberley,Petrol,TRUE,"12,800"
+"431,914",Mazda,mx-30,11459,AY72KNB,"Grey 0 litre Automatic Electric Mazda mx-30 prime-line at 11,459 miles",In Stock,Fengate,Electric,TRUE,"10,149"
+"415,919",Volkswagen,golf,12913,GF72OPT,"Grey 1.4 litre Semi Auto Hybrid Electric Volkswagen golf style at 12,913 miles",In Stock,Enfield,Hybrid Electric,TRUE,"18,100"
+"428,067",Nissan,leaf,40409,LS71XGA,"Grey 0 litre Automatic Electric Nissan leaf n-connecta at 40,409 miles",In Stock,Leeds,Electric,TRUE,"8,300"
+"426,352",Nissan,leaf,22799,LN20TXJ,"Black 0 litre Automatic Electric Nissan leaf tekna at 22,799 miles",In Stock,Leeds,Electric,TRUE,"8,800"
+"431,874",Omoda,5,5054,DA25YKR,"Red 0 litre Automatic Electric Omoda 5 noble at 5,054 miles",In Stock,Leeds,Electric,TRUE,"21,500"
+"434,150",Volkswagen,golf,31706,MW72BFP,"Grey 1.5 litre Manual Petrol Volkswagen golf style at 31,706 miles",In Stock,Leeds,Petrol,TRUE,"17,500"
+"434,422",Volkswagen,golf,10773,LM70YBG,"Black 1.5 litre Semi Auto Hybrid Electric Volkswagen golf life etsi dsg at 10,773 miles",In Stock,Leeds,Hybrid Electric,TRUE,"17,300"
+"435,747",Audi,q2,36636,BK22ADO,"Grey 1 litre Manual Petrol Audi q2 sport at 36,636 miles",In Stock,Wimbledon,Petrol,TRUE,"16,200"
+"437,763",Fiat,500e,27173,ML22OKP,"Blue 0 litre Automatic Electric Fiat 500e passion at 27,173 miles",In Stock,Camberley,Electric,TRUE,"10,700"
+"438,887",Suzuki,sx4,39525,PY72JFJ,"Blue 1.4 litre Manual Hybrid Electric Suzuki sx4 s-cross motion boosterjet mhev at 39,525 miles",In Stock,Enfield,Hybrid Electric,TRUE,"12,100"
+"430,437",Suzuki,sx4,20095,SH72KWC,"Blue 1.4 litre Manual Hybrid Electric Suzuki sx4 s-cross motion boosterjet mhev at 20,095 miles",In Stock,Enfield,Hybrid Electric,TRUE,"13,100"
+"431,601",Suzuki,swace,14565,SC72HXO,"White 1.8 litre Cvt Hybrid Electric Suzuki swace sz-t at 14,565 miles",In Stock,Cannock,Hybrid Electric,TRUE,"16,775"
+"428,934",Suzuki,swace,18503,WK72HKF,"Silver 1.8 litre Cvt Hybrid Electric Suzuki swace sz5 at 18,503 miles",In Stock,Leeds,Hybrid Electric,TRUE,"17,000"
+"403,095",Audi,q4,513,FL24XZM,Blue 0 litre Automatic Electric Audi q4 black edition at 513 miles,In Stock,Blue bell hill sales,Electric,TRUE,"30,800"
+"434,672",Suzuki,swace,19991,DV72RYA,"Blue 1.8 litre Cvt Hybrid Electric Suzuki swace sz5 at 19,991 miles",In Stock,Norwich,Hybrid Electric,TRUE,"17,544"
+"428,928",Volkswagen,t-roc,34184,GV72KYJ,"White 1.5 litre Semi Auto Petrol Volkswagen t-roc life at 34,184 miles",In Stock,Blue bell hill sales,Petrol,TRUE,"17,800"
+"430,248",Audi,q3,52628,MJ21XDW,"Blue 1.5 litre Manual Petrol Audi q3 sportback s line at 52,628 miles",In Stock,Camberley,Petrol,TRUE,"19,500"
+"436,344",Audi,q3,3232,YO21FDV,"Black 2 litre Manual Diesel Audi q3 s line black edition at 3,232 miles",In Stock,Camberley,Diesel,TRUE,"25,931"
+"431,431",Audi,q3,21464,RE72MZV,"Black 1.5 litre Semi Auto Petrol Audi q3 sportback sport mhev at 21,464 miles",In Stock,Camberley,Petrol,TRUE,"25,700"
+"427,958",Volkswagen,t-roc,10358,RA73MGE,"Grey 1.5 litre Semi Auto Petrol Volkswagen t-roc r-line at 10,358 miles",In Stock,Sheffield,Petrol,TRUE,"24,100"
+"429,165",Jaguar,f-pace,59504,OW72TZS,"White 2 litre Automatic Hybrid Electric Jaguar f-pace r-dynamic black at 59,504 miles",In Stock,Camberley,Hybrid Electric,TRUE,"28,500"
+"434,269",Jaguar,f-pace,53220,FX22TPF,"Black 2 litre Automatic Diesel Jaguar f-pace r-dynamic s mhev at 53,220 miles",In Stock,Fengate,Diesel,TRUE,"23,984"
+"430,137",Land Rover,discovery sport,68288,VE22NNG,"Silver 1.5 litre Automatic Hybrid Electric Land Rover discovery sport hse phev at 68,288 miles",In Stock,Blue bell hill sales,Hybrid Electric,TRUE,"21,500"
+"421,747",Audi,q3,15943,PA21UTC,"Black 1.5 litre Semi Auto Petrol Audi q3 s line mhev at 15,943 miles",In Stock,Enfield,Petrol,TRUE,"23,200"
+"423,064",Audi,q3,22135,LV71TUO,"Grey 1.5 litre Semi Auto Petrol Audi q3 s line mhev at 22,135 miles",In Stock,Sheffield,Petrol,TRUE,"23,300"
+"433,494",Audi,q3,18516,LL22TMU,"Grey 1.5 litre Semi Auto Petrol Audi q3 s line mhev at 18,516 miles",In Stock,Blue bell hill sales,Petrol,TRUE,"25,300"
+"438,289",Nissan,qashqai,51288,VO69NWJ,"Silver 1.5 litre Manual Diesel Nissan qashqai dci acenta premium at 51,288 miles",In Stock,Cannock,Diesel,TRUE,"10,000"
+"435,570",Vauxhall,crossland,27526,SA70MKX,"Black 1.2 litre Automatic Petrol Vauxhall crossland x elite nav at 27,526 miles",In Stock,Fengate,Petrol,TRUE,"10,969"
+"422,404",Vauxhall,crossland,35723,AX70DNV,"Grey 1.2 litre Automatic Petrol Vauxhall crossland elite nav at 35,723 miles",Reserved,Norwich,Petrol,TRUE,"10,998"
+"436,022",Citroen,c3,27576,WR21BXX,"Grey 1.2 litre Automatic Petrol Citroen c3 puretech shine plus s/s eat6 at 27,576 miles",In Stock,Fengate,Petrol,TRUE,"11,025"
+"428,611",Volkswagen,tiguan,26036,GK72OLN,"Grey 1.4 litre Semi Auto Hybrid Electric Volkswagen tiguan life at 26,036 miles",In Stock,Cannock,Hybrid Electric,TRUE,"19,500"
+"435,135",Volkswagen,tiguan,23491,VE72OWA,"White 1.5 litre Semi Auto Petrol Volkswagen tiguan r-line at 23,491 miles",Reserved,Cannock,Petrol,TRUE,"24,900"
+"434,491",Ford,kuga,69411,BD68VXV,"White 1.5 litre Manual Petrol Ford kuga st-line at 69,411 miles",In Stock,Fengate,Petrol,TRUE,"9,500"
+"432,976",Vauxhall,crossland,46755,HJ18GMY,"Black 1.2 litre Manual Petrol Vauxhall crossland x se ecotec s/s at 46,755 miles",In Stock,Norwich,Petrol,TRUE,"7,000"
+"436,491",Ford,ecosport,71498,YV19DYM,"Blue 1 litre Manual Petrol Ford ecosport zetec at 71,498 miles",In Stock,Norwich,Petrol,TRUE,"7,109"
+"434,646",Citroen,c5,55142,NL72UHZ,"White 1.6 litre Automatic Petrol Citroen c5 x puretech shine plus s/s eat8 at 55,142 miles",In Stock,Blue bell hill sales,Petrol,TRUE,"13,200"
+"434,138",Kia,rio,13618,NG71BRV,"Silver 1.2 litre Manual Petrol Kia rio 1 at 13,618 miles",In Stock,Wimbledon,Petrol,TRUE,"11,100"
+"435,658",Kia,rio,24383,FD71TEU,"White 1.2 litre Manual Petrol Kia rio 1 at 24,383 miles",In Stock,On site (west malling),Petrol,TRUE,"10,500"
+"435,719",Kia,rio,31115,VA21WGZ,"Grey 1 litre Semi Auto Petrol Kia rio 2 at 31,115 miles",In Stock,Cannock,Petrol,TRUE,"11,900"
+"431,166",Volkswagen,id3,14603,MW72URX,"White 0 litre Automatic Electric Volkswagen id3 life at 14,603 miles",In Stock,Leeds,Electric,TRUE,"13,400"
+"432,165",Seat,ibiza,13365,YG73CLW,"Black 1 litre Semi Auto Petrol Seat ibiza tsi xcellence dsg at 13,365 miles",In Stock,Enfield,Petrol,TRUE,"16,300"
+"437,354",Volkswagen,golf,25363,CA22MVP,"Black 2 litre Semi Auto Petrol Volkswagen golf r at 25,363 miles",In Stock,Camberley,Petrol,TRUE,"30,700"
+"433,268",Volkswagen,golf,31145,WO73VOA,"White 2 litre Semi Auto Petrol Volkswagen golf r at 31,145 miles",In Stock,Camberley,Petrol,TRUE,"29,200"
+"433,268",Volkswagen,golf,31145,WO73VOA,"White 2 litre Semi Auto Petrol Volkswagen golf r at 31,145 miles",In Stock,Camberley,Petrol,TRUE,"29,200"
+"402,004",Volkswagen,golf,17910,YD73OCR,"White 2 litre Semi Auto Petrol Volkswagen golf r at 17,910 miles",In Stock,Wimbledon,Petrol,TRUE,"29,700"
+"422,216",Volkswagen,golf,18499,AK23BBV,"White 2 litre Semi Auto Petrol Volkswagen golf r at 18,499 miles",In Stock,Cannock,Petrol,TRUE,"31,200"
+"432,499",Volkswagen,golf,23142,RA23YPT,"White 2 litre Semi Auto Petrol Volkswagen golf r at 23,142 miles",In Stock,Wimbledon,Petrol,TRUE,"31,300"
+"422,482",Volkswagen,golf,11005,VK73ZYY,"White 2 litre Semi Auto Petrol Volkswagen golf r at 11,005 miles",In Stock,Camberley,Petrol,TRUE,"31,900"
+"432,169",Volkswagen,golf,19712,VE73TFN,"Black 2 litre Semi Auto Petrol Volkswagen golf r at 19,712 miles",In Stock,Cannock,Petrol,TRUE,"32,000"
+"437,672",Lexus,ux,17341,KM72UYA,"Black 0 litre Automatic Electric Lexus ux 300e at 17,341 miles",In Stock,Enfield,Electric,TRUE,"15,500"
+"429,017",Toyota,chr,39944,BG19VEO,"White 1.8 litre Cvt Hybrid Electric Toyota chr excel at 39,944 miles",In Stock,Leeds,Hybrid Electric,TRUE,"15,800"
+"436,955",Mercedes,cla,40484,WM67EKP,"Black 1.6 litre Semi Auto Petrol Mercedes cla amg line at 40,484 miles",In Stock,Norwich,Petrol,TRUE,"15,221"
+"432,747",Ford,fiesta,21173,SY70UNW,"Blue 1 litre Manual Petrol Ford fiesta active x edition at 21,173 miles",In Stock,Cannock,Petrol,TRUE,"11,900"
+"437,188",Bmw,2 series,25811,HK22UGG,"Black 2 litre Automatic Petrol Bmw 2 series m sport gran coupe at 25,811 miles",In Stock,Blue bell hill sales,Petrol,TRUE,"21,500"
+"438,021",Mercedes,e-class,30300,WH19XTG,"Grey 2 litre Automatic Diesel Mercedes e-class amg line at 30,300 miles",Reserved,Norwich,Diesel,TRUE,"19,700"
+"434,431",Hyundai,i10,28878,GY72NTM,"Black 1.2 litre Automatic Petrol Hyundai i10 mpi premium at 28,878 miles",In Stock,On site (west malling),Petrol,TRUE,"12,600"

templates/chat-bot.html

@@ -0,0 +1,480 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>RAG Admin Console</title>
+    <link href="https://fonts.googleapis.com/css2?family=DM+Mono:wght@400;500&family=Sora:wght@300;400;500;600&display=swap" rel="stylesheet">
+    <style>
+        :root {
+            --bg:        #0b0d11;
+            --surface:   #12151c;
+            --border:    rgba(255,255,255,0.07);
+            --border-hi: rgba(99,210,255,0.35);
+            --text:      #e8ecf4;
+            --muted:     #5a6070;
+            --accent:    #63d2ff;
+            --accent-dk: #3ab8e8;
+            --success:   #3dffa0;
+            --danger:    #ff5c72;
+            --warn:      #ffc94a;
+            --glow:      0 0 24px rgba(99,210,255,0.12);
+        }
+
+        *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+
+        body {
+            font-family: 'Sora', sans-serif;
+            background: var(--bg);
+            color: var(--text);
+            min-height: 100vh;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            padding: 24px;
+            background-image:
+                linear-gradient(rgba(99,210,255,0.025) 1px, transparent 1px),
+                linear-gradient(90deg, rgba(99,210,255,0.025) 1px, transparent 1px);
+            background-size: 48px 48px;
+        }
+
+        /* Card */
+        .card {
+            background: var(--surface);
+            border: 1px solid var(--border);
+            border-radius: 16px;
+            width: 100%;
+            max-width: 420px;
+            overflow: hidden;
+            box-shadow: 0 32px 64px rgba(0,0,0,0.5), var(--glow);
+            animation: rise 0.5s cubic-bezier(0.22,1,0.36,1) both;
+        }
+        @keyframes rise {
+            from { opacity:0; transform: translateY(20px); }
+            to   { opacity:1; transform: translateY(0); }
+        }
+
+        /* Header */
+        .card-header {
+            padding: 26px 30px 22px;
+            border-bottom: 1px solid var(--border);
+            display: flex;
+            align-items: center;
+            gap: 14px;
+        }
+        .logo-mark {
+            width: 38px; height: 38px;
+            border-radius: 10px;
+            background: linear-gradient(135deg, #1a3a50, #0d2236);
+            border: 1px solid var(--border-hi);
+            display: flex; align-items: center; justify-content: center;
+            flex-shrink: 0;
+        }
+        .logo-mark svg { width: 20px; height: 20px; }
+        .header-text h1 {
+            font-size: 15px;
+            font-weight: 600;
+            letter-spacing: 0.01em;
+            color: var(--text);
+        }
+        .header-text p {
+            font-size: 12px;
+            color: var(--muted);
+            margin-top: 2px;
+            font-weight: 300;
+        }
+
+        /* Status Row */
+        .status-row {
+            padding: 14px 30px;
+            border-bottom: 1px solid var(--border);
+            display: flex;
+            align-items: center;
+            justify-content: space-between;
+        }
+        .status-label { font-size: 11px; color: var(--muted); font-family: 'DM Mono', monospace; letter-spacing: 0.06em; text-transform: uppercase; }
+        .pill {
+            display: inline-flex; align-items: center; gap: 7px;
+            padding: 5px 11px;
+            border-radius: 100px;
+            font-size: 11px; font-weight: 500;
+            font-family: 'DM Mono', monospace;
+            letter-spacing: 0.04em;
+            background: rgba(61,255,160,0.08);
+            color: var(--success);
+            border: 1px solid rgba(61,255,160,0.2);
+            transition: all 0.3s;
+        }
+        .pill.error {
+            background: rgba(255,92,114,0.08);
+            color: var(--danger);
+            border-color: rgba(255,92,114,0.2);
+        }
+        .pill.loading {
+            background: rgba(255,201,74,0.08);
+            color: var(--warn);
+            border-color: rgba(255,201,74,0.2);
+        }
+        .pill-dot { width: 6px; height: 6px; border-radius: 50%; background: currentColor; }
+        .pill-dot.pulse { animation: pulse 1.8s ease-in-out infinite; }
+        @keyframes pulse {
+            0%,100% { opacity:1; transform: scale(1); }
+            50%      { opacity:0.4; transform: scale(0.7); }
+        }
+
+        /* Body */
+        .card-body { padding: 26px 30px 30px; }
+
+        /* Fields */
+        .field { margin-bottom: 14px; }
+        .field label {
+            display: block;
+            font-size: 10px; font-weight: 500;
+            color: var(--muted);
+            margin-bottom: 7px;
+            letter-spacing: 0.09em;
+            text-transform: uppercase;
+            font-family: 'DM Mono', monospace;
+        }
+        .field input {
+            width: 100%;
+            padding: 10px 14px;
+            background: rgba(255,255,255,0.04);
+            border: 1px solid var(--border);
+            border-radius: 8px;
+            color: var(--text);
+            font-size: 14px;
+            font-family: 'Sora', sans-serif;
+            transition: border-color 0.2s, box-shadow 0.2s;
+            outline: none;
+        }
+        .field input::placeholder { color: var(--muted); opacity: 0.55; }
+        .field input:focus {
+            border-color: var(--border-hi);
+            box-shadow: 0 0 0 3px rgba(99,210,255,0.07);
+        }
+
+        /* Buttons */
+        .btn {
+            width: 100%;
+            padding: 11px 16px;
+            border: none;
+            border-radius: 9px;
+            font-family: 'Sora', sans-serif;
+            font-size: 13px; font-weight: 500;
+            cursor: pointer;
+            transition: all 0.2s;
+            display: flex; align-items: center; justify-content: center; gap: 8px;
+            letter-spacing: 0.02em;
+            margin-top: 8px;
+        }
+        .btn:active { transform: scale(0.98); }
+        .btn svg { flex-shrink: 0; }
+
+        .btn-primary { background: var(--accent); color: #06111a; }
+        .btn-primary:hover { background: var(--accent-dk); box-shadow: 0 0 20px rgba(99,210,255,0.22); }
+
+        .btn-ghost {
+            background: rgba(255,255,255,0.04);
+            color: var(--text);
+            border: 1px solid var(--border);
+        }
+        .btn-ghost:hover { background: rgba(255,255,255,0.07); border-color: rgba(255,255,255,0.12); }
+
+        .btn-danger-soft {
+            background: rgba(255,92,114,0.07);
+            color: var(--danger);
+            border: 1px solid rgba(255,92,114,0.18);
+        }
+        .btn-danger-soft:hover { background: rgba(255,92,114,0.13); }
+
+        /* Section labels */
+        .section-label {
+            font-size: 10px; font-weight: 500;
+            color: var(--muted);
+            text-transform: uppercase;
+            letter-spacing: 0.1em;
+            font-family: 'DM Mono', monospace;
+            margin-top: 20px;
+            margin-bottom: 9px;
+        }
+        .section-label:first-child { margin-top: 0; }
+
+        /* Log */
+        .log-wrap { display: none; margin-top: 16px; }
+        .log-header {
+            display: flex; align-items: center; justify-content: space-between;
+            margin-bottom: 6px;
+        }
+        .log-header span {
+            font-size: 10px;
+            font-family: 'DM Mono', monospace;
+            color: var(--muted);
+            text-transform: uppercase;
+            letter-spacing: 0.08em;
+        }
+        #log-status { transition: color 0.3s; }
+        .log-box {
+            background: #080a0e;
+            border: 1px solid var(--border);
+            border-radius: 8px;
+            padding: 14px;
+            font-family: 'DM Mono', monospace;
+            font-size: 11px;
+            line-height: 1.75;
+            color: #7ee8a2;
+            height: 130px;
+            overflow-y: auto;
+            white-space: pre-wrap;
+            word-break: break-all;
+        }
+        .log-box::-webkit-scrollbar { width: 4px; }
+        .log-box::-webkit-scrollbar-thumb { background: #2a2f3a; border-radius: 2px; }
+        .log-error { color: var(--danger); }
+
+        /* Logout */
+        .logout-row {
+            margin-top: 20px;
+            padding-top: 18px;
+            border-top: 1px solid var(--border);
+            display: flex;
+            align-items: center;
+            justify-content: space-between;
+        }
+        .logout-row span { font-size: 12px; color: var(--muted); }
+        .logout-btn {
+            font-family: 'DM Mono', monospace;
+            font-size: 11px;
+            background: none;
+            border: 1px solid var(--border);
+            color: var(--muted);
+            padding: 6px 13px;
+            border-radius: 6px;
+            cursor: pointer;
+            transition: all 0.2s;
+            letter-spacing: 0.05em;
+        }
+        .logout-btn:hover { color: var(--danger); border-color: rgba(255,92,114,0.3); background: rgba(255,92,114,0.05); }
+
+        input[type="number"]::-webkit-inner-spin-button,
+        input[type="number"]::-webkit-outer-spin-button { opacity: 0.4; }
+    </style>
+</head>
+<body>
+
+<div class="card">
+
+    <!-- Header -->
+    <div class="card-header">
+        <div class="logo-mark">
+            <svg viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+                <path d="M3 5h14M3 10h9M3 15h5" stroke="#63d2ff" stroke-width="1.8" stroke-linecap="round"/>
+                <circle cx="16" cy="14" r="3" stroke="#63d2ff" stroke-width="1.5"/>
+                <path d="M18.5 16.5L20 18" stroke="#63d2ff" stroke-width="1.5" stroke-linecap="round"/>
+            </svg>
+        </div>
+        <div class="header-text">
+            <h1>RAG Admin Console</h1>
+            <p>Knowledge base management</p>
+        </div>
+    </div>
+
+    <!-- Status -->
+    <div class="status-row">
+        <span class="status-label">System Status</span>
+        <span id="status-pill" class="pill loading">
+            <span class="pill-dot pulse"></span>
+            <span id="status-text">Checking...</span>
+        </span>
+    </div>
+
+    <!-- Body -->
+    <div class="card-body">
+
+        <!-- Login -->
+        <div id="login-section">
+            <div class="field">
+                <label>Username</label>
+                <input type="text" id="username" placeholder="admin@example.com" autocomplete="username">
+            </div>
+            <div class="field">
+                <label>Password</label>
+                <input type="password" id="password" placeholder="••••••••" autocomplete="current-password"
+                    onkeydown="if(event.key==='Enter') login()">
+            </div>
+            <button class="btn btn-primary" onclick="login()">
+                <svg width="14" height="14" viewBox="0 0 16 16" fill="none">
+                    <path d="M6 2H3a1 1 0 00-1 1v10a1 1 0 001 1h3M10 5l3 3-3 3M13 8H6"
+                        stroke="currentColor" stroke-width="1.6" stroke-linecap="round" stroke-linejoin="round"/>
+                </svg>
+                Sign In
+            </button>
+        </div>
+
+        <!-- Admin Panel -->
+        <div id="admin-section" style="display:none;">
+
+            <p class="section-label">URL Sources</p>
+            <button class="btn btn-ghost" onclick="performAction('/admin/fetch_rentry')">
+                <svg width="14" height="14" viewBox="0 0 16 16" fill="none">
+                    <path d="M13 8A5 5 0 112.5 6.5" stroke="currentColor" stroke-width="1.6" stroke-linecap="round"/>
+                    <path d="M2 3.5L2.5 6.5 5.5 6" stroke="currentColor" stroke-width="1.6" stroke-linecap="round" stroke-linejoin="round"/>
+                </svg>
+                Fetch &amp; Update from URL
+            </button>
+
+            <p class="section-label" style="margin-top:20px;">Local Index</p>
+            <div class="field">
+                <label>Max files (incremental)</label>
+                <input type="number" id="max-files" value="50" min="1" max="500">
+            </div>
+            <button class="btn btn-ghost" onclick="performAction('/admin/update_faiss_index')">
+                <svg width="14" height="14" viewBox="0 0 16 16" fill="none">
+                    <path d="M8 2v6m0 0l-2.5-2.5M8 8l2.5-2.5" stroke="currentColor" stroke-width="1.6" stroke-linecap="round" stroke-linejoin="round"/>
+                    <path d="M2 11v1a2 2 0 002 2h8a2 2 0 002-2v-1" stroke="currentColor" stroke-width="1.6" stroke-linecap="round"/>
+                </svg>
+                Update Index — New Files Only
+            </button>
+            <button class="btn btn-danger-soft" onclick="confirmRebuild()">
+                <svg width="14" height="14" viewBox="0 0 16 16" fill="none">
+                    <path d="M3 8a5 5 0 0110 0" stroke="currentColor" stroke-width="1.6" stroke-linecap="round"/>
+                    <path d="M13 5.5V8h-2.5M3 10.5V8h2.5" stroke="currentColor" stroke-width="1.6" stroke-linecap="round" stroke-linejoin="round"/>
+                </svg>
+                Rebuild Full Index
+            </button>
+
+            <!-- Log -->
+            <div class="log-wrap" id="log-wrap">
+                <div class="log-header">
+                    <span>Operation Log</span>
+                    <span id="log-status" style="color:var(--warn)">running…</span>
+                </div>
+                <div class="log-box" id="log-box"></div>
+            </div>
+
+            <!-- Logout -->
+            <div class="logout-row">
+                <span id="signed-in-label">Signed in as admin</span>
+                <button class="logout-btn" onclick="logout()">Sign out</button>
+            </div>
+        </div>
+
+    </div>
+</div>
+
+<script src="https://cdn.jsdelivr.net/npm/axios/dist/axios.min.js"></script>
+<script>
+    let authHeader = null;
+
+    window.onload = async () => {
+        await checkStatus();
+        const savedUser = localStorage.getItem('rag_admin_user');
+        const savedPass = localStorage.getItem('rag_admin_pass');
+        if (savedUser && savedPass) {
+            document.getElementById('username').value = savedUser;
+            document.getElementById('password').value = savedPass;
+            await login(true);
+        }
+    };
+
+    async function checkStatus() {
+        const pill = document.getElementById('status-pill');
+        const text = document.getElementById('status-text');
+        const dot  = pill.querySelector('.pill-dot');
+        try {
+            const res = await axios.get('/status');
+            dot.classList.remove('pulse');
+            if (res.data.rag_initialized) {
+                pill.className = 'pill';
+                text.textContent = 'Online';
+            } else {
+                pill.className = 'pill loading';
+                dot.classList.add('pulse');
+                text.textContent = 'Not Initialized';
+            }
+        } catch {
+            dot.classList.remove('pulse');
+            pill.className = 'pill error';
+            text.textContent = 'Offline';
+        }
+    }
+
+    async function login(isSilent = false) {
+        const u = document.getElementById('username').value.trim();
+        const p = document.getElementById('password').value;
+        try {
+            await axios.post('/admin/login', {}, { auth: { username: u, password: p } });
+            authHeader = { username: u, password: p };
+            localStorage.setItem('rag_admin_user', u);
+            localStorage.setItem('rag_admin_pass', p);
+            document.getElementById('login-section').style.display  = 'none';
+            document.getElementById('admin-section').style.display  = 'block';
+            document.getElementById('signed-in-label').textContent  = `Signed in as ${u}`;
+            document.getElementById('password').value = '';
+        } catch {
+            if (!isSilent) flashError();
+            logout();
+        }
+    }
+
+    function logout() {
+        authHeader = null;
+        localStorage.removeItem('rag_admin_user');
+        localStorage.removeItem('rag_admin_pass');
+        document.getElementById('username').value = '';
+        document.getElementById('password').value = '';
+        document.getElementById('login-section').style.display = 'block';
+        document.getElementById('admin-section').style.display = 'none';
+        document.getElementById('log-wrap').style.display = 'none';
+    }
+
+    function flashError() {
+        const el = document.getElementById('password');
+        el.style.borderColor = 'var(--danger)';
+        el.style.boxShadow   = '0 0 0 3px rgba(255,92,114,0.1)';
+        setTimeout(() => { el.style.borderColor = ''; el.style.boxShadow = ''; }, 2500);
+    }
+
+    function confirmRebuild() {
+        if (confirm('Rebuild the full index? This will re-process all documents and may take several minutes.')) {
+            performAction('/admin/rebuild_index');
+        }
+    }
+
+    async function performAction(url) {
+        const wrap   = document.getElementById('log-wrap');
+        const logBox = document.getElementById('log-box');
+        const logSt  = document.getElementById('log-status');
+
+        wrap.style.display = 'block';
+        logBox.textContent = 'Processing — this may take a minute…\n';
+        logSt.textContent  = 'running…';
+        logSt.style.color  = 'var(--warn)';
+
+        const payload = {};
+        if (url.includes('update')) {
+            payload.max_new_files = parseInt(document.getElementById('max-files').value, 10) || 50;
+        }
+
+        try {
+            const res = await axios.post(url, payload, { auth: authHeader });
+            logBox.textContent += '\n[SUCCESS]\n' + JSON.stringify(res.data, null, 2);
+            logSt.textContent   = 'done';
+            logSt.style.color   = 'var(--success)';
+            checkStatus();
+        } catch (e) {
+            const msg = e.response ? JSON.stringify(e.response.data, null, 2) : e.message;
+            logBox.innerHTML += `\n<span class="log-error">[ERROR]\n${msg}</span>`;
+            logSt.textContent  = 'failed';
+            logSt.style.color  = 'var(--danger)';
+            if (e.response?.status === 401) {
+                alert('Session expired. Please sign in again.');
+                logout();
+            }
+        } finally {
+            logBox.scrollTop = logBox.scrollHeight;
+        }
+    }
+</script>
+</body>
+</html>

utils.py

@@ -0,0 +1,150 @@
+import os
+import logging
+import re
+import shutil
+import tempfile
+import zipfile
+import requests
+from typing import Optional
+from bs4 import BeautifulSoup
+
+import gdown
+from pypdf import PdfReader
+import docx as python_docx
+
+logger = logging.getLogger(__name__)
+
+def extract_text_from_file(file_path: str, file_type: str) -> Optional[str]:
+    logger.info(f"[TEXT_EXTRACTION] Starting extraction from {file_type.upper()} file: {file_path}")
+    text_content = None
+    try:
+        if file_type == 'pdf':
+            reader = PdfReader(file_path)
+            text_content = "".join(page.extract_text() + "\n" for page in reader.pages if page.extract_text())
+        elif file_type == 'docx':
+            doc = python_docx.Document(file_path)
+            text_content = "\n".join(para.text for para in doc.paragraphs if para.text)
+        elif file_type == 'txt':
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                text_content = f.read()
+        else:
+            logger.warning(f"[TEXT_EXTRACTION] Unsupported file type: {file_type}")
+            return None
+
+        if not text_content or not text_content.strip():
+            logger.warning(f"[TEXT_EXTRACTION] No text content extracted from {file_path}")
+            return None
+        
+        return text_content.strip()
+    except Exception as e:
+        logger.error(f"[TEXT_EXTRACTION] Error extracting text: {e}", exc_info=True)
+        return None
+
+FAISS_RAG_SUPPORTED_EXTENSIONS = {
+    'pdf': lambda path: extract_text_from_file(path, 'pdf'),
+    'docx': lambda path: extract_text_from_file(path, 'docx'),
+    'txt': lambda path: extract_text_from_file(path, 'txt'),
+    'csv': lambda path: "CSV_HANDLED_NATIVELY" # Bypassed directly in components to allow row-by-row chunks
+}
+
+def fetch_and_clean_url(url: str, output_txt_path: str) -> bool:
+    """Fetches HTML from a URL, cleans it, and saves it as pure text."""
+    try:
+        logger.info(f"[URL_FETCH] Fetching and cleaning data from: {url}")
+        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}
+        response = requests.get(url, headers=headers, timeout=15)
+        response.raise_for_status()
+        
+        soup = BeautifulSoup(response.text, 'html.parser')
+        
+        # Remove non-content tags completely
+        for tag in soup(["script", "style", "nav", "footer", "header", "noscript"]):
+            tag.decompose()
+        
+        # Attempt to find specific content divs if they exist (generalizes rentry and others)
+        entry = soup.find('div', class_='entry-text')
+        if entry:
+            text = entry.get_text(separator='\n', strip=True)
+        else:
+            # Fallback for other sites or layout changes
+            text = soup.body.get_text(separator='\n', strip=True) if soup.body else soup.get_text(separator='\n', strip=True)
+        
+        # Clean up excessive blank lines
+        text = re.sub(r'\n\s*\n', '\n\n', text)
+        
+        # Ensure directory exists
+        os.makedirs(os.path.dirname(output_txt_path), exist_ok=True)
+        
+        with open(output_txt_path, 'w', encoding='utf-8') as f:
+            f.write(text)
+            
+        logger.info(f"[URL_FETCH] Success! Text saved to {output_txt_path}")
+        return True
+    except Exception as e:
+        logger.error(f"[URL_FETCH] Error fetching/cleaning URL {url}: {e}", exc_info=True)
+        return False
+
+def get_id_from_gdrive_input(url_or_id: str) -> Optional[str]:
+    if not url_or_id: return None
+    match_folder = re.search(r"/folders/([a-zA-Z0-9_-]+)", url_or_id)
+    if match_folder: return match_folder.group(1)
+    match_file_d = re.search(r"/d/([a-zA-Z0-9_-]+)", url_or_id)
+    if match_file_d: return match_file_d.group(1)
+    match_uc = re.search(r"id=([a-zA-Z0-9_-]+)", url_or_id)
+    if match_uc: return match_uc.group(1)
+    return url_or_id if len(url_or_id) > 10 else None
+
+def download_gdrive_file(file_id_or_url: str, target_path: str) -> bool:
+    logger.info(f"[GDRIVE_SINGLE] Downloading file. Input: {file_id_or_url}")
+    file_id = get_id_from_gdrive_input(file_id_or_url)
+    if not file_id: return False
+
+    try:
+        os.makedirs(os.path.dirname(target_path), exist_ok=True)
+        gdown.download(id=file_id, output=target_path, quiet=False, fuzzy=True)
+        
+        if os.path.exists(target_path) and os.path.getsize(target_path) > 0:
+            return True
+        return False
+    except Exception as e:
+        logger.error(f"[GDRIVE_SINGLE] Error: {e}", exc_info=True)
+        return False
+
+def download_and_unzip_gdrive_folder(folder_id_or_url: str, target_dir_for_contents: str) -> bool:
+    logger.info(f"[GDRIVE] Downloading folder. Input: {folder_id_or_url}")
+    folder_id = get_id_from_gdrive_input(folder_id_or_url)
+    if not folder_id: return False
+
+    temp_dir = tempfile.mkdtemp()
+    try:
+        gdown.download_folder(id=folder_id, output=temp_dir, quiet=False, use_cookies=False)
+        if not os.path.exists(target_dir_for_contents):
+            os.makedirs(target_dir_for_contents)
+
+        src_root = temp_dir
+        if len(os.listdir(temp_dir)) == 1 and os.path.isdir(os.path.join(temp_dir, os.listdir(temp_dir)[0])):
+            src_root = os.path.join(temp_dir, os.listdir(temp_dir)[0])
+
+        for item in os.listdir(src_root):
+            shutil.move(os.path.join(src_root, item), os.path.join(target_dir_for_contents, item))
+        return True
+    except Exception as e:
+        logger.error(f"[GDRIVE] Error: {e}", exc_info=True)
+        return False
+    finally:
+        shutil.rmtree(temp_dir, ignore_errors=True)
+
+def download_and_unzip_gdrive_file(file_id_or_url: str, target_extraction_dir: str) -> bool:
+    logger.info(f"[GDRIVE_ZIP] Downloading ZIP. Input: {file_id_or_url}")
+    file_id = get_id_from_gdrive_input(file_id_or_url)
+    if not file_id: return False
+
+    temp_zip = os.path.join(tempfile.gettempdir(), "temp_download.zip")
+    try:
+        gdown.download(id=file_id, output=temp_zip, quiet=False)
+        with zipfile.ZipFile(temp_zip, 'r') as zip_ref:
+            zip_ref.extractall(target_extraction_dir)
+        return True
+    except Exception as e:
+        logger.error(f"[GDRIVE_ZIP] Error: {e}", exc_info=True)
+        return False