Dynamic source update endpoint

app.py

@@ -3,6 +3,7 @@ from flask_cors import CORS
 import os
 import logging
 import functools
+from datetime import datetime
 import pandas as pd
 import threading
 import time
@@ -23,7 +24,7 @@ from config import (
     EXTERNAL_URL, URL_UPDATE_PERIOD_MINUTES, URL_FETCH_ENABLED,
     RAG_CSV_MAX_RESULTS, RAG_CSV_CONFIDENCE_THRESHOLD
 )
-from utils import download_and_unzip_gdrive_file, download_gdrive_file, fetch_and_clean_url
+from utils import download_and_unzip_gdrive_file, download_gdrive_file, fetch_and_clean_url, FAISS_RAG_SUPPORTED_EXTENSIONS
 
 # Logging Setup
 logging.basicConfig(level=logging.INFO)
@@ -38,6 +39,9 @@ rag_system = None
 user_df = None
 _APP_BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 
+# Source file constants
+SOURCES_ARCHIVE_DIR = os.path.join(_APP_BASE_DIR, 'sources_archive')
+
 # --- Helper: Load Users ---
 def load_users_from_csv():
     global user_df
@@ -164,6 +168,20 @@ def url_periodic_loop():
         logger.info(f"[URL_UPDATE] Triggering scheduled periodic update...")
         trigger_url_update()
 
+def rebuild_rag_system():
+    global rag_system
+
+    if URL_FETCH_ENABLED and EXTERNAL_URL:
+        result = trigger_url_update()
+        if "error" in result:
+            return result
+        return {"status": "Index rebuilt successfully using combined local & URL sources"}
+
+    rag_system = initialize_and_get_rag_system(force_rebuild=True)
+    if rag_system is None:
+        return {"error": "Failed to rebuild RAG system"}
+    return {"status": "Index rebuilt successfully"}
+
 # --- Startup Logic ---
 def run_startup_tasks():
     global rag_system
@@ -304,17 +322,72 @@ def update_faiss_index():
 @app.route('/admin/rebuild_index', methods=['POST'])
 @require_admin_auth
 def rebuild_index():
-    global rag_system
     try:
-        if URL_FETCH_ENABLED and EXTERNAL_URL:
-            result = trigger_url_update()
-            if "error" in result:
-                return jsonify(result), 500
-            return jsonify({"status": "Index rebuilt successfully using combined local & URL sources"}), 200
-        else:
-            rag_system = initialize_and_get_rag_system(force_rebuild=True)
-            return jsonify({"status": "Index rebuilt successfully"}), 200
+        result = rebuild_rag_system()
+        if "error" in result:
+            return jsonify(result), 500
+        return jsonify(result), 200
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/admin/upload/<path:filename>', methods=['POST'])
+@require_admin_auth
+def upload_vehicle_csv(filename):
+    uploaded_file = request.files.get('file') or request.files.get('upload') or request.files.get('source_file')
+    if not uploaded_file:
+        return jsonify({"error": "No CSV file uploaded. Use multipart/form-data with field 'file'."}), 400
+
+    safe_filename = os.path.basename((filename or '').strip())
+    if not safe_filename or safe_filename != filename.strip():
+        return jsonify({"error": "Invalid filename."}), 400
+
+    file_ext = safe_filename.rsplit('.', 1)[-1].lower() if '.' in safe_filename else ''
+    if file_ext not in FAISS_RAG_SUPPORTED_EXTENSIONS:
+        supported_types = sorted(FAISS_RAG_SUPPORTED_EXTENSIONS.keys())
+        return jsonify({
+            "error": "Unsupported file type.",
+            "supported_extensions": supported_types
+        }), 400
+
+    os.makedirs(RAG_SOURCES_DIR, exist_ok=True)
+    os.makedirs(SOURCES_ARCHIVE_DIR, exist_ok=True)
+
+    target_path = os.path.join(RAG_SOURCES_DIR, safe_filename)
+    temp_upload_path = os.path.join(RAG_SOURCES_DIR, f".{safe_filename}.uploading")
+    archive_path = None
+
+    try:
+        uploaded_file.save(temp_upload_path)
+
+        if os.path.getsize(temp_upload_path) == 0:
+            os.remove(temp_upload_path)
+            return jsonify({"error": "Uploaded file is empty."}), 400
+
+        if os.path.exists(target_path):
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+            archive_dir = os.path.join(SOURCES_ARCHIVE_DIR, timestamp)
+            os.makedirs(archive_dir, exist_ok=True)
+            archive_path = os.path.join(archive_dir, safe_filename)
+            shutil.move(target_path, archive_path)
+
+        os.replace(temp_upload_path, target_path)
+
+        result = rebuild_rag_system()
+        if "error" in result:
+            return jsonify(result), 500
+
+        response_payload = {
+            "status": "success",
+            "message": f"{safe_filename} uploaded and index rebuilt successfully.",
+            "filename": safe_filename,
+            "saved_to": target_path,
+            "archived_previous_to": archive_path
+        }
+        return jsonify(response_payload), 200
     except Exception as e:
+        logger.error(f"[SOURCE_UPLOAD] Error updating {safe_filename}: {e}", exc_info=True)
+        if os.path.exists(temp_upload_path):
+            os.remove(temp_upload_path)
         return jsonify({"error": str(e)}), 500
 
 # Retained specific endpoint name to ensure the frontend doesn't break

postman.json

@@ -125,6 +125,51 @@
 			},
 			"response": []
 		},
+		{
+			"name": "Admin - Upload Source File",
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "1234",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "admin",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "formdata",
+					"formdata": [
+						{
+							"key": "file",
+							"type": "file",
+							"src": ""
+						}
+					]
+				},
+				"url": {
+					"raw": "{{base_url}}/admin/upload/vehicle.csv",
+					"host": [
+						"{{base_url}}"
+					],
+					"path": [
+						"admin",
+						"upload",
+						"vehicle.csv"
+					]
+				},
+				"description": "Uploads a replacement source file using the filename in the URL, archives the previous file with the same name, and rebuilds the index."
+			},
+			"response": []
+		},
 		{
 			"name": "Public - Status",
 			"request": {