Replace gen_llm with nvidia_llm and add frontend test

Dockerfile

@@ -50,7 +50,7 @@ RUN rm -f requirements_gpu.txt Dockerfile_bak
 RUN chmod +x start.sh
 
 # ── Environment configuration ─────────────────────────────────────────────────
-# HF_MODE=1 activates low-resource path in config.py, gen_llm.py, embed_llm.py
+# HF_MODE=1 activates low-resource path in config.py, nvidia_llm.py, embed_llm.py
 ENV HF_MODE=1
 ENV ADMIN_MODE=1
 ENV PORT=7860

README.md

@@ -167,7 +167,7 @@ export HF_PRIVATE_TOKEN=$(secret-tool lookup api huggingface)
 
 **Terminal 1 - LLM Generation Server (port 8002):**
 ```bash
-python agents/gen_llm.py
+python agents/nvidia_llm.py
 # Expected output:
 # * Running on http://127.0.0.1:8002
 ```
@@ -241,7 +241,7 @@ healthexpert/
 │   ├── crew.py                              # Crew orchestration
 │   ├── llm.py                               # LLM integration
 │   ├── tools.py                             # Agent tools
-│   ├── gen_llm.py                           # LLM generation server (port 8002)
+│   ├── nvidia_llm.py                        # LLM generation server (port 8002)
 │   └── embed_llm.py                         # Embedding server (port 8003)
 │
 ├── pipeline/                                # Data processing

agents/gen_llm.py

@@ -1,441 +0,0 @@
-# gen_llm.py
-# General-purpose LLM Generation Server — port 8002
-#
-# Modes:
-#   CPU  : llama.cpp on CPU (default; works without GPU)
-#   GPU  : llama.cpp with GPU layers (enabled per-request via use_gpu=true)
-#
-# Multi-user concurrency model:
-#   A single inference worker thread owns all model.create_chat_completion calls.
-#   HTTP requests enqueue a job (data + result holder + done_event) and block
-#   until their result is ready.  If the queue is full or the request times out
-#   the endpoint returns 503 {"error": "Server busy — try again shortly."} so
-#   the caller can retry without hanging indefinitely.
-#
-# Exposes: POST /v1/completions  (OpenAI-compatible)
-#          POST /v1/kv_cache     (no-op — llama.cpp manages natively)
-#          GET  /health          (includes queue_depth for the UI busy badge)
-#
-# GPU/CPU control:
-#   - Pass use_gpu=true in the request body to run on GPU (if available).
-#   - Pass cpu_threads=N in the request body to override thread count (CPU mode).
-#   - If GPU is unavailable, use_gpu is silently ignored.
-#
-# Run: python agents/gen_llm.py
-#      → http://127.0.0.1:8002
-
-from __future__ import annotations
-
-import os
-import warnings
-warnings.filterwarnings("ignore")
-os.environ["PYTHONWARNINGS"]    = "ignore"
-os.environ["LLAMA_NUMA"] = "1"           # Enable NUMA optimizations
-
-import logging
-import threading
-import queue as _queue_module
-import time
-from flask import Flask, request, jsonify
-
-# ── Logging ───────────────────────────────────────────────────────────────────
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s [%(name)s] %(levelname)s %(message)s",
-    datefmt="%Y-%m-%d %H:%M:%S",
-)
-log = logging.getLogger("gen_llm")
-logging.getLogger("werkzeug").setLevel(logging.ERROR)
-logging.getLogger("httpx").setLevel(logging.WARNING)
-
-# ── Config ────────────────────────────────────────────────────────────────────
-MODEL_REPO   = os.getenv("GEN_MODEL_ID", "Jackrong/Qwen3.5-2B-Claude-4.6-Opus-Reasoning-Distilled-GGUF")
-MODEL_FILE   = os.getenv("GEN_MODEL_FILENAME", "Qwen3.5-2B.Q4_K_M.gguf")
-HOST         = os.getenv("GEN_HOST",  "127.0.0.1")
-PORT         = int(os.getenv("GEN_PORT", "8002"))
-
-# Default CPU thread count (overridable per-request)
-DEFAULT_CPU_THREADS = int(os.getenv("GEN_CPU_THREADS", "2"))
-
-# Maximum number of requests that can wait in the inference queue.
-_QUEUE_MAX_SIZE    = int(os.getenv("GEN_QUEUE_MAX", "8"))
-
-# Per-request timeout (seconds).  Matches config.py LLM_TIMEOUT default.
-_REQUEST_TIMEOUT_S = int(os.getenv("LLM_TIMEOUT", "600"))
-
-# ── Device / GPU Detection ────────────────────────────────────────────────────
-_gpu_available = False
-_gpu_id = "cpu"
-
-try:
-    import torch
-    if torch.cuda.is_available():
-        _cuda_idx = int(os.getenv("GEN_CUDA_DEVICE", "0"))
-        _gpu_id = f"cuda:{_cuda_idx}"
-        _gpu_available = True
-        log.info("GPU DETECTED — %s available for on-demand inference", _gpu_id)
-    else:
-        log.info("No CUDA GPU detected — CPU-only inference available")
-except ImportError:
-    log.info("torch not available — GPU detection skipped, CPU-only mode")
-
-# Do NOT set CUDA_VISIBLE_DEVICES="" here — we need GPU access to be possible.
-# GPU layers are set per-model-instance (see _load_model below).
-
-log.info("━" * 60)
-log.info("gen_llm starting — gpu_available=%s  model=%s (%s)", _gpu_available, MODEL_REPO, MODEL_FILE)
-log.info("Default CPU threads=%d  Queue: max_size=%d  request_timeout=%ds",
-         DEFAULT_CPU_THREADS, _QUEUE_MAX_SIZE, _REQUEST_TIMEOUT_S)
-log.info("━" * 60)
-
-
-# ── Model Loading ─────────────────────────────────────────────────────────────
-# We maintain up to two model instances: CPU and (optionally) GPU.
-# This avoids full model reload on every request while allowing GPU offload.
-
-_model_lock  = threading.Lock()
-_models: dict[str, object] = {}   # key: "cpu" or "gpu"
-_model_ready = threading.Event()
-_model_path  = None
-
-
-def _load_model(use_gpu: bool = False):
-    """Load and cache a model instance. Returns the cached instance if already loaded."""
-    mode_key = "gpu" if (use_gpu and _gpu_available) else "cpu"
-    
-    with _model_lock:
-        if mode_key in _models:
-            return _models[mode_key]
-        
-        global _model_path
-        if _model_path is None:
-            log.info("Downloading/Locating model from Hub: %s/%s", MODEL_REPO, MODEL_FILE)
-            from huggingface_hub import hf_hub_download
-            _model_path = hf_hub_download(repo_id=MODEL_REPO, filename=MODEL_FILE)
-        
-        n_gpu_layers = -1 if (use_gpu and _gpu_available) else 0
-        cpu_threads  = DEFAULT_CPU_THREADS
-        
-        log.info("Loading GGUF model — mode=%s  n_gpu_layers=%d  threads=%d", 
-                 mode_key, n_gpu_layers, cpu_threads)
-        
-        from llama_cpp import Llama
-        m = Llama(
-            model_path=_model_path,
-            n_ctx=8192,
-            n_batch=512,
-            n_threads=cpu_threads,
-            n_gpu_layers=n_gpu_layers,
-            use_mmap=True,
-            use_mlock=True,
-            numa=True,
-            flash_attn=True,
-            verbose=False,
-        )
-        _models[mode_key] = m
-        log.info("Model instance [%s] ready!", mode_key)
-        return m
-
-
-# Pre-load CPU model at startup (always available)
-try:
-    _load_model(use_gpu=False)
-    _model_ready.set()
-    log.info("CPU model pre-loaded and ready.")
-except Exception as e:
-    log.error("Failed to load CPU model: %s", e)
-    raise
-
-
-# ── Flask app ─────────────────────────────────────────────────────────────────
-app = Flask(__name__)
-
-# ── Inference Queue (multi-user serialization) ─────────────────────────────────
-_inference_queue: _queue_module.Queue = _queue_module.Queue(maxsize=_QUEUE_MAX_SIZE)
-
-
-def _run_inference(data: dict) -> dict:
-    """Execute one completion request.  Called only from the inference worker thread."""
-    raw_prompt = data.get("prompt", "")
-    if not raw_prompt:
-        return {"error": "Field 'prompt' is required."}
-
-    prompts = raw_prompt if isinstance(raw_prompt, list) and (len(raw_prompt) == 0 or not isinstance(raw_prompt[0], dict)) else [raw_prompt]
-
-    use_gpu     = bool(data.get("use_gpu", False))
-    cpu_threads = int(data.get("cpu_threads", DEFAULT_CPU_THREADS))
-
-    # Select model instance (GPU if requested and available, else CPU)
-    model = _load_model(use_gpu=use_gpu)
-    active_device = _gpu_id if (use_gpu and _gpu_available) else "cpu"
-
-    # Apply cpu_threads override if running on CPU and different from default
-    # Note: llama_cpp doesn't support live thread changes; we log the intent.
-    if not (use_gpu and _gpu_available) and cpu_threads != DEFAULT_CPU_THREADS:
-        log.info("cpu_threads=%d requested (model loaded with %d — static per-instance)", 
-                 cpu_threads, DEFAULT_CPU_THREADS)
-
-    _default_max = 1024
-    max_new_tokens  = int(  data.get("max_tokens",        _default_max))
-    
-    # Reasoning models need large token budgets for internal monologue.
-    # Enforce a minimum of 2048 tokens to prevent truncation, unless 
-    # explicitly asking for very small probe tests (<100 tokens).
-    if 100 < max_new_tokens < 2048:
-        max_new_tokens = 2048
-    temperature     = float(data.get("temperature",       0.7))
-    top_p           = float(data.get("top_p",             0.95))
-    top_k           = int(  data.get("top_k",             40))
-    repeat_penalty  = float(data.get("repeat_penalty",    1.15))
-    freq_penalty    = float(data.get("frequency_penalty", 0.1))
-
-    choices = []
-    total_prompt_tokens     = 0
-    total_completion_tokens = 0
-
-    # ── Sliding-window repetition detector (mirrors ai_workbench) ──
-    REP_WINDOW    = 120   # characters to treat as one "phrase"
-    REP_THRESHOLD = 2     # how many duplicate occurrences to tolerate
-
-    def _is_repeating(text: str) -> bool:
-        if len(text) < REP_WINDOW * (REP_THRESHOLD + 1):
-            return False
-        tail = text[-REP_WINDOW:]
-        preceding = text[: -REP_WINDOW]
-        count = 0
-        start = 0
-        while True:
-            idx = preceding.find(tail, start)
-            if idx == -1:
-                break
-            count += 1
-            if count >= REP_THRESHOLD:
-                return True
-            start = idx + 1
-        return False
-
-    for i, prompt in enumerate(prompts):
-        if isinstance(prompt, list):
-            messages = prompt
-        else:
-            messages = [
-                {"role": "system", "content": "You are a helpful, respectful and honest assistant."},
-                {"role": "user", "content": prompt}
-            ]
-
-        # Call chat completion API using streaming with full sampling controls
-        stream = model.create_chat_completion(
-            messages=messages,
-            max_tokens=max_new_tokens,
-            temperature=temperature if temperature > 0.15 else 0.0,
-            top_p=top_p,
-            top_k=top_k,
-            repeat_penalty=repeat_penalty,
-            frequency_penalty=freq_penalty,
-            stream=True,
-        )
-
-        full_output = ""
-        prompt_len = len(str(messages)) // 4
-        completion_len = 0
-        
-        print(f"\n[CONSOLE STREAM] Generating for: {MODEL_REPO}")
-        print("-" * 30)
-
-        for chunk in stream:
-            if "choices" in chunk and len(chunk["choices"]) > 0:
-                choice = chunk["choices"][0]
-                text_part = choice.get("delta", {}).get("content", "")
-                if not text_part:
-                    text_part = choice.get("text", "") # fallback if delta not present
-                    
-                if text_part:
-                    print(text_part, end="", flush=True)
-                    full_output += text_part
-                    completion_len += 1
-
-                if _is_repeating(full_output):
-                    print("\n[CONSOLE STREAM] Repetition detected — cutting off generation.")
-                    full_output = full_output[:-REP_WINDOW].strip()
-                    break
-
-        print("\n" + "-" * 30)
-
-        answer_text = full_output.strip()
-
-        total_prompt_tokens     += prompt_len
-        total_completion_tokens += completion_len
-
-        # Strip <think>...</think> block robustly (handles 4 failure modes)
-        think_text  = ""
-        think_end = answer_text.find("</think>")
-        think_start = answer_text.find("<think>")
-        
-        if think_end != -1:
-            # Case 1: Both <think> and </think> present
-            if think_start != -1 and think_start < think_end:
-                think_text = answer_text[think_start + len("<think>"):think_end].strip()
-                answer_text = (answer_text[:think_start] + "\n" + answer_text[think_end + len("</think>"):]).strip()
-            else:
-                # Case 2: Only </think> found — model started thinking implicitly
-                think_text = answer_text[:think_end].strip()
-                answer_text = answer_text[think_end + len("</think>"):].strip()
-        elif think_start != -1:
-            # Case 3: Orphaned <think> with NO </think> — model exhausted tokens mid-thought
-            think_text = answer_text[think_start + len("<think>"):].strip()
-            answer_text = answer_text[:think_start].strip()
-        
-        # Case 4: No tags at all — detect untagged thinking patterns from tiny models
-        if not answer_text or (not think_text and answer_text):
-            _THINK_PREFIXES = (
-                "Thinking Process:", "Let me analyze", "Let me think",
-                "I need to analyze", "Let me break this down",
-                "Let me review", "Let me examine", "Let me consider",
-                "I'll analyze", "Step 1:", "1.  **Analyze",
-            )
-            stripped = answer_text.lstrip("\n ")
-            for prefix in _THINK_PREFIXES:
-                if stripped.startswith(prefix):
-                    think_text = stripped
-                    answer_text = ""
-                    break
-
-        log.info("Prompt %d → %d new tokens (device=%s, gpu=%s, threads=%d)",
-                 i, completion_len, active_device, use_gpu and _gpu_available, cpu_threads)
-
-        choices.append({
-            "index":    i,
-            "text":     answer_text,
-            "thinking": think_text,
-        })
-
-    return {
-        "model":   MODEL_REPO,
-        "choices": choices,
-        "usage": {
-            "prompt_tokens":     total_prompt_tokens,
-            "completion_tokens": total_completion_tokens,
-        },
-        "device": active_device,
-    }
-
-
-def _inference_worker() -> None:
-    log.info("Inference worker thread started (pid=%d)", os.getpid())
-
-    while True:
-        try:
-            item = _inference_queue.get(timeout=1.0)
-        except _queue_module.Empty:
-            continue
-
-        req_data, result_holder, done_event = item
-        try:
-            result_holder[0] = _run_inference(req_data)
-        except Exception as exc:
-            log.error("Inference worker error: %s", exc)
-            result_holder[0] = {"error": f"Inference failed: {exc}"}
-        finally:
-            done_event.set()
-            _inference_queue.task_done()
-
-
-_worker_thread = threading.Thread(target=_inference_worker, name="inference-worker", daemon=True)
-_worker_thread.start()
-
-
-# ── Routes ────────────────────────────────────────────────────────────────────
-
-@app.route("/v1/kv_cache", methods=["POST"])
-def kv_cache():
-    """KV cache is managed natively by llama.cpp. This is a no-op."""
-    return jsonify({"status": "skipped", "reason": "llama.cpp manages KV cache natively"})
-
-
-@app.route("/health", methods=["GET"])
-def health():
-    import psutil
-    mem          = psutil.virtual_memory()
-    ram_used_gb  = round((mem.total - mem.available) / 1024 ** 3, 2)
-    ram_total_gb = round(mem.total / 1024 ** 3, 2)
-
-    queue_depth = _inference_queue.qsize()
-    is_ready    = _model_ready.is_set()
-    loaded_modes = list(_models.keys())
-
-    return jsonify({
-        "status":           "ok" if is_ready else "loading",
-        "model":            MODEL_REPO,
-        "gpu_available":    _gpu_available,
-        "gpu_id":           _gpu_id,
-        "loaded_modes":     loaded_modes,
-        "default_threads":  DEFAULT_CPU_THREADS,
-        "kv_cache_length":  0,
-        "kv_cache_enabled": False,
-        "torch_compile":    False,
-        "vram_free_gib":    0.0,
-        "ram_used_gb":      ram_used_gb,
-        "ram_total_gb":     ram_total_gb,
-        "queue_depth":      queue_depth,
-        "queue_max":        _QUEUE_MAX_SIZE,
-        "model_ready":      is_ready,
-    })
-
-
-@app.route("/v1/completions", methods=["POST"])
-def completions():
-    if not _model_ready.is_set():
-        return jsonify({
-            "error":       "Model is still loading — please try again in a few seconds.",
-            "retry_after": 5,
-        }), 503
-
-    data: dict = request.get_json(force=True) or {}
-
-    current_depth = _inference_queue.qsize()
-    if current_depth >= _QUEUE_MAX_SIZE:
-        log.warning("Inference queue full (%d/%d) — rejecting request.", current_depth, _QUEUE_MAX_SIZE)
-        return jsonify({
-            "error":       "Server busy — all inference slots are occupied. Please try again shortly.",
-            "retry_after": max(5, current_depth * 3),
-            "queue_depth": current_depth,
-            "queue_max":   _QUEUE_MAX_SIZE,
-        }), 503
-
-    result_holder: list = [None]
-    done_event = threading.Event()
-
-    try:
-        _inference_queue.put_nowait((data, result_holder, done_event))
-    except _queue_module.Full:
-        return jsonify({
-            "error":       "Server busy — inference queue full. Please try again shortly.",
-            "retry_after": 5,
-        }), 503
-
-    completed = done_event.wait(timeout=_REQUEST_TIMEOUT_S)
-
-    if not completed:
-        return jsonify({
-            "error":       f"Request timed out after {_REQUEST_TIMEOUT_S}s. ",
-            "retry_after": 10,
-        }), 503
-
-    result = result_holder[0]
-    if result is None:
-        return jsonify({"error": "Internal error: inference worker returned no result."}), 500
-
-    if "error" in result:
-        return jsonify(result), 500
-
-    return jsonify(result)
-
-
-if __name__ == "__main__":
-    import signal, sys
-
-    def sigint_handler(sig, frame):
-        sys.exit(0)
-    signal.signal(signal.SIGINT, sigint_handler)
-
-    app.run(host=HOST, port=PORT, debug=False, threaded=True)

agents/llm.py

@@ -68,7 +68,7 @@ class LocalLLM(BaseLLM):
         except requests.exceptions.ConnectionError:
             self._last_prompt_tokens     = 0
             self._last_completion_tokens = 0
-            return "[LLM OFFLINE] Cannot connect to the inference server. Is gen_llm.py running?"
+            return "[LLM OFFLINE] Cannot connect to the inference server. Is nvidia_llm.py running?"
         except requests.exceptions.Timeout:
             self._last_prompt_tokens     = 0
             self._last_completion_tokens = 0

agents/nvidia_llm.py

@@ -0,0 +1,195 @@
+import os
+import logging
+import threading
+import queue as _queue_module
+import time
+from flask import Flask, request, jsonify
+from openai import OpenAI
+
+# ── Logging ───────────────────────────────────────────────────────────────────
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(name)s] %(levelname)s %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+log = logging.getLogger("nvidia_llm")
+logging.getLogger("werkzeug").setLevel(logging.ERROR)
+logging.getLogger("httpx").setLevel(logging.WARNING)
+
+# ── Config ────────────────────────────────────────────────────────────────────
+HOST         = os.getenv("NVIDIA_HOST",  "127.0.0.1")
+PORT         = int(os.getenv("NVIDIA_PORT", "8002"))
+NVIDIA_API_KEY = os.getenv("NVIDIA_API_KEY")
+
+_QUEUE_MAX_SIZE    = int(os.getenv("NVIDIA_QUEUE_MAX", "8"))
+_REQUEST_TIMEOUT_S = int(os.getenv("NVIDIA_LLM_TIMEOUT", "600"))
+
+DEFAULT_MODEL = "MuXodious/Qwen2.5-7B-Instruct-1M-Thinking-Claude-Gemini-GPT5.2-DISTILL-PaperWitch-heresy"
+
+# ── Flask app ─────────────────────────────────────────────────────────────────
+app = Flask(__name__)
+
+@app.after_request
+def after_request(response):
+    response.headers.add('Access-Control-Allow-Origin', '*')
+    response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
+    response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
+    return response
+
+# ── Inference Queue (multi-user serialization) ─────────────────────────────────
+_inference_queue: _queue_module.Queue = _queue_module.Queue(maxsize=_QUEUE_MAX_SIZE)
+
+def _run_inference(data: dict) -> dict:
+    raw_prompt = data.get("prompt", "")
+    if not raw_prompt:
+        return {"error": "Field 'prompt' is required."}
+
+    prompts = raw_prompt if isinstance(raw_prompt, list) and (len(raw_prompt) == 0 or not isinstance(raw_prompt[0], dict)) else [raw_prompt]
+    
+    max_tokens = int(data.get("max_tokens", 1024))
+    temperature = float(data.get("temperature", 0.5))
+    top_p = float(data.get("top_p", 1.0))
+    model_name = data.get("model", DEFAULT_MODEL)
+
+    if not NVIDIA_API_KEY:
+        log.warning("NVIDIA_API_KEY is not set. API calls might fail if the token is required.")
+
+    client = OpenAI(
+        base_url="https://nim.api.nvidia.com/v1",
+        api_key=NVIDIA_API_KEY or "dummy-key-if-not-required-locally"
+    )
+
+    choices = []
+    
+    for i, prompt in enumerate(prompts):
+        if isinstance(prompt, list):
+            messages = prompt
+        else:
+            messages = [
+                {"role": "user", "content": prompt}
+            ]
+
+        try:
+            completion = client.chat.completions.create(
+                model=model_name,
+                messages=messages,
+                temperature=temperature,
+                top_p=top_p,
+                max_tokens=max_tokens,
+                stream=True
+            )
+
+            full_output = ""
+            print(f"\n[CONSOLE STREAM] Generating via NVIDIA for: {model_name}")
+            print("-" * 30)
+
+            for chunk in completion:
+                if chunk.choices and chunk.choices[0].delta.content is not None:
+                    content = chunk.choices[0].delta.content
+                    print(content, end="", flush=True)
+                    full_output += content
+
+            print("\n" + "-" * 30)
+            
+            choices.append({
+                "index": i,
+                "text": full_output.strip(),
+                "thinking": ""
+            })
+
+        except Exception as e:
+            log.error(f"Error calling NVIDIA API: {e}")
+            choices.append({
+                "index": i,
+                "text": f"Error: {str(e)}",
+                "thinking": ""
+            })
+
+    return {
+        "model": model_name,
+        "choices": choices,
+        "device": "cloud_nvidia"
+    }
+
+def _inference_worker() -> None:
+    log.info("Inference worker thread started (pid=%d)", os.getpid())
+    while True:
+        try:
+            item = _inference_queue.get(timeout=1.0)
+        except _queue_module.Empty:
+            continue
+
+        req_data, result_holder, done_event = item
+        try:
+            result_holder[0] = _run_inference(req_data)
+        except Exception as exc:
+            log.error("Inference worker error: %s", exc)
+            result_holder[0] = {"error": f"Inference failed: {exc}"}
+        finally:
+            done_event.set()
+            _inference_queue.task_done()
+
+_worker_thread = threading.Thread(target=_inference_worker, name="inference-worker", daemon=True)
+_worker_thread.start()
+
+# ── Routes ────────────────────────────────────────────────────────────────────
+
+@app.route("/health", methods=["GET"])
+def health():
+    return jsonify({
+        "status": "ok",
+        "queue_depth": _inference_queue.qsize(),
+        "queue_max": _QUEUE_MAX_SIZE
+    })
+
+@app.route("/v1/completions", methods=["POST", "OPTIONS"])
+def completions():
+    if request.method == "OPTIONS":
+        return jsonify({}), 200
+
+    data: dict = request.get_json(force=True) or {}
+
+    current_depth = _inference_queue.qsize()
+    if current_depth >= _QUEUE_MAX_SIZE:
+        return jsonify({
+            "error": "Server busy — all inference slots are occupied. Please try again shortly.",
+            "retry_after": 5
+        }), 503
+
+    result_holder: list = [None]
+    done_event = threading.Event()
+
+    try:
+        _inference_queue.put_nowait((data, result_holder, done_event))
+    except _queue_module.Full:
+        return jsonify({
+            "error": "Server busy — inference queue full. Please try again shortly.",
+            "retry_after": 5,
+        }), 503
+
+    completed = done_event.wait(timeout=_REQUEST_TIMEOUT_S)
+
+    if not completed:
+        return jsonify({
+            "error": f"Request timed out after {_REQUEST_TIMEOUT_S}s. ",
+            "retry_after": 10,
+        }), 503
+
+    result = result_holder[0]
+    if result is None:
+        return jsonify({"error": "Internal error: inference worker returned no result."}), 500
+
+    if "error" in result:
+        return jsonify(result), 500
+
+    return jsonify(result)
+
+if __name__ == "__main__":
+    import signal, sys
+
+    def sigint_handler(sig, frame):
+        sys.exit(0)
+    signal.signal(signal.SIGINT, sigint_handler)
+
+    log.info(f"Starting NVIDIA LLM agent on http://{HOST}:{PORT}")
+    app.run(host=HOST, port=PORT, debug=False, threaded=True)

app.py

@@ -240,7 +240,7 @@ def get_session_token() -> str:
     return token
 
 def trigger_kv_cache_update(session_token: str = "admin"):
-    """Fetches all text and sends it to gen_llm to update KV cache."""
+    """Fetches all text and sends it to nvidia_llm to update KV cache."""
     def _update(token):
         from pipeline import vector_store
         import requests
@@ -327,7 +327,7 @@ def status():
     vec_count  = vector_store.count()
     graph_stat = graph_store.get_stats()
 
-    # Probe gen_llm
+    # Probe nvidia_llm
     import requests as req
     gen_ok, embed_ok = False, False
     gen_info = {}
@@ -354,7 +354,7 @@ def status():
     return jsonify({
         "vector_db":  {"status": "ok", "chunks": vec_count},
         "graph_db":   graph_stat,
-        "gen_llm":    {
+        "nvidia_llm":    {
             "endpoint": config.LLM_BASE_URL,
             "online":   gen_ok,
             "model":    "-".join(gen_info.get("model", config.LLM_MODEL_ID).split("-")[:2]) if "-" in gen_info.get("model", config.LLM_MODEL_ID) else gen_info.get("model", config.LLM_MODEL_ID),
@@ -1041,7 +1041,7 @@ def ingest_v1_sync():
 
 @app.route("/api/probe/gen", methods=["POST"])
 def probe_gen():
-    """Quick smoke-test for the gen_llm server."""
+    """Quick smoke-test for the nvidia_llm server."""
     import requests as req
     try:
         r = req.post(

nvidia_frontend_test.html

@@ -0,0 +1,204 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>NVIDIA LLM Local Endpoint Tester</title>
+    <style>
+        body {
+            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+            background-color: #121212;
+            color: #e0e0e0;
+            margin: 0;
+            padding: 20px;
+            display: flex;
+            flex-direction: column;
+            align-items: center;
+        }
+        h1 {
+            color: #76b900; /* NVIDIA Green */
+            margin-bottom: 20px;
+        }
+        .container {
+            width: 100%;
+            max-width: 800px;
+            background-color: #1e1e1e;
+            border-radius: 10px;
+            padding: 20px;
+            box-shadow: 0 4px 15px rgba(0,0,0,0.5);
+        }
+        .form-group {
+            margin-bottom: 15px;
+        }
+        label {
+            display: block;
+            margin-bottom: 5px;
+            font-weight: bold;
+            color: #bbb;
+        }
+        textarea, input[type="text"], input[type="number"] {
+            width: 100%;
+            padding: 10px;
+            border: 1px solid #333;
+            border-radius: 5px;
+            background-color: #2a2a2a;
+            color: #fff;
+            box-sizing: border-box;
+        }
+        textarea {
+            resize: vertical;
+            height: 120px;
+        }
+        button {
+            background-color: #76b900;
+            color: #000;
+            border: none;
+            padding: 10px 20px;
+            border-radius: 5px;
+            font-weight: bold;
+            cursor: pointer;
+            transition: background-color 0.2s;
+            font-size: 16px;
+        }
+        button:hover {
+            background-color: #8ce100;
+        }
+        button:disabled {
+            background-color: #444;
+            color: #888;
+            cursor: not-allowed;
+        }
+        .response-area {
+            margin-top: 20px;
+        }
+        .response-box {
+            background-color: #2a2a2a;
+            border: 1px solid #444;
+            border-radius: 5px;
+            padding: 15px;
+            min-height: 100px;
+            white-space: pre-wrap;
+            overflow-x: auto;
+            color: #dcdcdc;
+            line-height: 1.5;
+        }
+        .error {
+            color: #ff5252;
+            font-weight: bold;
+        }
+        .spinner {
+            display: none;
+            margin-left: 10px;
+            border: 3px solid rgba(255,255,255,0.1);
+            border-top: 3px solid #76b900;
+            border-radius: 50%;
+            width: 15px;
+            height: 15px;
+            animation: spin 1s linear infinite;
+            vertical-align: middle;
+        }
+        @keyframes spin {
+            0% { transform: rotate(0deg); }
+            100% { transform: rotate(360deg); }
+        }
+    </style>
+</head>
+<body>
+
+    <h1>NVIDIA LLM Tester</h1>
+    
+    <div class="container">
+        <div class="form-group">
+            <label for="endpoint">Local API Endpoint:</label>
+            <input type="text" id="endpoint" value="http://127.0.0.1:8003/v1/completions">
+        </div>
+        
+        <div class="form-group">
+            <label for="prompt">Prompt:</label>
+            <textarea id="prompt" placeholder="Enter your prompt here...">What are the top 3 features of NVIDIA GPUs for AI workloads?</textarea>
+        </div>
+        
+        <div class="form-group" style="display: flex; gap: 15px;">
+            <div style="flex: 1;">
+                <label for="max_tokens">Max Tokens:</label>
+                <input type="number" id="max_tokens" value="1024">
+            </div>
+            <div style="flex: 1;">
+                <label for="temperature">Temperature:</label>
+                <input type="number" id="temperature" value="0.5" step="0.1" min="0" max="2">
+            </div>
+        </div>
+
+        <button id="send-btn">
+            Send Request
+            <span class="spinner" id="spinner"></span>
+        </button>
+
+        <div class="response-area">
+            <label>Response:</label>
+            <div class="response-box" id="response-box">Awaiting input...</div>
+        </div>
+    </div>
+
+    <script>
+        document.getElementById('send-btn').addEventListener('click', async () => {
+            const endpoint = document.getElementById('endpoint').value.trim();
+            const promptText = document.getElementById('prompt').value.trim();
+            const maxTokens = parseInt(document.getElementById('max_tokens').value) || 1024;
+            const temperature = parseFloat(document.getElementById('temperature').value) || 0.5;
+            const responseBox = document.getElementById('response-box');
+            const sendBtn = document.getElementById('send-btn');
+            const spinner = document.getElementById('spinner');
+
+            if (!promptText) {
+                alert("Please enter a prompt.");
+                return;
+            }
+
+            // UI feedback
+            sendBtn.disabled = true;
+            spinner.style.display = 'inline-block';
+            responseBox.innerHTML = '<i>Processing request...</i>';
+            responseBox.classList.remove('error');
+
+            const payload = {
+                prompt: promptText,
+                max_tokens: maxTokens,
+                temperature: temperature
+            };
+
+            try {
+                const response = await fetch(endpoint, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify(payload)
+                });
+
+                if (!response.ok) {
+                    const errorData = await response.json().catch(() => null);
+                    const errorMsg = errorData && errorData.error ? errorData.error : `HTTP error ${response.status}`;
+                    throw new Error(errorMsg);
+                }
+
+                const data = await response.json();
+                
+                if (data.choices && data.choices.length > 0) {
+                    responseBox.textContent = data.choices[0].text;
+                } else if (data.error) {
+                    throw new Error(data.error);
+                } else {
+                    responseBox.textContent = "Received unexpected response format:\n" + JSON.stringify(data, null, 2);
+                }
+            } catch (err) {
+                responseBox.textContent = `Error: ${err.message}`;
+                responseBox.classList.add('error');
+            } finally {
+                sendBtn.disabled = false;
+                spinner.style.display = 'none';
+            }
+        });
+    </script>
+</body>
+</html>

start.sh

@@ -7,7 +7,7 @@
 #   bash start.sh -hf -noadmin # HF mode with admin controls disabled (public endpoint)
 #
 # Environment variables set by this script:
-#   HF_MODE=1       → Activates low-resource CPU path in config.py, gen_llm.py, embed_llm.py
+#   HF_MODE=1       → Activates low-resource CPU path in config.py, nvidia_llm.py, embed_llm.py
 #   ADMIN_MODE=0    → Disables admin API routes and hides UI admin controls
 #   GEN_MODEL_ID    → Overridden for HF mode (microsoft/Phi-3.5-mini-instruct)
 #   EMBED_MODEL_ID  → Overridden for HF mode (bge-small-en-v1.5)
@@ -90,7 +90,7 @@ else
     fi
   done
   # Also kill by process name for orphaned workers
-  pkill -9 -f "agents/gen_llm.py"  2>/dev/null || true
+  pkill -9 -f "agents/nvidia_llm.py"  2>/dev/null || true
   pkill -9 -f "agents/embed_llm.py" 2>/dev/null || true
   sleep 2
   echo "[pre-flight] Done."
@@ -106,11 +106,11 @@ python agents/embed_llm.py &
 EMBED_PID=$!
 echo "      embed_llm PID: $EMBED_PID"
 
-# ── Start gen_llm on port 8002 ────────────────────────────────────────────────
-echo "[2/3] Starting gen_llm (port 8002)..."
-python agents/gen_llm.py &
+# ── Start nvidia_llm on port 8002 ────────────────────────────────────────────────
+echo "[2/3] Starting nvidia_llm (port 8002)..."
+python agents/nvidia_llm.py &
 GEN_PID=$!
-echo "      gen_llm PID: $GEN_PID"
+echo "      nvidia_llm PID: $GEN_PID"
 
 # ── Wait for microservices to initialise ──────────────────────────────────────
 echo "Waiting for LLM microservices to initialise..."

static/app.js

@@ -112,7 +112,7 @@ async function refreshStatus() {
 
     const vecOk   = (d.vector_db?.chunks ?? -1) >= 0;
     const graphOk = d.graph_db?.available;
-    const genOk   = d.gen_llm?.online;
+    const genOk   = d.nvidia_llm?.online;
     const embedOk = d.embed_llm?.online;
     
     state.isAdmin = !!d.is_admin;
@@ -128,13 +128,13 @@ async function refreshStatus() {
     setPill('status-graph',  graphOk, graphOk ? `Kuzu DB · ${d.graph_db.nodes} nodes, ${d.graph_db.relationships} edges` : 'Kuzu DB · offline');
     
     const genStatus = genOk 
-        ? `Gen · ${(d.gen_llm?.model || '').split('/').pop()} (GPU: ${d.gen_llm?.gpu_id} | KV: ${d.gen_llm?.kv_cache_length} tkns)` 
+        ? `Gen · ${(d.nvidia_llm?.model || '').split('/').pop()} (GPU: ${d.nvidia_llm?.gpu_id} | KV: ${d.nvidia_llm?.kv_cache_length} tkns)` 
         : 'Gen LLM · offline';
     setPill('status-gen',    genOk,   genStatus);
     
     setPill('status-embed',  embedOk, embedOk ? `Embed · ${(d.embed_llm?.model || '').split('/').pop()}` : 'Embed LLM · offline');
 
-    if (!genOk)   diag.warn('gen_llm server is offline');
+    if (!genOk)   diag.warn('nvidia_llm server is offline');
     if (!embedOk) diag.warn('embed_llm server is offline');
   } catch(err) {
     diag.error('Status refresh failed:', err);
@@ -612,7 +612,7 @@ async function pollJobStatus(jobId) {
 
 // ── Default prompt buttons ─────────────────────────────────────────────────────
 $('preset-gen').addEventListener('click', async () => {
-  diag.info('Probing gen_llm server…');
+  diag.info('Probing nvidia_llm server…');
   notify('Testing Gen LLM server (port 8002)…', 'info', 3000);
   const btn = $('preset-gen');
   btn.disabled = true;

templates/index.html

@@ -399,7 +399,7 @@
             return;
           }
           const vecOk = (data.vector_db?.chunks ?? -1) >= 0;
-          const genOk = data.gen_llm?.online;
+          const genOk = data.nvidia_llm?.online;
           const embedOk = data.embed_llm?.online;
           // Check for active ingestion / graphdb tasks via our system info endpoint as well
           let isIngesting = false;