AI Agent commited on
Commit
73ef8a0
·
1 Parent(s): b63aac9

maintenance: show stub page until STUP enrollment window opens

Browse files
Files changed (1) hide show
  1. app.py +73 -1270
app.py CHANGED
@@ -1,1277 +1,80 @@
1
- # app.py - healthexpert UI
2
-
3
- """Document AI Expert — Flask Application."""
4
- from __future__ import annotations
5
- import sys
6
-
7
- # ── CLI switch parsing (must happen BEFORE config import) ─────────────────────
8
- # -hf → sets HF_MODE=1 (low-resource CPU mode)
9
- # -noadmin → sets ADMIN_MODE=0 (disables admin routes and UI controls)
10
- for _arg in sys.argv[1:]:
11
- if _arg in ("-hf", "--hf"):
12
- import os as _os
13
- _os.environ["HF_MODE"] = "1"
14
- elif _arg in ("-noadmin", "--noadmin"):
15
- import os as _os
16
- _os.environ["ADMIN_MODE"] = "0"
17
-
18
- import os
19
- os.environ["PYTHONWARNINGS"] = "ignore"
20
- # Suppress HuggingFace Hub unauthenticated-request noise before any imports
21
- os.environ.setdefault("HF_HUB_DISABLE_IMPLICIT_TOKEN", "1")
22
- os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
23
- import uuid, json, threading, subprocess, logging, warnings, signal, time
24
- from pathlib import Path
25
-
26
- warnings.filterwarnings("ignore", category=ImportWarning)
27
- warnings.filterwarnings("ignore", category=DeprecationWarning)
28
- warnings.filterwarnings("ignore", category=UserWarning)
29
- warnings.filterwarnings("ignore", message=".*register_constant.*")
30
- warnings.filterwarnings("ignore", message=".*Enum subclass.*")
31
- warnings.filterwarnings("ignore", message=".*unauthenticated.*")
32
- from flask import Flask, render_template, request, jsonify, Response, stream_with_context
33
-
34
- import sys
35
- sys.path.insert(0, str(Path(__file__).parent))
36
- import config
37
- from pipeline import vector_store, graph_store, embedder, document_loader, chunker
38
- from agents.crew import run_ingest_crew, run_query_crew
39
-
40
- log = logging.getLogger("app")
41
- logging.basicConfig(level=logging.INFO,
42
- format="%(asctime)s [app] %(levelname)s %(message)s")
43
-
44
- # ── Silence noisy third-party loggers ─────────────────────────────────────────
45
- for _quiet in (
46
- "werkzeug",
47
- "numexpr",
48
- "httpx",
49
- "filelock",
50
- "pikepdf",
51
- "pikepdf._core",
52
- "unstructured",
53
- "unstructured.partition",
54
- "unstructured.partition.pdf",
55
- "pdfminer",
56
- "pdfminer.pdfdocument",
57
- "pdfminer.pdfpage",
58
- "pdfminer.converter",
59
- "huggingface_hub",
60
- "huggingface_hub.utils",
61
- "huggingface_hub.utils._validators",
62
- "transformers",
63
- "sentence_transformers",
64
- "detectron2",
65
- "pytesseract",
66
- "PIL",
67
- "torch",
68
- "torch.utils",
69
- "torch.utils._pytree",
70
- ):
71
- logging.getLogger(_quiet).setLevel(logging.ERROR)
72
-
73
- # Capture all Python warnings and route them to the py.warnings logger, then silence it
74
- logging.captureWarnings(True)
75
- logging.getLogger("py.warnings").setLevel(logging.ERROR)
76
-
77
- # ── App setup ─────────────────────────────────────────────────────────────────
78
- app = Flask(__name__)
79
- app.secret_key = config.SECRET_KEY
80
- app.config["MAX_CONTENT_LENGTH"] = config.MAX_CONTENT_LENGTH
81
- os.makedirs(config.UPLOAD_FOLDER, exist_ok=True)
82
-
83
- from flask_limiter import Limiter
84
- from flask_limiter.util import get_remote_address
85
- from werkzeug.utils import escape
86
- from huggingface_hub import HfApi, hf_hub_download
87
-
88
- # Configure Rate Limiter (Memory Storage for MVP)
89
- limiter = Limiter(
90
- get_remote_address,
91
- app=app,
92
- default_limits=["200 per day", "10 per minute"],
93
- storage_uri="memory://"
94
- )
95
-
96
- @app.after_request
97
- def add_security_headers(response):
98
- response.headers['X-Content-Type-Options'] = 'nosniff'
99
- response.headers['X-Frame-Options'] = 'SAMEORIGIN'
100
- response.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains'
101
- response.headers['Content-Security-Policy'] = "default-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.jsdelivr.net;"
102
- return response
103
-
104
- # Global background API for dataset upload
105
- _hf_api = None
106
- def get_hf_api():
107
- global _hf_api
108
- if _hf_api is None:
109
- token = os.environ.get("HF_PRIVATE_TOKEN") or os.environ.get("HF_TOKEN")
110
- if token:
111
- _hf_api = HfApi(token=token)
112
- return _hf_api
113
-
114
- def async_sync_log(local_path: str, repo_path: str):
115
- def _upload():
116
- api = get_hf_api()
117
- if api:
118
- try:
119
- api.upload_file(
120
- path_or_fileobj=local_path,
121
- path_in_repo=repo_path,
122
- repo_id="Sam-max1/mat_data",
123
- repo_type="dataset"
124
- )
125
- except Exception as e:
126
- log.warning(f"Failed to push {repo_path} to mat_data: {e}")
127
- threading.Thread(target=_upload, daemon=True).start()
128
-
129
- # In-memory job tracker for async ingestion
130
- _jobs: dict[str, dict] = {}
131
- _active_graph_tasks = 0
132
- _session_uploads: dict[str, int] = {}
133
-
134
- # Global thread pool to allow concurrent RAG execution
135
- from concurrent.futures import ThreadPoolExecutor
136
- _query_executor = ThreadPoolExecutor(max_workers=2)
137
-
138
- # Auto-ingest background progress tracker
139
- _auto_ingest_status: dict = {
140
- "running": False,
141
- "done": False,
142
- "total": 0,
143
- "completed": 0,
144
- "current_file": None,
145
- "results": [],
146
- "error": None,
147
- }
148
-
149
- # RBAC Session Tracking
150
- _active_sessions: dict[str, float] = {}
151
- SESSION_TIMEOUT_SECONDS = 600 # 10 minutes
152
-
153
- def _allowed(filename: str) -> bool:
154
- return Path(filename).suffix.lower() in config.ALLOWED_EXTENSIONS
155
-
156
- def is_admin() -> bool:
157
- """Return True if the request comes from an admin-privileged context.
158
-
159
- In HF mode with ADMIN_MODE=1: admin is granted to all localhost requests.
160
- With ADMIN_MODE=0 (-noadmin): always False — no admin access regardless of IP.
161
- """
162
- if not config.ADMIN_MODE:
163
- return False
164
- if config.HF_MODE:
165
- # In HF mode, admin is only valid from the loopback (e.g. start.sh itself)
166
- return request.remote_addr in ("127.0.0.1", "::1")
167
- return request.remote_addr in ("127.0.0.1", "::1", "localhost")
168
-
169
- @app.before_request
170
- def block_external_apis():
171
- """Hard block all external API (headless) access in public mode."""
172
- if not config.ADMIN_MODE:
173
- if request.path.startswith("/api/v1/"):
174
- return jsonify({"error": "Headless API access is disabled in public mode."}), 403
175
-
176
- def log_session(event_type: str, token: str, ip: str):
177
- try:
178
- log_dir = Path(__file__).parent / "app" / "logs"
179
- log_dir.mkdir(parents=True, exist_ok=True)
180
- session_file = log_dir / "nitdaa_sessions.json"
181
-
182
- from datetime import datetime, timezone, timedelta
183
- ist = timezone(timedelta(hours=5, minutes=30))
184
- ts = datetime.now(ist).strftime("%Y-%m-%d %H:%M:%S")
185
-
186
- entry = {"timestamp": ts, "event": event_type, "session_token": token, "ip_address": ip}
187
- with open(session_file, "a") as f:
188
- f.write(json.dumps(entry) + "\n")
189
-
190
- async_sync_log(str(session_file), "nitdaa_sessions.json")
191
- except Exception as e:
192
- log.error(f"Failed to log session: {e}")
193
-
194
- _known_sessions = {}
195
-
196
- def log_query_summary(token: str, ip: str, query: str, chunks_retrieved: int, gen_time: float, success: bool, error: str = "", job_id: str = ""):
197
- try:
198
- log_dir = Path(__file__).parent / "app" / "logs"
199
- log_dir.mkdir(parents=True, exist_ok=True)
200
- summary_file = log_dir / "nitdaa_summary.json"
201
-
202
- from datetime import datetime, timezone, timedelta
203
- ist = timezone(timedelta(hours=5, minutes=30))
204
- ts = datetime.now(ist).strftime("%Y-%m-%d %H:%M:%S")
205
-
206
- entry = {
207
- "timestamp": ts,
208
- "job_id": job_id,
209
- "ip_address": ip,
210
- "session_token": token,
211
- "question": query,
212
- "chunks_retrieved": chunks_retrieved,
213
- "generation_time_sec": gen_time,
214
- "success": success,
215
- "error": error
216
  }
217
- with open(summary_file, "a") as f:
218
- f.write(json.dumps(entry) + "\n")
219
-
220
- async_sync_log(str(summary_file), "nitdaa_summary.json")
221
- except Exception as e:
222
- log.error(f"Failed to log query summary: {e}")
223
-
224
- def get_session_token() -> str:
225
- """Return session token if the user is not an admin, else 'admin'."""
226
- if is_admin():
227
- return "admin"
228
- token = request.headers.get("X-Session-Token") or request.form.get("session_token")
229
- if not token and request.json:
230
- token = request.json.get("session_token")
231
- if not token:
232
- token = "anonymous"
233
-
234
- ip = request.remote_addr
235
- if token not in _known_sessions and token not in ("admin", "anonymous"):
236
- _known_sessions[token] = ip
237
- log_session("CONNECT", token, ip)
238
-
239
- _active_sessions[token] = time.time()
240
- return token
241
-
242
- def trigger_kv_cache_update(session_token: str = "admin"):
243
- """Fetches all text and sends it to nvidia_llm to update KV cache."""
244
- def _update(token):
245
- from pipeline import vector_store
246
- import requests
247
- text = vector_store.get_all_text(session_token=token)
248
- log.info("Triggering KV cache update with %d chars...", len(text))
249
- try:
250
- requests.post(f"{config.LLM_BASE_URL}/v1/kv_cache", json={"text": text}, timeout=120)
251
- log.info("KV Cache updated successfully.")
252
- except Exception as e:
253
- log.error("Failed to update KV Cache: %s", e)
254
- threading.Thread(target=_update, args=(session_token,), daemon=True).start()
255
-
256
-
257
- def _run_docker(action: str) -> tuple[bool, str]:
258
- """Run docker compose action ('up', 'down', 'restart') and return (ok, message)."""
259
- compose_file = str(Path(__file__).parent / "docker-compose.yml")
260
- cmd_map = {
261
- "up": ["docker", "compose", "-f", compose_file, "up", "-d"],
262
- "down": ["docker", "compose", "-f", compose_file, "down"],
263
- "restart": ["docker", "compose", "-f", compose_file, "restart"],
264
- }
265
- cmd = cmd_map.get(action)
266
- if cmd is None:
267
- return False, f"Unknown action: {action}"
268
- try:
269
- result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
270
- ok = result.returncode == 0
271
- out = (result.stdout + result.stderr).strip()
272
- log.info("docker compose %s → rc=%d %s", action, result.returncode, out[:200])
273
- return ok, out or ("OK" if ok else "Command returned non-zero exit code")
274
- except subprocess.TimeoutExpired:
275
- return False, "docker compose timed out after 60 s"
276
- except FileNotFoundError:
277
- return False, "docker binary not found — ensure Docker is installed"
278
- except Exception as exc:
279
- return False, str(exc)
280
-
281
-
282
- # ── Graceful Shutdown ─────────────────────────────────────────────────────────
283
-
284
- def _graceful_shutdown(signum, frame):
285
- log.error(f"Received signal {signum}. Triggering kill switch for graceful shutdown...")
286
- _run_docker("down")
287
- import time
288
- time.sleep(1)
289
- os._exit(0)
290
-
291
- signal.signal(signal.SIGINT, _graceful_shutdown)
292
- signal.signal(signal.SIGTERM, _graceful_shutdown)
293
-
294
-
295
- # ── Background Cleanup Agent ──────────────────────────────────────────────────
296
-
297
- def _cleanup_agent():
298
- while True:
299
- time.sleep(60)
300
- now = time.time()
301
- expired = [token for token, last_active in _active_sessions.items()
302
- if token != "admin" and token != "anonymous" and (now - last_active) > SESSION_TIMEOUT_SECONDS]
303
- for token in expired:
304
- log.info(f"Cleanup Agent: Session '{token}' inactive for 10 mins. Purging data...")
305
- if token in _known_sessions:
306
- log_session("DISCONNECT", token, _known_sessions[token])
307
- del _known_sessions[token]
308
- vector_store.delete_by_session(token)
309
- graph_store.delete_by_session(token)
310
- del _active_sessions[token]
311
- trigger_kv_cache_update(token)
312
-
313
- threading.Thread(target=_cleanup_agent, daemon=True).start()
314
-
315
-
316
- # ── Routes ────────────────────────────────────────────────────────────────────
317
-
318
- @app.route("/")
319
- def index():
320
- return render_template("index.html", config=config)
321
-
322
-
323
- @app.route("/api/status")
324
- @limiter.exempt
325
- def status():
326
- """Health check for all backends."""
327
- vec_count = vector_store.count()
328
- graph_stat = graph_store.get_stats()
329
-
330
- # Probe nvidia_llm
331
- import requests as req
332
- gen_ok, embed_ok = False, False
333
- gen_info = {}
334
- try:
335
- r = req.get(f"{config.LLM_BASE_URL}/health", timeout=3)
336
- gen_ok = r.status_code == 200
337
- if gen_ok:
338
- gen_info = r.json()
339
- except req.exceptions.ReadTimeout:
340
- # LLM is busy generating, which is fine
341
- gen_ok = True
342
- gen_info = {"status": "busy", "model": config.LLM_MODEL_ID}
343
- except Exception as e:
344
- pass
345
-
346
- try:
347
- r = req.get(f"{config.EMBED_BASE_URL}/health", timeout=3)
348
- embed_ok = r.status_code == 200
349
- except req.exceptions.ReadTimeout:
350
- embed_ok = True
351
- except Exception as e:
352
- pass
353
-
354
- return jsonify({
355
- "vector_db": {"status": "ok", "chunks": vec_count},
356
- "graph_db": graph_stat,
357
- "nvidia_llm": {
358
- "endpoint": config.LLM_BASE_URL,
359
- "online": gen_ok,
360
- "model": "-".join(gen_info.get("model", config.LLM_MODEL_ID).split("-")[:2]) if "-" in gen_info.get("model", config.LLM_MODEL_ID) else gen_info.get("model", config.LLM_MODEL_ID),
361
- "gpu_id": gen_info.get("gpu_id", "cpu"),
362
- "kv_cache_length": gen_info.get("kv_cache_length", 0),
363
- },
364
- "embed_llm": {
365
- "endpoint": config.EMBED_EMBEDDINGS_URL,
366
- "model": config.EMBEDDING_MODEL,
367
- "online": embed_ok,
368
- },
369
- "is_admin": is_admin(),
370
- "hf_mode": config.HF_MODE,
371
- "admin_mode": config.ADMIN_MODE,
372
- })
373
-
374
-
375
- @app.route("/api/sysinfo")
376
- @limiter.exempt
377
- def sysinfo():
378
- """System resource info for the UI resource banner.
379
- Returns CPU model/count, load %, RAM used/total (GB), disk free/total (GB).
380
- """
381
- try:
382
- import psutil
383
- mem = psutil.virtual_memory()
384
- disk = psutil.disk_usage("/")
385
- cpu_freq = psutil.cpu_freq()
386
-
387
- # RAM in GB
388
- ram_total_gb = round(mem.total / 1024 ** 3, 1)
389
- ram_used_gb = round((mem.total - mem.available) / 1024 ** 3, 1)
390
- ram_pct = mem.percent
391
-
392
- # Disk in GB
393
- disk_total_gb = round(disk.total / 1024 ** 3, 1)
394
- disk_free_gb = round(disk.free / 1024 ** 3, 1)
395
- disk_pct = round(disk.percent, 1)
396
-
397
- # CPU
398
- cpu_pct = psutil.cpu_percent(interval=0.2)
399
- cpu_count = psutil.cpu_count(logical=True)
400
- cpu_phys = psutil.cpu_count(logical=False) or cpu_count
401
-
402
- # CPU brand (Linux: read /proc/cpuinfo)
403
- cpu_brand = "CPU"
404
- try:
405
- with open("/proc/cpuinfo") as f:
406
- for line in f:
407
- if "model name" in line:
408
- cpu_brand = line.split(":", 1)[1].strip()
409
- # Shorten common long strings
410
- cpu_brand = cpu_brand.replace("(R)", "").replace("(TM)", "").strip()
411
- break
412
- except Exception:
413
- pass
414
-
415
- cpu_mhz = round(cpu_freq.current, 0) if cpu_freq else None
416
-
417
- # GPU availability detection
418
- gpu_available = False
419
- try:
420
- import torch
421
- gpu_available = torch.cuda.is_available()
422
- except Exception:
423
- pass
424
-
425
- return jsonify({
426
- "cpu_brand": cpu_brand,
427
- "cpu_cores": cpu_count,
428
- "cpu_phys": cpu_phys,
429
- "cpu_mhz": cpu_mhz,
430
- "cpu_pct": cpu_pct,
431
- "ram_total_gb": ram_total_gb,
432
- "ram_used_gb": ram_used_gb,
433
- "ram_pct": ram_pct,
434
- "disk_total_gb": disk_total_gb,
435
- "disk_free_gb": disk_free_gb,
436
- "disk_pct": disk_pct,
437
- "hf_mode": config.HF_MODE,
438
- "active_graph_tasks": _active_graph_tasks,
439
- "gpu_available": gpu_available,
440
- })
441
- except Exception as exc:
442
- log.warning("sysinfo failed: %s", exc)
443
- return jsonify({"error": str(exc)}), 500
444
-
445
-
446
- @app.route("/api/documents")
447
- def list_documents():
448
- token = get_session_token()
449
- docs = vector_store.list_documents(session_token=token)
450
- return jsonify({"documents": docs, "total": len(docs)})
451
-
452
-
453
- # ── Admin Controls ────────────────────────────────────────────────────────────
454
-
455
- @app.route("/api/docker/<action>", methods=["POST"])
456
- def docker_control(action: str):
457
- """Control Kuzu docker container. action: up | down | restart"""
458
- if not config.ADMIN_MODE:
459
- return jsonify({"error": "Admin mode is disabled on this deployment."}), 403
460
- if not is_admin():
461
- return jsonify({"error": "Only admins can control docker containers."}), 403
462
- if action not in ("up", "down", "restart"):
463
- return jsonify({"error": f"Unknown action '{action}'. Use: up, down, restart"}), 400
464
- log.info("Docker action requested: %s", action)
465
- ok, msg = _run_docker(action)
466
- return jsonify({"ok": ok, "action": action, "output": msg}), (200 if ok else 500)
467
-
468
- @app.route("/api/admin/purge", methods=["POST"])
469
- def admin_purge():
470
- """Wipe all databases clean."""
471
- if not config.ADMIN_MODE:
472
- return jsonify({"error": "Admin mode is disabled on this deployment."}), 403
473
- if not is_admin():
474
- return jsonify({"error": "Admin only"}), 403
475
- try:
476
- vector_store.purge()
477
- graph_store.purge()
478
- global _jobs
479
- _jobs.clear()
480
- trigger_kv_cache_update("admin")
481
- log.warning("Admin triggered database purge.")
482
- return jsonify({"ok": True, "msg": "Databases purged successfully."})
483
- except Exception as e:
484
- log.error("Failed to purge databases: %s", e)
485
- return jsonify({"ok": False, "error": str(e)}), 500
486
-
487
- @app.route("/api/admin/kill", methods=["POST"])
488
- def admin_kill():
489
- """Abruptly stop Docker containers and terminate the Flask application."""
490
- if not config.ADMIN_MODE:
491
- return jsonify({"error": "Admin mode is disabled on this deployment."}), 403
492
- if not is_admin():
493
- return jsonify({"error": "Admin only"}), 403
494
-
495
- log.error("KILL SWITCH ACTIVATED. Shutting down docker and terminating process.")
496
- _run_docker("down")
497
-
498
- def _shutdown():
499
- import time
500
- time.sleep(1) # Allow HTTP response to send
501
- os._exit(0)
502
- threading.Thread(target=_shutdown, daemon=True).start()
503
- return jsonify({"ok": True, "msg": "Kill switch activated. Application terminating."})
504
-
505
-
506
- # ── Ingestion ─────────────────────────────────────────────────────────────────
507
-
508
- def _extract_entities_async(
509
- docs: list[dict],
510
- orig_name: str,
511
- tier: str,
512
- token: str,
513
- ) -> None:
514
- """Fire-and-forget entity extraction → Kuzu graph using fast local spaCy pipeline (non-LLM)."""
515
- if not graph_store.is_available():
516
- return
517
-
518
- global _active_graph_tasks
519
- _active_graph_tasks += 1
520
- try:
521
- import spacy
522
- try:
523
- nlp = spacy.load("en_core_web_sm")
524
- except OSError:
525
- log.warning("spaCy model 'en_core_web_sm' not found. Attempting to download...")
526
- try:
527
- import spacy.cli
528
- spacy.cli.download("en_core_web_sm")
529
- nlp = spacy.load("en_core_web_sm")
530
- except Exception as e:
531
- log.error("Failed to download or load spaCy model 'en_core_web_sm': %s. Graph extraction skipped.", e)
532
- return
533
-
534
- text = "\n\n".join(d["text"] for d in docs)
535
-
536
- # spaCy max length limit
537
- if len(text) > 1000000:
538
- text = text[:1000000]
539
-
540
- log.info("Entity extraction (spaCy) starting for %s...", orig_name)
541
- doc = nlp(text)
542
-
543
- entities = []
544
- # Group entities by sentence to establish co-occurrence relationships
545
- for sent in doc.sents:
546
- # Filter for specific entity types
547
- sent_ents = [ent for ent in sent.ents if ent.label_ in {"PERSON", "ORG", "GPE", "LOC", "FAC", "PRODUCT", "EVENT", "WORK_OF_ART", "LAW"}]
548
- if not sent_ents:
549
- continue
550
-
551
- # Map spaCy labels to our schema types
552
- def _map_type(label: str) -> str:
553
- if label == "PERSON": return "Person"
554
- if label == "ORG": return "Organization"
555
- if label in {"GPE", "LOC", "FAC"}: return "Location"
556
- if label == "EVENT": return "Event"
557
- if label == "PRODUCT": return "Object"
558
- if label in {"WORK_OF_ART", "LAW"}: return "Rule"
559
- return "Concept"
560
-
561
- # Create entity objects and cross-link within the same sentence
562
- for i, ent1 in enumerate(sent_ents):
563
- name1 = ent1.text.strip()
564
- if not name1 or len(name1) < 2:
565
- continue
566
-
567
- relations = []
568
- for j, ent2 in enumerate(sent_ents):
569
- if i != j:
570
- name2 = ent2.text.strip()
571
- if name2 and name2 != name1:
572
- relations.append({"target": name2, "rel": "RELATED_TO"})
573
-
574
- # Deduplicate relations
575
- unique_rels = []
576
- seen_targets = set()
577
- for r in relations:
578
- if r["target"] not in seen_targets:
579
- seen_targets.add(r["target"])
580
- unique_rels.append(r)
581
-
582
- entities.append({
583
- "name": name1,
584
- "type": _map_type(ent1.label_),
585
- "relations": unique_rels
586
- })
587
-
588
- # Deduplicate the entities list by name before sending to Kuzu
589
- dedup_entities = {}
590
- for ent in entities:
591
- if ent["name"] not in dedup_entities:
592
- dedup_entities[ent["name"]] = ent
593
- else:
594
- # Merge relations
595
- existing_rels = {r["target"] for r in dedup_entities[ent["name"]]["relations"]}
596
- for rel in ent["relations"]:
597
- if rel["target"] not in existing_rels:
598
- dedup_entities[ent["name"]]["relations"].append(rel)
599
- existing_rels.add(rel["target"])
600
-
601
- final_entities = list(dedup_entities.values())
602
-
603
- if final_entities:
604
- graph_store.store_entities(final_entities, orig_name, tier=tier, session_token=token)
605
- log.info("Entity extraction (spaCy) for %s: %d unique entities stored in Kuzu", orig_name, len(final_entities))
606
- else:
607
- log.info("Entity extraction (spaCy) for %s: No entities found", orig_name)
608
-
609
- except Exception as exc:
610
- log.warning("Entity extraction background task failed for %s: %s", orig_name, exc)
611
- finally:
612
- _active_graph_tasks -= 1
613
-
614
- def process_document_pipeline(path: str, orig_name: str, tier: str, token: str, delete_after: bool = True) -> dict:
615
- step_log = []
616
- added = 0
617
- try:
618
- step_log.append(f"[{orig_name}] Starting ingestion pipeline…")
619
- log.info("Ingesting %s", orig_name)
620
-
621
- # Step 1: load
622
- step_log.append(f"[{orig_name}] Loading document…")
623
- docs = document_loader.load_document(path)
624
- step_log.append(f"[{orig_name}] Loaded {len(docs)} page(s).")
625
- log.info("%s loaded — %d pages", orig_name, len(docs))
626
-
627
- # Step 2: chunk
628
- step_log.append(f"[{orig_name}] Chunking…")
629
- chunks = chunker.chunk_documents(docs)
630
- if not chunks:
631
- raise ValueError("No text could be extracted from this document.")
632
- step_log.append(f"[{orig_name}] Created {len(chunks)} chunks.")
633
- log.info("%s → %d chunks", orig_name, len(chunks))
634
-
635
- # Step 3: embed
636
- step_log.append(f"[{orig_name}] Embedding via embed_llm (port 8003)…")
637
- texts = [c["text"] for c in chunks]
638
- embeddings = embedder.embed_texts(texts)
639
- step_log.append(f"[{orig_name}] Embedded {len(embeddings)} vectors (dim={len(embeddings[0]) if embeddings else '?'}).")
640
- log.info("%s embedded", orig_name)
641
-
642
- # Step 4: store in vector DB
643
- step_log.append(f"[{orig_name}] Storing in ChromaDB (tier: {tier}, session: {token})…")
644
- if config.HF_MODE and vector_store.count() + len(chunks) > 10000:
645
- allowed = 10000 - vector_store.count()
646
- if allowed <= 0:
647
- raise ValueError("Vector database full (10000 chunk limit).")
648
- chunks = chunks[:allowed]
649
- embeddings = embeddings[:allowed]
650
- step_log.append(f"[{orig_name}] WARNING: Truncated to {allowed} chunks due to global 10000 chunk limit.")
651
-
652
- doc_id = uuid.uuid4().hex[:8]
653
- added = vector_store.add_chunks(chunks, embeddings, doc_id, tier=tier, session_token=token)
654
- step_log.append(f"[{orig_name}] Stored {added} chunks in vector DB (doc_id={doc_id}).")
655
- log.info("%s stored %d chunks in ChromaDB", orig_name, added)
656
-
657
- # Step 5: entity extraction → graph (non-blocking — runs in daemon thread)
658
- if graph_store.is_available():
659
- step_log.append(f"[{orig_name}] Entity extraction queued (background thread)…")
660
- threading.Thread(
661
- target=_extract_entities_async,
662
- args=(docs, orig_name, tier, token),
663
- daemon=True,
664
- name=f"entity-{orig_name[:20]}",
665
- ).start()
666
- else:
667
- step_log.append(f"[{orig_name}] Kuzu offline — graph extraction skipped.")
668
-
669
- return {"ok": True, "result": f"Ingested {added} chunks", "log": step_log, "added": added}
670
- except Exception as exc:
671
- step_log.append(f"[{orig_name}] ERROR: {exc}")
672
- log.exception("Ingestion failed for %s", orig_name)
673
- return {"ok": False, "result": str(exc), "log": step_log, "added": added}
674
- finally:
675
- if delete_after and os.path.exists(path):
676
- try:
677
- os.remove(path)
678
- log.info("Deleted local upload file: %s", path)
679
- except OSError as e:
680
- log.warning("Failed to delete %s: %s", path, e)
681
-
682
-
683
-
684
- @app.route("/api/ingest", methods=["POST"])
685
- @limiter.limit("10 per minute")
686
- def ingest():
687
- """Upload and asynchronously ingest one or more documents."""
688
- log.info("Ingest request received. Files in request: %s",
689
- list(request.files.keys()))
690
-
691
- if "files" not in request.files:
692
- log.warning("No 'files' key in request.files")
693
- return jsonify({"error": "No files uploaded — send a multipart/form-data POST with field name 'files'"}), 400
694
-
695
- files = request.files.getlist("files")
696
- tier = request.form.get("tier", "extended")
697
- token = get_session_token()
698
- log.info("Received %d file(s): %s to tier: %s (session: %s)", len(files), [f.filename for f in files], tier, token)
699
-
700
- if tier == "foundation" and not is_admin():
701
- return jsonify({"error": "Only admins can upload to the Foundation tier."}), 403
702
-
703
- if not files or all(not f.filename for f in files):
704
- return jsonify({"error": "File list is empty or filenames are blank"}), 400
705
-
706
- # ── Security Limits ──
707
- if config.HF_MODE:
708
- current_uploads = _session_uploads.get(token, 0)
709
- if current_uploads + len(files) > 5:
710
- return jsonify({"error": f"Session limit exceeded. You can only upload 5 files per session. (Current: {current_uploads})"}), 429
711
-
712
- current_chunks = vector_store.count()
713
- if current_chunks >= 10000:
714
- return jsonify({"error": "Vector database is full (10000 chunk limit reached). Please wait for an admin to purge."}), 429
715
-
716
- _session_uploads[token] = current_uploads + len(files)
717
-
718
- job_id = uuid.uuid4().hex[:8]
719
- saved_paths = []
720
- rejected = []
721
-
722
- for f in files:
723
- if not f.filename:
724
- rejected.append("(unnamed file)")
725
- continue
726
- if not _allowed(f.filename):
727
- ext = Path(f.filename).suffix or "(no extension)"
728
- rejected.append(f"{f.filename} — unsupported type '{ext}'")
729
- log.warning("Rejected file %s — extension not in ALLOWED_EXTENSIONS", f.filename)
730
- continue
731
- dest_dir = Path(__file__).parent / "kbdocs"
732
- dest_dir.mkdir(parents=True, exist_ok=True)
733
- dest = os.path.join(str(dest_dir), Path(f.filename).name)
734
- try:
735
- f.save(dest)
736
- file_size = os.path.getsize(dest)
737
- log.info("Saved %s → %s (%d bytes)", f.filename, dest, file_size)
738
- saved_paths.append((dest, f.filename))
739
- except Exception as exc:
740
- rejected.append(f"{f.filename} — save failed: {exc}")
741
- log.error("Failed to save %s: %s", f.filename, exc)
742
-
743
- if not saved_paths:
744
- msg = "No valid files found."
745
- if rejected:
746
- msg += " Rejected: " + "; ".join(rejected)
747
- log.error("Ingest aborted — %s", msg)
748
- return jsonify({"error": msg}), 400
749
-
750
- _jobs[job_id] = {
751
- "status": "running",
752
- "results": [],
753
- "total": len(saved_paths),
754
- "rejected": rejected,
755
- "log": [],
756
- }
757
- log.info("Job %s created for %d file(s)", job_id, len(saved_paths))
758
-
759
- def _worker(sess_token):
760
- config.current_session.set(sess_token)
761
- for path, orig_name in saved_paths:
762
- res = process_document_pipeline(path, orig_name, tier, token, delete_after=False)
763
- res["file"] = orig_name
764
- _jobs[job_id]["results"].append(res)
765
- _jobs[job_id]["log"].extend(res["log"])
766
-
767
- _jobs[job_id]["status"] = "done"
768
- log.info("Job %s complete — %d results", job_id,
769
- len(_jobs[job_id]["results"]))
770
- trigger_kv_cache_update(sess_token)
771
-
772
- threading.Thread(target=_worker, args=(token,), daemon=True).start()
773
- return jsonify({
774
- "job_id": job_id,
775
- "files": [p[1] for p in saved_paths],
776
- "rejected": rejected,
777
- })
778
-
779
-
780
- @app.route("/api/ingest/status/<job_id>")
781
- def ingest_status(job_id: str):
782
- job = _jobs.get(job_id)
783
- if not job:
784
- return jsonify({"error": "Unknown job"}), 404
785
- return jsonify(job)
786
-
787
-
788
- @app.route("/api/documents/<path:source_name>", methods=["DELETE"])
789
- def delete_document(source_name: str):
790
- tier = request.args.get("tier", "extended")
791
- log.info("Delete request for: %s (tier: %s)", source_name, tier)
792
-
793
- if tier == "foundation" and not is_admin():
794
- return jsonify({"error": "Only admins can delete from the Foundation tier."}), 403
795
-
796
- token = get_session_token()
797
-
798
- deleted_vec = vector_store.delete_document(source_name, session_token=token)
799
- graph_store.delete_source(source_name, session_token=token)
800
- log.info("Deleted %d chunks for '%s'", deleted_vec, source_name)
801
-
802
- trigger_kv_cache_update(token)
803
-
804
- return jsonify({"deleted_chunks": deleted_vec, "source": source_name})
805
-
806
-
807
- # ── Query ─────────────────────────────────────────────────────────────────────
808
-
809
- _query_jobs = {}
810
-
811
- @app.route("/api/query/start", methods=["POST"])
812
- @limiter.limit("120 per minute")
813
- def query_start():
814
- """Starts a RAG query job and returns a job_id."""
815
- data = request.get_json()
816
- q = escape((data or {}).get("query", "").strip())
817
- top_k = (data or {}).get("top_k")
818
- max_tokens = (data or {}).get("max_tokens")
819
- use_vector = (data or {}).get("use_vector", True)
820
- use_graph = False # Disabled per request
821
- use_bm25 = False # Disabled per request
822
- use_gpu = bool((data or {}).get("use_gpu", False))
823
- cpu_threads = int((data or {}).get("cpu_threads", 2))
824
- llm_mode = (data or {}).get("llm_mode", "expert")
825
-
826
- if not q:
827
- return jsonify({"error": "Empty query"}), 400
828
-
829
- token = get_session_token()
830
- chunk_count = vector_store.count()
831
- if chunk_count == 0:
832
- return jsonify({"error": "No documents ingested yet. Please upload documents first."}), 400
833
-
834
- log.info("Query received (%d chars) | vector store has %d chunks", len(q), chunk_count)
835
- remote_addr = request.remote_addr
836
-
837
- job_id = uuid.uuid4().hex[:8]
838
- _query_jobs[job_id] = {
839
- "events": [],
840
- "done": False,
841
- "error": None
842
- }
843
-
844
- def _run():
845
- config.current_session.set(token)
846
- try:
847
- def cb(status):
848
- if isinstance(status, dict):
849
- _query_jobs[job_id]["events"].append(status)
850
- else:
851
- _query_jobs[job_id]["events"].append({"status": status})
852
-
853
- ans, metrics = run_query_crew(q, top_k=top_k, max_tokens=max_tokens, use_vector=use_vector, use_graph=use_graph, use_bm25=use_bm25, session_token=token, status_callback=cb, use_gpu=use_gpu, cpu_threads=cpu_threads, llm_mode=llm_mode)
854
-
855
- for i in range(0, len(ans), 80):
856
- _query_jobs[job_id]["events"].append({"chunk": ans[i:i + 80]})
857
- _query_jobs[job_id]["events"].append({"metrics": metrics})
858
- _query_jobs[job_id]["events"].append({"done": True})
859
- _query_jobs[job_id]["done"] = True
860
-
861
- gen_time = metrics.get("time_seconds", 0)
862
- log_query_summary(token, remote_addr, q, top_k or 10, gen_time, True, "", job_id)
863
- except Exception as e:
864
- log.exception("Query failed")
865
- _query_jobs[job_id]["events"].append({"error": str(e)})
866
- _query_jobs[job_id]["done"] = True
867
- log_query_summary(token, remote_addr, q, top_k or 10, 0, False, str(e), job_id)
868
-
869
- _query_executor.submit(_run)
870
-
871
- return jsonify({"job_id": job_id})
872
-
873
- @app.route("/api/query/stream/<job_id>")
874
- def query_stream(job_id):
875
- """Streams events for a specific query job starting from an offset."""
876
- offset = int(request.args.get("offset", 0))
877
- job = _query_jobs.get(job_id)
878
-
879
- if not job:
880
- return jsonify({"error": "Job not found or expired"}), 404
881
-
882
- def _generate():
883
- import time
884
- idx = offset
885
- while True:
886
- while idx < len(job["events"]):
887
- event = job["events"][idx]
888
- yield f"data: {json.dumps(event)}\n\n"
889
- if "error" in event or "done" in event:
890
- return
891
- idx += 1
892
-
893
- if job.get("error") or job.get("done"):
894
- break
895
-
896
- time.sleep(0.5)
897
- yield ": keep-alive\n\n"
898
-
899
- return Response(
900
- stream_with_context(_generate()),
901
- mimetype="text/event-stream",
902
- headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
903
- )
904
-
905
- @app.route("/api/feedback", methods=["POST"])
906
- @limiter.limit("20 per minute")
907
- def api_feedback():
908
- data = request.get_json() or {}
909
- job_id = data.get("job_id", "")
910
- rating = data.get("rating", "")
911
- stars = data.get("stars", None)
912
- text = data.get("text", "")
913
- token = get_session_token()
914
-
915
- try:
916
- log_dir = Path(__file__).parent / "app" / "logs"
917
- log_dir.mkdir(parents=True, exist_ok=True)
918
- summary_file = log_dir / "nitdaa_summary.json"
919
-
920
- from datetime import datetime, timezone, timedelta
921
- ist = timezone(timedelta(hours=5, minutes=30))
922
- ts = datetime.now(ist).strftime("%Y-%m-%d %H:%M:%S")
923
-
924
- entry = {
925
- "timestamp": ts,
926
- "session_token": token,
927
- "job_id": job_id,
928
- "type": "feedback",
929
  }
930
- if rating:
931
- entry["feedback_rating"] = rating
932
- if stars is not None:
933
- entry["feedback_stars"] = stars
934
- if text:
935
- entry["feedback_text"] = text
936
-
937
- with open(summary_file, "a") as f:
938
- f.write(json.dumps(entry) + "\n")
939
-
940
- async_sync_log(str(summary_file), "nitdaa_summary.json")
941
- return jsonify({"ok": True})
942
- except Exception as e:
943
- log.error(f"Failed to save feedback: {e}")
944
- return jsonify({"error": str(e)}), 500
945
-
946
-
947
- # ── Headless API v1 ───────────────────────────────────────────────────────────
948
-
949
- @app.route("/api/v1/query", methods=["POST"])
950
- def query_v1():
951
- """Headless RAG query — synchronous JSON response."""
952
- data = request.get_json()
953
- q = (data or {}).get("query", "").strip()
954
- top_k = (data or {}).get("top_k")
955
- llm_mode = (data or {}).get("llm_mode", "expert")
956
- if not q:
957
- return jsonify({"error": "Empty query"}), 400
958
-
959
- token = get_session_token()
960
- chunk_count = vector_store.count()
961
- if chunk_count == 0:
962
- return jsonify({"error": "No documents ingested yet."}), 400
963
-
964
- log.info("v1 Query received (%d chars) | session: %s", len(q), token)
965
- config.current_session.set(token)
966
- try:
967
- ans, metrics = run_query_crew(q, top_k=top_k, session_token=token, llm_mode=llm_mode)
968
- return jsonify({"answer": ans, "metrics": metrics})
969
- except Exception as e:
970
- log.exception("v1 Query pipeline error")
971
- return jsonify({"error": str(e)}), 500
972
-
973
- @app.route("/api/v1/ingest/sync", methods=["POST"])
974
- def ingest_v1_sync():
975
- """Headless synchronous document ingestion."""
976
- if "files" not in request.files:
977
- return jsonify({"error": "No files uploaded"}), 400
978
-
979
- files = request.files.getlist("files")
980
- tier = request.form.get("tier", "extended")
981
- token = get_session_token()
982
-
983
- if tier == "foundation" and not is_admin():
984
- return jsonify({"error": "Only admins can upload to the Foundation tier."}), 403
985
-
986
- saved_paths = []
987
- rejected = []
988
- for f in files:
989
- if not f.filename: continue
990
- if not _allowed(f.filename):
991
- rejected.append(f.filename)
992
- continue
993
- dest_dir = Path(__file__).parent / "kbdocs"
994
- dest_dir.mkdir(parents=True, exist_ok=True)
995
- dest = os.path.join(str(dest_dir), Path(f.filename).name)
996
- f.save(dest)
997
- saved_paths.append((dest, f.filename))
998
-
999
- if not saved_paths:
1000
- return jsonify({"error": "No valid files", "rejected": rejected}), 400
1001
-
1002
- config.current_session.set(token)
1003
- results = []
1004
-
1005
- for path, orig_name in saved_paths:
1006
- try:
1007
- docs = document_loader.load_document(path)
1008
- chunks = chunker.chunk_documents(docs)
1009
- if not chunks:
1010
- raise ValueError("No text extracted")
1011
-
1012
- texts = [c["text"] for c in chunks]
1013
- embeddings = embedder.embed_texts(texts)
1014
-
1015
- doc_id = uuid.uuid4().hex[:8]
1016
- added = vector_store.add_chunks(chunks, embeddings, doc_id, tier=tier, session_token=token)
1017
-
1018
- # Entity extraction is fire-and-forget (non-blocking)
1019
- if graph_store.is_available():
1020
- threading.Thread(
1021
- target=_extract_entities_async,
1022
- args=(docs, orig_name, tier, token),
1023
- daemon=True,
1024
- name=f"entity-{orig_name[:20]}",
1025
- ).start()
1026
-
1027
- results.append({
1028
- "file": orig_name,
1029
- "status": "success",
1030
- "chunks_added": added,
1031
- "entities_queued": graph_store.is_available(),
1032
- })
1033
- except Exception as e:
1034
- results.append({"file": orig_name, "status": "error", "error": str(e)})
1035
- finally:
1036
- pass # delete_after is False for these sync uploads
1037
-
1038
- trigger_kv_cache_update(token)
1039
- return jsonify({"results": results, "rejected": rejected})
1040
-
1041
-
1042
- # ── LLM probe endpoints (used by default prompt buttons) ─────────────────────
1043
-
1044
- @app.route("/api/probe/gen", methods=["POST"])
1045
- def probe_gen():
1046
- """Quick smoke-test for the nvidia_llm server."""
1047
- import requests as req
1048
- try:
1049
- r = req.post(
1050
- config.LLM_COMPLETIONS_URL,
1051
- json={"prompt": "Hello, reply with one sentence.", "max_tokens": 64,
1052
- "temperature": 0.7, "top_p": 0.9},
1053
- timeout=60,
1054
- )
1055
- r.raise_for_status()
1056
- data = r.json()
1057
- text = data["choices"][0]["text"].strip()
1058
- return jsonify({"ok": True, "model": data.get("model"), "response": text})
1059
- except Exception as exc:
1060
- log.error("probe_gen failed: %s", exc)
1061
- return jsonify({"ok": False, "error": str(exc)}), 502
1062
-
1063
-
1064
- @app.route("/api/probe/embed", methods=["POST"])
1065
- def probe_embed():
1066
- """Quick smoke-test for the embed_llm server."""
1067
- import requests as req
1068
- try:
1069
- r = req.post(
1070
- config.EMBED_EMBEDDINGS_URL,
1071
- json={"input": "Document test sentence."},
1072
- timeout=60,
1073
  )
1074
- r.raise_for_status()
1075
- data = r.json()
1076
- vec = data["data"][0]["embedding"]
1077
- return jsonify({
1078
- "ok": True,
1079
- "model": data.get("model"),
1080
- "dim": len(vec),
1081
- "sample": vec[:5],
1082
- })
1083
- except Exception as exc:
1084
- log.error("probe_embed failed: %s", exc)
1085
- return jsonify({"ok": False, "error": str(exc)}), 502
1086
-
1087
-
1088
- def start_auto_ingest_thread():
1089
- def _auto_ingest_worker():
1090
- global _auto_ingest_status
1091
- import requests, time, shutil, os
1092
- from huggingface_hub import snapshot_download, hf_hub_download
1093
- from pathlib import Path
1094
-
1095
- token = os.environ.get("HF_PRIVATE_TOKEN") or os.environ.get("HF_TOKEN")
1096
-
1097
- # --- Wait for LLM services to boot before doing anything ---
1098
- log.info("Auto-ingest: waiting for LLM services to boot...")
1099
- for _ in range(30):
1100
- try:
1101
- r1 = requests.get(f"{config.EMBED_BASE_URL}/health", timeout=2)
1102
- r2 = requests.get(f"{config.LLM_BASE_URL}/health", timeout=2)
1103
- if r1.status_code == 200 and r2.status_code == 200:
1104
- break
1105
- except Exception:
1106
- pass
1107
- time.sleep(2)
1108
- else:
1109
- log.warning("Auto-ingest aborted: LLM services not online.")
1110
- _auto_ingest_status["error"] = "LLM services not online within 60s"
1111
- _auto_ingest_status["done"] = True
1112
- return
1113
-
1114
- if not token:
1115
- log.error("HF_PRIVATE_TOKEN or HF_TOKEN environment variable is not set. Dataset synchronization will be skipped.")
1116
- _auto_ingest_status["error"] = "HF Token missing"
1117
- _auto_ingest_status["done"] = True
1118
- return
1119
-
1120
- # --- 2-Way Log Sync on Startup ---
1121
- log_dir = Path(__file__).parent / "app" / "logs"
1122
- log_dir.mkdir(parents=True, exist_ok=True)
1123
- try:
1124
- for log_file in ["nitdaa_sessions.json", "nitdaa_summary.json"]:
1125
- local_p = log_dir / log_file
1126
- try:
1127
- dl_path = hf_hub_download(repo_id="Sam-max1/mat_data", filename=log_file, repo_type="dataset", token=token)
1128
- if os.path.exists(dl_path):
1129
- remote_lines = set(open(dl_path).readlines())
1130
- if local_p.exists():
1131
- for line in open(local_p).readlines():
1132
- if line not in remote_lines:
1133
- remote_lines.add(line)
1134
- with open(local_p, "w") as f:
1135
- for line in sorted(list(remote_lines)):
1136
- f.write(line)
1137
- log.info(f"Successfully merged {log_file} from mat_data.")
1138
- except Exception as e:
1139
- log.warning(f"Could not download {log_file} from mat_data (it may not exist yet): {e}")
1140
- except Exception as e:
1141
- log.warning(f"Log sync failed: {e}")
1142
- # ---------------------------------
1143
-
1144
- kbdocs_dir = Path(__file__).parent / "kbdocs"
1145
- kbdocs_dir.mkdir(parents=True, exist_ok=True)
1146
-
1147
- tmp_sync_dir = Path("/tmp/he_data_sync")
1148
- if tmp_sync_dir.exists():
1149
- shutil.rmtree(tmp_sync_dir)
1150
- tmp_sync_dir.mkdir(exist_ok=True)
1151
-
1152
- log.info("Syncing fresh files from Sam-max1/he-data to local /tmp...")
1153
- try:
1154
- snapshot_download(
1155
- repo_id="Sam-max1/he-data",
1156
- repo_type="dataset",
1157
- local_dir=str(tmp_sync_dir),
1158
- token=token,
1159
- ignore_patterns=[".git*"]
1160
- )
1161
- except Exception as e:
1162
- log.error(f"Failed to download he-data dataset: {e}")
1163
- _auto_ingest_status["error"] = f"Download failed: {e}"
1164
- _auto_ingest_status["done"] = True
1165
- return
1166
-
1167
- from pipeline import vector_store, graph_store
1168
-
1169
- local_files = {f.name: f.stat().st_size for f in kbdocs_dir.glob("*.*") if f.is_file()}
1170
- remote_files = {f.name: f.stat().st_size for f in tmp_sync_dir.glob("*.*") if f.is_file()}
1171
-
1172
- is_different = False
1173
- if set(local_files.keys()) != set(remote_files.keys()):
1174
- is_different = True
1175
- else:
1176
- for k in local_files:
1177
- if local_files[k] != remote_files[k]:
1178
- is_different = True
1179
- break
1180
-
1181
- if is_different:
1182
- log.info("Detected changes in Sam-max1/he-data! Purging databases and re-syncing kbdocs.")
1183
- vector_store.purge()
1184
- if graph_store.is_available():
1185
- graph_store.purge()
1186
-
1187
- shutil.rmtree(kbdocs_dir)
1188
- shutil.copytree(tmp_sync_dir, kbdocs_dir)
1189
-
1190
- files_to_ingest = [f for f in kbdocs_dir.glob("*.*") if f.is_file() and _allowed(f.name)]
1191
- if not files_to_ingest:
1192
- log.info("No valid files to ingest in he-data.")
1193
- _auto_ingest_status["done"] = True
1194
- return
1195
-
1196
- config.current_session.set("admin")
1197
- _auto_ingest_status["running"] = True
1198
- _auto_ingest_status["total"] = len(files_to_ingest)
1199
- _auto_ingest_status["completed"] = 0
1200
- _auto_ingest_status["results"] = []
1201
- _auto_ingest_status["done"] = False
1202
-
1203
- for path in files_to_ingest:
1204
- _auto_ingest_status["current_file"] = path.name
1205
- log.info(f"Auto-ingesting file: {path.name}")
1206
- res = process_document_pipeline(str(path), path.name, "foundation", "admin", delete_after=False)
1207
- _auto_ingest_status["completed"] += 1
1208
- _auto_ingest_status["results"].append({
1209
- "file": path.name,
1210
- "ok": res["ok"],
1211
- "result": res["result"],
1212
- })
1213
- if res["ok"]:
1214
- log.info("Auto-ingest successful for %s", path.name)
1215
- else:
1216
- log.error("Auto-ingest failed for %s: %s", path.name, res["result"])
1217
-
1218
- _auto_ingest_status["running"] = False
1219
- _auto_ingest_status["done"] = True
1220
- _auto_ingest_status["current_file"] = None
1221
- trigger_kv_cache_update("admin")
1222
-
1223
- log.info("=== Full Data Re-Ingestion Complete ===")
1224
- else:
1225
- log.info("kbdocs is completely up to date with he-data. No ingestion needed.")
1226
- _auto_ingest_status["done"] = True
1227
-
1228
- log.info(f"Vector DB Chunks: {vector_store.count()}")
1229
- if graph_store.is_available():
1230
- stats = graph_store.get_stats()
1231
- log.info(f"Kuzu DB Nodes: {stats.get('nodes', 0)}, Edges: {stats.get('edges', 0)}")
1232
-
1233
- threading.Thread(target=_auto_ingest_worker, daemon=True).start()
1234
-
1235
 
1236
- @app.route("/api/auto-ingest/status")
1237
- @limiter.exempt
1238
- def auto_ingest_status():
1239
- """Return real-time progress of the background kbdocs auto-ingestion."""
1240
- return jsonify(_auto_ingest_status)
1241
 
1242
 
1243
  if __name__ == "__main__":
1244
- mode_label = "HF / CPU" if config.HF_MODE else "GPU / Desktop"
1245
- admin_label = "ENABLED" if config.ADMIN_MODE else "DISABLED (public mode)"
1246
- run_port = int(os.environ.get("PORT", 5050))
1247
- ui_url = f"http://127.0.0.1:{run_port}" if not config.HF_MODE else "<HF Spaces URL>"
1248
- print("=" * 64)
1249
- print(" HealthExpert — Document AI Expert")
1250
- print(f" UI : {ui_url}")
1251
- print(f" Mode : {mode_label}")
1252
- print(f" Admin : {admin_label}")
1253
- print(f" Gen LLM : {config.LLM_COMPLETIONS_URL} [{config.LLM_MODEL_ID}]")
1254
- print(f" Embed LLM : {config.EMBED_EMBEDDINGS_URL} [{config.EMBEDDING_MODEL}]")
1255
- print(f" ChromaDB : {config.CHROMA_PERSIST_DIR} (embedded)")
1256
- print(f" Kuzu DB : {config.KUZU_DB_PATH} (embedded)")
1257
- print(f" KV Cache : {'DISABLED (HF mode)' if not config.KV_CACHE_ENABLED else 'ENABLED'}")
1258
- print("=" * 64)
1259
-
1260
- import urllib3
1261
- urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
1262
-
1263
- cert_path = str(Path(__file__).parent / "cert.pem")
1264
- key_path = str(Path(__file__).parent / "key.pem")
1265
-
1266
- start_auto_ingest_thread()
1267
-
1268
- # SSL: skip in HF mode (HF Spaces handles TLS termination at their proxy)
1269
- if os.path.exists(cert_path) and os.path.exists(key_path) and not config.HF_MODE:
1270
- app.run(host="0.0.0.0", port=run_port, debug=False, threaded=True,
1271
- ssl_context=(cert_path, key_path))
1272
- else:
1273
- if config.HF_MODE:
1274
- log.info("HF mode — running HTTP (TLS handled by HF Spaces proxy).")
1275
- else:
1276
- log.warning("SSL certificates not found — running in HTTP mode.")
1277
- app.run(host="0.0.0.0", port=run_port, debug=False, threaded=True)
 
1
+ """Maintenance stub — shown while STUP enrollment window is closed."""
2
+
3
+ import gradio as gr
4
+
5
+ def maintenance_ui():
6
+ with gr.Blocks(
7
+ title="NITDAA – Under Maintenance",
8
+ css="""
9
+ body {
10
+ background: linear-gradient(135deg, #0f0c29, #302b63, #24243e);
11
+ min-height: 100vh;
12
+ display: flex;
13
+ align-items: center;
14
+ justify-content: center;
15
+ font-family: 'Segoe UI', system-ui, sans-serif;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
  }
17
+ .gradio-container {
18
+ background: transparent !important;
19
+ max-width: 700px !important;
20
+ margin: 0 auto !important;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
  }
22
+ #card {
23
+ background: rgba(255, 255, 255, 0.07);
24
+ backdrop-filter: blur(16px);
25
+ border: 1px solid rgba(255, 255, 255, 0.15);
26
+ border-radius: 24px;
27
+ padding: 56px 48px;
28
+ box-shadow: 0 24px 64px rgba(0,0,0,0.4);
29
+ text-align: center;
30
+ }
31
+ #icon { font-size: 72px; margin-bottom: 24px; }
32
+ #title {
33
+ font-size: 2.4rem;
34
+ font-weight: 700;
35
+ color: #ffffff;
36
+ margin: 0 0 16px;
37
+ letter-spacing: -0.5px;
38
+ }
39
+ #divider {
40
+ width: 64px;
41
+ height: 4px;
42
+ background: linear-gradient(90deg, #667eea, #764ba2);
43
+ border-radius: 2px;
44
+ margin: 0 auto 28px;
45
+ }
46
+ #msg {
47
+ font-size: 1.2rem;
48
+ color: rgba(255,255,255,0.82);
49
+ line-height: 1.65;
50
+ margin: 0;
51
+ }
52
+ #sub {
53
+ margin-top: 36px;
54
+ font-size: 0.88rem;
55
+ color: rgba(255,255,255,0.4);
56
+ letter-spacing: 0.5px;
57
+ }
58
+ """,
59
+ ) as demo:
60
+ gr.HTML(
61
+ """
62
+ <div id="card">
63
+ <div id="icon">🔧</div>
64
+ <h1 id="title">Site Under Maintenance</h1>
65
+ <div id="divider"></div>
66
+ <p id="msg">
67
+ Will be back after opening of<br>
68
+ <strong style="color:#a78bfa;">STUP enrollment window</strong>
69
+ </p>
70
+ <p id="sub">NITDAA &nbsp;·&nbsp; NIT DAA Assistant</p>
71
+ </div>
72
+ """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
74
 
75
+ return demo
 
 
 
 
76
 
77
 
78
  if __name__ == "__main__":
79
+ app = maintenance_ui()
80
+ app.launch()