import json import os import re import sys import threading import time import traceback from datetime import datetime, timezone # Hugging Face ZeroGPU compatibility try: import spaces has_spaces = True except ImportError: has_spaces = False def gpu_decorator(duration=120): def decorator(fn): if has_spaces and hasattr(spaces, "GPU"): return spaces.GPU(duration=duration)(fn) return fn return decorator # Database integration try: from src.db import mongo_get_certificate, mongo_save_certificate except ImportError: from db import mongo_get_certificate, mongo_save_certificate # Engine import try: from src.audit.engine import AuditError, check_feasibility, run_full_audit except ImportError: try: from src.engine import AuditError, check_feasibility, run_full_audit except ImportError: try: from audit.engine import AuditError, check_feasibility, run_full_audit except ImportError: from engine import AuditError, check_feasibility, run_full_audit # Formatting import try: from src.display.formatting import ( build_top_3_cards_html, render_audit_details_panel, styled_error, styled_loading, styled_message, ) except ImportError: try: from src.formatting import ( build_top_3_cards_html, render_audit_details_panel, styled_error, styled_loading, styled_message, ) except ImportError: try: from display.formatting import ( build_top_3_cards_html, render_audit_details_panel, styled_error, styled_loading, styled_message, ) except ImportError: from formatting import ( build_top_3_cards_html, render_audit_details_panel, styled_error, styled_loading, styled_message, ) # Utils import try: from src.display.utils import BENCHMARK_COLS, COLS except ImportError: try: from src.utils import BENCHMARK_COLS, COLS except ImportError: try: from display.utils import BENCHMARK_COLS, COLS except ImportError: from utils import BENCHMARK_COLS, COLS # Envs & Populate imports try: from src.envs import AUDIT_DEVICE, EVAL_RESULTS_PATH, MAX_AUDIT_PARAMS_BILLION, TOKEN except ImportError: from envs import AUDIT_DEVICE, EVAL_RESULTS_PATH, MAX_AUDIT_PARAMS_BILLION, TOKEN try: from src.populate import get_leaderboard_df, get_top_3_eval_cards except ImportError: from populate import get_leaderboard_df, get_top_3_eval_cards def clean_model_name(raw_name: str) -> str: """Robust extraction and sanitization of Hugging Face Model IDs.""" if not raw_name: return "" name = str(raw_name).strip() if name.lower() in ("none", "null", "undefined", ""): return "" # Extract from href="..." or markdown [text](url) href_match = re.search(r'href=["\'](?:https?://huggingface\.co/)?([^"\']+)["\']', name) if href_match: name = href_match.group(1) else: md_match = re.search(r'\((?:https?://huggingface\.co/)?([^)]+)\)', name) if md_match: name = md_match.group(1) # Strip HTML tags & markdown name = re.sub(r'<[^>]+>', '', name) name = re.sub(r'\[([^\]]+)\]', r'\1', name) # Clean domain & query strings name = name.replace("https://huggingface.co/", "").replace("http://huggingface.co/", "") name = name.split("?")[0].split("#")[0] # Strip trailing branch paths like /tree/main, /blob/main name = re.sub(r'/(tree|blob|resolve)/.*$', '', name) return name.strip().strip("/") def get_certificate_by_model_name(model_name: str) -> dict: clean_name = clean_model_name(model_name) if not clean_name: return {} # 1. Look up in MongoDB Atlas cert = mongo_get_certificate(clean_name) if cert and cert.get("status") == "ok": return cert # 2. Fallback to local files if os.path.exists(EVAL_RESULTS_PATH): safe_prefix = clean_name.replace("/", "__").lower() for f in os.listdir(EVAL_RESULTS_PATH): if f.lower().startswith(safe_prefix) and f.endswith(".json"): try: with open(os.path.join(EVAL_RESULTS_PATH, f), "r", encoding="utf-8") as fp: return json.load(fp) except Exception: continue return {} def _save_cert(model_id: str, revision: str, cert: dict) -> str: # Only save valid audit certificates to permanent storage if not cert or cert.get("status") != "ok": return "" # 1. Save to local disk cache os.makedirs(EVAL_RESULTS_PATH, exist_ok=True) safe_name = model_id.replace("/", "__") + f"_{revision}.json" out_path = os.path.join(EVAL_RESULTS_PATH, safe_name) try: with open(out_path, "w", encoding="utf-8") as f: json.dump(cert, f, indent=2) except Exception as e: print(f"Local file write error: {e}", flush=True) # 2. Persist permanently to MongoDB Atlas mongo_save_certificate(cert) return out_path @gpu_decorator(duration=120) def execute_direct_xray_audit(model_id: str, revision: str = "main", trust_remote_code: bool = True) -> dict: now_str = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") print(f"[LLM-X-RAY] Verifying repository metadata for '{model_id}'...", flush=True) try: feas = check_feasibility(model_id, revision, TOKEN, MAX_AUDIT_PARAMS_BILLION, trust_remote_code=trust_remote_code) if not feas.ok: return { "status": "error", "config": {"model_name": model_id, "model_sha": revision, "params": feas.param_count_b}, "error_message": feas.reason, "audited_at": now_str, } result = run_full_audit( model_id=model_id, revision=revision, device="cuda" if has_spaces else AUDIT_DEVICE, token=TOKEN, trust_remote_code=trust_remote_code, progress_callback=None, ) return { "status": "ok", "config": { "model_name": model_id, "model_sha": revision, "architecture": feas.architecture or "CausalLM", "params": feas.param_count_b or 0.5, "precision": "bfloat16", "license": "open-source", }, "audited_at": now_str, **result, } except Exception as e: traceback.print_exc() sys.stdout.flush() return { "status": "error", "config": {"model_name": model_id, "model_sha": revision}, "error_message": str(e), "audited_at": now_str, } def audit_or_search_model(url_or_id: str, trust_remote_code: bool = True): clean_model = clean_model_name(url_or_id) current_df = get_leaderboard_df(EVAL_RESULTS_PATH, "", COLS, BENCHMARK_COLS) current_top3 = build_top_3_cards_html(get_top_3_eval_cards(EVAL_RESULTS_PATH)) current_panel = render_audit_details_panel({}) if not clean_model: yield ( styled_error("Please enter a valid Hugging Face Model ID or URL (e.g. Qwen/Qwen2.5-0.5B-Instruct)."), current_df, current_top3, current_panel, ) return if clean_model.startswith("spaces/"): yield ( styled_error(f"{clean_model} is a Hugging Face Space, not a Model. Please enter a Model ID (e.g. Qwen/Qwen2.5-0.5B-Instruct or SupraLabs/Supra2-Nano)."), current_df, current_top3, current_panel, ) return if clean_model.startswith("datasets/"): yield ( styled_error(f"{clean_model} is a Dataset, not a Model. Please enter a Model ID."), current_df, current_top3, current_panel, ) return # Check for existing cert in MongoDB Atlas / local cache (Instant Return) existing_cert = get_certificate_by_model_name(clean_model) if existing_cert and existing_cert.get("status") == "ok": yield ( styled_message(f"Loaded existing audit certificate for {clean_model} from MongoDB Atlas."), current_df, current_top3, render_audit_details_panel(existing_cert), ) return yield ( styled_loading(f"Connecting to HF Hub for {clean_model}...", "Requesting ZeroGPU slice & downloading model weights..."), current_df, current_top3, current_panel, ) result_holder = {} done_event = threading.Event() def _worker(): try: result_holder["cert"] = execute_direct_xray_audit( clean_model, "main", trust_remote_code=trust_remote_code ) except Exception as err: traceback.print_exc() sys.stdout.flush() result_holder["cert"] = { "status": "error", "config": {"model_name": clean_model}, "error_message": str(err), } finally: done_event.set() thread = threading.Thread(target=_worker, daemon=True) thread.start() stages = [ "Downloading model weights and tokenizer tensors...", "Layer A: Performing SVD Spectral Tomography across weight tensors...", "Layer B: Streaming activations and calculating operator covariance...", "Layer C: Running empirical factual probe battery...", "Finalizing operator risk and registering certificate...", ] stage_idx = 0 while not done_event.is_set(): current_stage = stages[min(stage_idx, len(stages) - 1)] yield ( styled_loading(f"🔬 Auditing {clean_model} on ZeroGPU...", current_stage), current_df, current_top3, current_panel, ) done_event.wait(timeout=2.5) stage_idx += 1 thread.join() cert = result_holder.get("cert", {}) # Save to MongoDB and local disk only on success if cert and cert.get("status") == "ok": _save_cert(clean_model, cert.get("config", {}).get("model_sha", "main"), cert) updated_df = get_leaderboard_df(EVAL_RESULTS_PATH, "", COLS, BENCHMARK_COLS) updated_top3 = build_top_3_cards_html(get_top_3_eval_cards(EVAL_RESULTS_PATH)) updated_panel = render_audit_details_panel(cert) if cert.get("status") == "error": err_msg = cert.get("error_message", "Unknown error.") yield ( styled_error(f"Audit could not be completed for {clean_model}: {err_msg}"), updated_df, updated_top3, updated_panel, ) else: yield ( styled_message(f"🎉 Audit Complete for {clean_model}! Persisted in MongoDB Atlas."), updated_df, updated_top3, updated_panel, )