import spaces # must be imported before torch import html import json import sys import time import uuid from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) import gradio as gr import torch from huggingface_hub import snapshot_download MODEL_ID = "MachinePulseAI/karpo-decision-31B" MODEL_PATH = Path(snapshot_download(MODEL_ID)) from karpo_decision.decider import Decider # noqa: E402 from karpo_decision.encoding import LengthError # noqa: E402 from karpo_decision.families.gemma import fused, triton_attention # noqa: E402 from karpo_decision.protocol import QUESTION_TYPES, RequestError, validate # noqa: E402 # The vendored package wraps six pointwise kernels in torch.compile ("fused by torch.compile on # GPU and eager elsewhere" — the same math). ZeroGPU's forked worker cannot host the compile # daemon, so point the wrapped names at their eager implementations before the Decider is built. fused.rms_norm = fused._rms_norm fused.add_norm = fused._add_norm fused.add_scale = fused._add_scale fused.gelu_mul = fused._gelu_mul fused.rope = fused._rope fused._fused_norm_rope = fused._norm_rope decider = Decider.from_pretrained(MODEL_PATH, device="cuda") # Gemma's global layers attend with head size 512, past flash attention's 256, and cuDNN's FROST kernels are # sm_100-only; on this Space's sm_120 GPU the dense memory-efficient kernel also rejects GQA. The authors' kernel for # exactly this hardware is triton_attention ("outputs within bf16 rounding of it" vs the memory-efficient path) — # their serve engine wires it the same way (serve/engine.py). triton_attention.enable(decider.model) # The authors' own example from the model card (Order 1182). EX_RETURN_STATE = "Order 1182: the customer asks to return a blender bought 45 days ago. Policy: returns within 30 days." EX_RETURN_Q = { "eligible": {"type": "noul", "instructions": "Is the return eligible under the policy?"}, "action": { "type": "choice", "instructions": "What should the agent do?", "criteria": { "refund": "Issue a full refund", "decline": "Politely decline the return", "escalate": "Escalate to a supervisor", }, }, } EX_INVOICE_STATE = { "invoice": { "vendor": "Acme Corp", "total": 1250.0, "currency": "USD", "due_date": "2026-10-02", "status": "overdue", "collection_attempts": 2, } } EX_INVOICE_Q = { "status": { "type": "choice", "instructions": "What is the invoice status?", "criteria": { "paid": "Invoice is paid", "overdue": "Invoice is past due", "draft": "Not sent yet", }, }, "large": {"type": "noul", "instructions": "Is the total above 1000 USD?"}, "follow_up": { "type": "score", "instructions": "How urgently should this invoice be followed up?", "criteria": ["No follow-up needed", "This week", "Today"], }, } EX_INCIDENT_STATE = ( "Checkout has been failing for every customer for the last hour. Three support agents report the " "same error at the payment step. The public status page still shows all systems operational." ) EX_INCIDENT_Q = { "outage": {"type": "noul", "instructions": "Is the service effectively down?"}, "severity": { "type": "score", "instructions": "How severe is the customer impact?", "criteria": ["No impact", "Minor", "Major", "Critical"], }, "owner": { "type": "choice", "instructions": "Which team should own this incident?", "criteria": { "technical": "Bugs, errors, outages", "billing": "Payments and invoices", "comms": "Status page and communications", }, }, } EX_REVIEW_STATE = ( "Review: \"The headphones arrived in two days and sound incredible, but the left earcup crackles " "after about 20 minutes of use. Support sent a replacement no questions asked.\"" ) EX_REVIEW_Q = { "sentiment": { "type": "score", "instructions": "How positive is the review overall?", "criteria": ["Very negative", "Negative", "Mixed", "Positive", "Very positive"], }, "defect": {"type": "noul", "instructions": "Does the review report a hardware defect?"}, "action": { "type": "choice", "instructions": "What should the store do next?", "criteria": { "restock": "Keep selling as is", "inspect": "Check the affected batch", "recall": "Recall the product", }, }, } def _dump(value) -> str: return value if isinstance(value, str) else json.dumps(value, indent=2, ensure_ascii=False) def _parse_state(state: str): text = (state or "").strip() if not text: raise gr.Error("Please provide a state (plain text or JSON).") try: return json.loads(text) except json.JSONDecodeError: return text def _parse_questions(questions: str) -> dict: try: parsed = json.loads(questions or "") except json.JSONDecodeError as error: raise gr.Error(f"Questions must be valid JSON: {error}") if not isinstance(parsed, dict) or not parsed: raise gr.Error("Questions must be a non-empty JSON object keyed by question ID.") for qid, question in parsed.items(): if not isinstance(question, dict) or question.get("type") not in QUESTION_TYPES: raise gr.Error(f"{qid}: type must be one of {', '.join(QUESTION_TYPES)}.") if question["type"] == "choice" and not isinstance(question.get("criteria"), dict): raise gr.Error(f"{qid}: choice criteria must be an object of option key -> description.") if question["type"] == "score" and not ( isinstance(question.get("criteria"), list) and len(question["criteria"]) >= 2 ): raise gr.Error(f"{qid}: score criteria must list at least two level descriptions, lowest first.") if question["type"] == "noul" and not isinstance(question.get("criteria") or {}, dict): raise gr.Error(f"{qid}: noul criteria, when given, must map false and true to descriptions.") return parsed def _render(questions: dict, answers: dict, usage: dict, latency_ms: float) -> str: cards = [] for qid, question in questions.items(): answer = answers[qid] kind = answer["type"] if kind == "noul": p_true = answer["noul"] best = "true" if p_true >= 0.5 else "false" options = [("true", "true", p_true), ("false", "false", 1.0 - p_true)] headline = f"→ {best} · P(true) = {p_true:.3f}" elif kind == "choice": best = answer["choice"] options = [] for key, desc in question["criteria"].items(): label = f"{key} — {desc}" if desc else key options.append((key, label, answer["probabilities"][key])) headline = f"→ {html.escape(str(best))} · confidence {answer['confidence']:.2f}" else: # score probabilities = answer["probabilities"] best = max(probabilities, key=probabilities.get) options = [(key, f"{key} — {level}", probabilities[key]) for key, level in answer["legend"].items()] headline = ( f"→ expected level {answer['score']:.2f} of {len(options) - 1} · " f"most likely {html.escape(str(answer['legend'][best]))} · " f"confidence {answer['confidence']:.2f}" ) rows = [] for key, label, value in options: highlight = "karpo-best" if key == best else "" rows.append( f'