import spaces # must be imported before torch import html import json import sys import time import uuid from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) import gradio as gr import torch from huggingface_hub import snapshot_download MODEL_ID = "MachinePulseAI/karpo-decision-31B" MODEL_PATH = Path(snapshot_download(MODEL_ID)) from karpo_decision.decider import Decider # noqa: E402 from karpo_decision.encoding import LengthError # noqa: E402 from karpo_decision.families.gemma import fused, triton_attention # noqa: E402 from karpo_decision.protocol import QUESTION_TYPES, RequestError, validate # noqa: E402 # The vendored package wraps six pointwise kernels in torch.compile ("fused by torch.compile on # GPU and eager elsewhere" — the same math). ZeroGPU's forked worker cannot host the compile # daemon, so point the wrapped names at their eager implementations before the Decider is built. fused.rms_norm = fused._rms_norm fused.add_norm = fused._add_norm fused.add_scale = fused._add_scale fused.gelu_mul = fused._gelu_mul fused.rope = fused._rope fused._fused_norm_rope = fused._norm_rope decider = Decider.from_pretrained(MODEL_PATH, device="cuda") # Gemma's global layers attend with head size 512, past flash attention's 256, and cuDNN's FROST kernels are # sm_100-only; on this Space's sm_120 GPU the dense memory-efficient kernel also rejects GQA. The authors' kernel for # exactly this hardware is triton_attention ("outputs within bf16 rounding of it" vs the memory-efficient path) — # their serve engine wires it the same way (serve/engine.py). triton_attention.enable(decider.model) # The authors' own example from the model card (Order 1182). EX_RETURN_STATE = "Order 1182: the customer asks to return a blender bought 45 days ago. Policy: returns within 30 days." EX_RETURN_Q = { "eligible": {"type": "noul", "instructions": "Is the return eligible under the policy?"}, "action": { "type": "choice", "instructions": "What should the agent do?", "criteria": { "refund": "Issue a full refund", "decline": "Politely decline the return", "escalate": "Escalate to a supervisor", }, }, } EX_INVOICE_STATE = { "invoice": { "vendor": "Acme Corp", "total": 1250.0, "currency": "USD", "due_date": "2026-10-02", "status": "overdue", "collection_attempts": 2, } } EX_INVOICE_Q = { "status": { "type": "choice", "instructions": "What is the invoice status?", "criteria": { "paid": "Invoice is paid", "overdue": "Invoice is past due", "draft": "Not sent yet", }, }, "large": {"type": "noul", "instructions": "Is the total above 1000 USD?"}, "follow_up": { "type": "score", "instructions": "How urgently should this invoice be followed up?", "criteria": ["No follow-up needed", "This week", "Today"], }, } EX_INCIDENT_STATE = ( "Checkout has been failing for every customer for the last hour. Three support agents report the " "same error at the payment step. The public status page still shows all systems operational." ) EX_INCIDENT_Q = { "outage": {"type": "noul", "instructions": "Is the service effectively down?"}, "severity": { "type": "score", "instructions": "How severe is the customer impact?", "criteria": ["No impact", "Minor", "Major", "Critical"], }, "owner": { "type": "choice", "instructions": "Which team should own this incident?", "criteria": { "technical": "Bugs, errors, outages", "billing": "Payments and invoices", "comms": "Status page and communications", }, }, } EX_REVIEW_STATE = ( "Review: \"The headphones arrived in two days and sound incredible, but the left earcup crackles " "after about 20 minutes of use. Support sent a replacement no questions asked.\"" ) EX_REVIEW_Q = { "sentiment": { "type": "score", "instructions": "How positive is the review overall?", "criteria": ["Very negative", "Negative", "Mixed", "Positive", "Very positive"], }, "defect": {"type": "noul", "instructions": "Does the review report a hardware defect?"}, "action": { "type": "choice", "instructions": "What should the store do next?", "criteria": { "restock": "Keep selling as is", "inspect": "Check the affected batch", "recall": "Recall the product", }, }, } def _dump(value) -> str: return value if isinstance(value, str) else json.dumps(value, indent=2, ensure_ascii=False) def _parse_state(state: str): text = (state or "").strip() if not text: raise gr.Error("Please provide a state (plain text or JSON).") try: return json.loads(text) except json.JSONDecodeError: return text def _parse_questions(questions: str) -> dict: try: parsed = json.loads(questions or "") except json.JSONDecodeError as error: raise gr.Error(f"Questions must be valid JSON: {error}") if not isinstance(parsed, dict) or not parsed: raise gr.Error("Questions must be a non-empty JSON object keyed by question ID.") for qid, question in parsed.items(): if not isinstance(question, dict) or question.get("type") not in QUESTION_TYPES: raise gr.Error(f"{qid}: type must be one of {', '.join(QUESTION_TYPES)}.") if question["type"] == "choice" and not isinstance(question.get("criteria"), dict): raise gr.Error(f"{qid}: choice criteria must be an object of option key -> description.") if question["type"] == "score" and not ( isinstance(question.get("criteria"), list) and len(question["criteria"]) >= 2 ): raise gr.Error(f"{qid}: score criteria must list at least two level descriptions, lowest first.") if question["type"] == "noul" and not isinstance(question.get("criteria") or {}, dict): raise gr.Error(f"{qid}: noul criteria, when given, must map false and true to descriptions.") return parsed def _render(questions: dict, answers: dict, usage: dict, latency_ms: float) -> str: cards = [] for qid, question in questions.items(): answer = answers[qid] kind = answer["type"] if kind == "noul": p_true = answer["noul"] best = "true" if p_true >= 0.5 else "false" options = [("true", "true", p_true), ("false", "false", 1.0 - p_true)] headline = f"→ {best} · P(true) = {p_true:.3f}" elif kind == "choice": best = answer["choice"] options = [] for key, desc in question["criteria"].items(): label = f"{key} — {desc}" if desc else key options.append((key, label, answer["probabilities"][key])) headline = f"→ {html.escape(str(best))} · confidence {answer['confidence']:.2f}" else: # score probabilities = answer["probabilities"] best = max(probabilities, key=probabilities.get) options = [(key, f"{key} — {level}", probabilities[key]) for key, level in answer["legend"].items()] headline = ( f"→ expected level {answer['score']:.2f} of {len(options) - 1} · " f"most likely {html.escape(str(answer['legend'][best]))} · " f"confidence {answer['confidence']:.2f}" ) rows = [] for key, label, value in options: highlight = "karpo-best" if key == best else "" rows.append( f'
{html.escape(str(label))}
' f'
' f'
{value * 100:.1f}%
' ) instructions = question.get("instructions") or qid cards.append( f'
{html.escape(qid)}' f'{kind}
' f'
{html.escape(str(instructions))}
' f'
{headline}
{"".join(rows)}
' ) footer = ( f'{usage["input_tokens"]} input tokens · {usage["sequences"]} sequence(s) · ' f"forward pass {latency_ms:.0f} ms · Powered by " f'karpo-decision (MachinePulse)' ) return '
' + "".join(cards) + f'
{footer}
' def _gpu_duration(request: dict, *args, **kwargs) -> int: # A cold worker streams ~62.5 GB of weights before the first forward pass; the reference # forward itself is fast (the card reports a 94 ms median per request on one GPU). chars = len(str(request.get("state") or "")) + len(str(request.get("questions") or "")) return int(min(60, 20 + chars / 2000)) @spaces.GPU(duration=_gpu_duration, size="xlarge") def _forward(request: dict): torch.cuda.synchronize() start = time.perf_counter() response = decider.decide(request) torch.cuda.synchronize() latency_ms = (time.perf_counter() - start) * 1000 return response, latency_ms # karpo.ai's playground calls systemone through Gradio's API: the model's name in its answers, and # the title of an error about the request itself, whose message karpo-sites-api shows the reader. SYSTEMONE_MODEL = "karpo-decision-31B" REQUEST_ERROR_TITLE = "invalid_request" def systemone(request: str) -> dict: """System One for karpo.ai's playground: a request's JSON in, the response's JSON out. Args: request: A System One request as JSON text, {"state", "questions"}. Returns: {"id", "model", "answers", "usage", "latency_ms"}; latency_ms is the forward pass on the GPU. """ try: body = json.loads(request) validate(body) except (json.JSONDecodeError, RequestError) as error: raise gr.Error(str(error), title=REQUEST_ERROR_TITLE) try: response, latency_ms = _forward({"state": body["state"], "questions": body["questions"]}) except (LengthError, RequestError) as error: raise gr.Error(str(error), title=REQUEST_ERROR_TITLE) return { "id": f"decision-{uuid.uuid4().hex}", "model": SYSTEMONE_MODEL, "answers": response["answers"], "usage": {"inputTokens": response["usage"]["input_tokens"], "outputTokens": 0}, "latency_ms": latency_ms, } def decide(state: str, questions: str): """Answer typed questions about a state with karpo-decision-31B. Args: state: The situation to decide on, as plain text or a JSON value. questions: JSON object mapping question IDs to questions. Each question has a `type` (`noul` = true/false, `choice` = named options, `score` = ordered levels), optional `instructions`, and `criteria` (object of option key -> description for choice, list of level descriptions lowest-first for score, optional false/true descriptions for noul). Returns: An HTML rendering of the per-option probabilities, and the System One JSON response (answers with probabilities, confidence and usage) for every question. """ parsed_questions = _parse_questions(questions) request = {"state": _parse_state(state), "questions": parsed_questions} try: validate(request) except RequestError as error: raise gr.Error(str(error)) try: response, latency_ms = _forward(request) except LengthError as error: raise gr.Error(f"Request too long: {error}") except RequestError as error: raise gr.Error(f"Invalid request: {error}") return _render(parsed_questions, response["answers"], response["usage"], latency_ms), response CSS = """ #col-container { max-width: 1200px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } .karpo-wrap { display: flex; flex-direction: column; gap: 12px; } .karpo-card { border: 1px solid var(--border-color-primary); border-radius: 10px; padding: 12px 14px; } .karpo-head { display: flex; gap: 8px; align-items: center; margin-bottom: 2px; } .karpo-id { font-weight: 700; font-family: var(--font-mono); } .karpo-type { font-size: 0.75em; padding: 1px 8px; border-radius: 999px; background: var(--color-accent-soft); } .karpo-instr { opacity: 0.8; font-size: 0.92em; } .karpo-answer { margin: 6px 0 8px; } .karpo-row { display: grid; grid-template-columns: minmax(120px, 40%) 1fr 56px; gap: 8px; align-items: center; margin: 3px 0; font-size: 0.9em; } .karpo-label { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .karpo-bar { height: 12px; border-radius: 6px; background: var(--block-background-fill); border: 1px solid var(--border-color-primary); overflow: hidden; } .karpo-fill { height: 100%; background: var(--neutral-400); } .karpo-best .karpo-fill { background: var(--color-accent); } .karpo-best .karpo-label { font-weight: 600; } .karpo-val { text-align: right; font-family: var(--font-mono); } .karpo-foot { font-size: 0.8em; opacity: 0.7; } """ with gr.Blocks(title="karpo-decision-31B") as demo: with gr.Column(elem_id="col-container"): gr.Markdown( "# karpo-decision-31B — typed decisions from one forward pass\n" "[MachinePulseAI/karpo-decision-31B](https://huggingface.co/MachinePulseAI/karpo-decision-31B) " "is the #1 model on the [Decision Index](https://huggingface.co/spaces/multimodalart/jev-decision-index) " "(Full 73.9). Give it a **state** — any text or JSON — and a **schema of typed questions**: " "`noul` (true/false), `choice` (named options) or `score` (ordered levels). " "It returns a probability for every option of every question in one forward pass; no text is generated. " "Text only; up to 32,768 tokens per sequence." ) with gr.Row(): with gr.Column(scale=1): state = gr.Textbox( label="State (plain text or JSON)", lines=5, placeholder="Describe the situation, paste a JSON record, a support ticket, a log line…", ) questions = gr.Code( label="Questions (JSON schema)", language="json", value=_dump(EX_RETURN_Q), lines=18, ) run = gr.Button("Decide", variant="primary") with gr.Column(scale=1): result_html = gr.HTML(label="Decisions") with gr.Accordion("SystemOne JSON response", open=False): result_json = gr.JSON(label="Response") run.click(decide, inputs=[state, questions], outputs=[result_html, result_json]) gr.api(systemone, api_name="systemone") gr.Examples( label="Examples", examples=[ [EX_RETURN_STATE, _dump(EX_RETURN_Q)], [_dump(EX_INVOICE_STATE), _dump(EX_INVOICE_Q)], [EX_INCIDENT_STATE, _dump(EX_INCIDENT_Q)], [EX_REVIEW_STATE, _dump(EX_REVIEW_Q)], ], inputs=[state, questions], outputs=[result_html, result_json], fn=decide, cache_examples=True, cache_mode="lazy", ) if __name__ == "__main__": demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)