lionel-mp's picture
Add systemone: the System One JSON over the Gradio API, for karpo.ai
049e5e2 verified
Raw History Blame Contribute Delete
15.9 kB
import spaces # must be imported before torch
import html
import json
import sys
import time
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
import gradio as gr
import torch
from huggingface_hub import snapshot_download
MODEL_ID = "MachinePulseAI/karpo-decision-31B"
MODEL_PATH = Path(snapshot_download(MODEL_ID))
from karpo_decision.decider import Decider # noqa: E402
from karpo_decision.encoding import LengthError # noqa: E402
from karpo_decision.families.gemma import fused, triton_attention # noqa: E402
from karpo_decision.protocol import QUESTION_TYPES, RequestError, validate # noqa: E402
# The vendored package wraps six pointwise kernels in torch.compile ("fused by torch.compile on
# GPU and eager elsewhere" — the same math). ZeroGPU's forked worker cannot host the compile
# daemon, so point the wrapped names at their eager implementations before the Decider is built.
fused.rms_norm = fused._rms_norm
fused.add_norm = fused._add_norm
fused.add_scale = fused._add_scale
fused.gelu_mul = fused._gelu_mul
fused.rope = fused._rope
fused._fused_norm_rope = fused._norm_rope
decider = Decider.from_pretrained(MODEL_PATH, device="cuda")
# Gemma's global layers attend with head size 512, past flash attention's 256, and cuDNN's FROST kernels are
# sm_100-only; on this Space's sm_120 GPU the dense memory-efficient kernel also rejects GQA. The authors' kernel for
# exactly this hardware is triton_attention ("outputs within bf16 rounding of it" vs the memory-efficient path) —
# their serve engine wires it the same way (serve/engine.py).
triton_attention.enable(decider.model)
# The authors' own example from the model card (Order 1182).
EX_RETURN_STATE = "Order 1182: the customer asks to return a blender bought 45 days ago. Policy: returns within 30 days."
EX_RETURN_Q = {
"eligible": {"type": "noul", "instructions": "Is the return eligible under the policy?"},
"action": {
"type": "choice",
"instructions": "What should the agent do?",
"criteria": {
"refund": "Issue a full refund",
"decline": "Politely decline the return",
"escalate": "Escalate to a supervisor",
},
},
}
EX_INVOICE_STATE = {
"invoice": {
"vendor": "Acme Corp",
"total": 1250.0,
"currency": "USD",
"due_date": "2026-10-02",
"status": "overdue",
"collection_attempts": 2,
}
}
EX_INVOICE_Q = {
"status": {
"type": "choice",
"instructions": "What is the invoice status?",
"criteria": {
"paid": "Invoice is paid",
"overdue": "Invoice is past due",
"draft": "Not sent yet",
},
},
"large": {"type": "noul", "instructions": "Is the total above 1000 USD?"},
"follow_up": {
"type": "score",
"instructions": "How urgently should this invoice be followed up?",
"criteria": ["No follow-up needed", "This week", "Today"],
},
}
EX_INCIDENT_STATE = (
"Checkout has been failing for every customer for the last hour. Three support agents report the "
"same error at the payment step. The public status page still shows all systems operational."
)
EX_INCIDENT_Q = {
"outage": {"type": "noul", "instructions": "Is the service effectively down?"},
"severity": {
"type": "score",
"instructions": "How severe is the customer impact?",
"criteria": ["No impact", "Minor", "Major", "Critical"],
},
"owner": {
"type": "choice",
"instructions": "Which team should own this incident?",
"criteria": {
"technical": "Bugs, errors, outages",
"billing": "Payments and invoices",
"comms": "Status page and communications",
},
},
}
EX_REVIEW_STATE = (
"Review: \"The headphones arrived in two days and sound incredible, but the left earcup crackles "
"after about 20 minutes of use. Support sent a replacement no questions asked.\""
)
EX_REVIEW_Q = {
"sentiment": {
"type": "score",
"instructions": "How positive is the review overall?",
"criteria": ["Very negative", "Negative", "Mixed", "Positive", "Very positive"],
},
"defect": {"type": "noul", "instructions": "Does the review report a hardware defect?"},
"action": {
"type": "choice",
"instructions": "What should the store do next?",
"criteria": {
"restock": "Keep selling as is",
"inspect": "Check the affected batch",
"recall": "Recall the product",
},
},
}
def _dump(value) -> str:
return value if isinstance(value, str) else json.dumps(value, indent=2, ensure_ascii=False)
def _parse_state(state: str):
text = (state or "").strip()
if not text:
raise gr.Error("Please provide a state (plain text or JSON).")
try:
return json.loads(text)
except json.JSONDecodeError:
return text
def _parse_questions(questions: str) -> dict:
try:
parsed = json.loads(questions or "")
except json.JSONDecodeError as error:
raise gr.Error(f"Questions must be valid JSON: {error}")
if not isinstance(parsed, dict) or not parsed:
raise gr.Error("Questions must be a non-empty JSON object keyed by question ID.")
for qid, question in parsed.items():
if not isinstance(question, dict) or question.get("type") not in QUESTION_TYPES:
raise gr.Error(f"{qid}: type must be one of {', '.join(QUESTION_TYPES)}.")
if question["type"] == "choice" and not isinstance(question.get("criteria"), dict):
raise gr.Error(f"{qid}: choice criteria must be an object of option key -> description.")
if question["type"] == "score" and not (
isinstance(question.get("criteria"), list) and len(question["criteria"]) >= 2
):
raise gr.Error(f"{qid}: score criteria must list at least two level descriptions, lowest first.")
if question["type"] == "noul" and not isinstance(question.get("criteria") or {}, dict):
raise gr.Error(f"{qid}: noul criteria, when given, must map false and true to descriptions.")
return parsed
def _render(questions: dict, answers: dict, usage: dict, latency_ms: float) -> str:
cards = []
for qid, question in questions.items():
answer = answers[qid]
kind = answer["type"]
if kind == "noul":
p_true = answer["noul"]
best = "true" if p_true >= 0.5 else "false"
options = [("true", "true", p_true), ("false", "false", 1.0 - p_true)]
headline = f"→ <b>{best}</b> · P(true) = {p_true:.3f}"
elif kind == "choice":
best = answer["choice"]
options = []
for key, desc in question["criteria"].items():
label = f"{key} — {desc}" if desc else key
options.append((key, label, answer["probabilities"][key]))
headline = f"→ <b>{html.escape(str(best))}</b> · confidence {answer['confidence']:.2f}"
else: # score
probabilities = answer["probabilities"]
best = max(probabilities, key=probabilities.get)
options = [(key, f"{key} — {level}", probabilities[key]) for key, level in answer["legend"].items()]
headline = (
f"→ expected level <b>{answer['score']:.2f}</b> of {len(options) - 1} · "
f"most likely <b>{html.escape(str(answer['legend'][best]))}</b> · "
f"confidence {answer['confidence']:.2f}"
)
rows = []
for key, label, value in options:
highlight = "karpo-best" if key == best else ""
rows.append(
f'<div class="karpo-row {highlight}"><div class="karpo-label">{html.escape(str(label))}</div>'
f'<div class="karpo-bar"><div class="karpo-fill" style="width:{value * 100:.1f}%"></div></div>'
f'<div class="karpo-val">{value * 100:.1f}%</div></div>'
)
instructions = question.get("instructions") or qid
cards.append(
f'<div class="karpo-card"><div class="karpo-head"><span class="karpo-id">{html.escape(qid)}</span>'
f'<span class="karpo-type">{kind}</span></div>'
f'<div class="karpo-instr">{html.escape(str(instructions))}</div>'
f'<div class="karpo-answer">{headline}</div>{"".join(rows)}</div>'
)
footer = (
f'{usage["input_tokens"]} input tokens · {usage["sequences"]} sequence(s) · '
f"forward pass {latency_ms:.0f} ms · Powered by "
f'<a href="https://huggingface.co/MachinePulseAI/karpo-decision-31B">karpo-decision (MachinePulse)</a>'
)
return '<div class="karpo-wrap">' + "".join(cards) + f'<div class="karpo-foot">{footer}</div></div>'
def _gpu_duration(request: dict, *args, **kwargs) -> int:
# A cold worker streams ~62.5 GB of weights before the first forward pass; the reference
# forward itself is fast (the card reports a 94 ms median per request on one GPU).
chars = len(str(request.get("state") or "")) + len(str(request.get("questions") or ""))
return int(min(60, 20 + chars / 2000))
@spaces.GPU(duration=_gpu_duration, size="xlarge")
def _forward(request: dict):
torch.cuda.synchronize()
start = time.perf_counter()
response = decider.decide(request)
torch.cuda.synchronize()
latency_ms = (time.perf_counter() - start) * 1000
return response, latency_ms
# karpo.ai's playground calls systemone through Gradio's API: the model's name in its answers, and
# the title of an error about the request itself, whose message karpo-sites-api shows the reader.
SYSTEMONE_MODEL = "karpo-decision-31B"
REQUEST_ERROR_TITLE = "invalid_request"
def systemone(request: str) -> dict:
"""System One for karpo.ai's playground: a request's JSON in, the response's JSON out.
Args:
request: A System One request as JSON text, {"state", "questions"}.
Returns:
{"id", "model", "answers", "usage", "latency_ms"}; latency_ms is the forward pass on the GPU.
"""
try:
body = json.loads(request)
validate(body)
except (json.JSONDecodeError, RequestError) as error:
raise gr.Error(str(error), title=REQUEST_ERROR_TITLE)
try:
response, latency_ms = _forward({"state": body["state"], "questions": body["questions"]})
except (LengthError, RequestError) as error:
raise gr.Error(str(error), title=REQUEST_ERROR_TITLE)
return {
"id": f"decision-{uuid.uuid4().hex}",
"model": SYSTEMONE_MODEL,
"answers": response["answers"],
"usage": {"inputTokens": response["usage"]["input_tokens"], "outputTokens": 0},
"latency_ms": latency_ms,
}
def decide(state: str, questions: str):
"""Answer typed questions about a state with karpo-decision-31B.
Args:
state: The situation to decide on, as plain text or a JSON value.
questions: JSON object mapping question IDs to questions. Each question has a `type`
(`noul` = true/false, `choice` = named options, `score` = ordered levels),
optional `instructions`, and `criteria` (object of option key -> description for
choice, list of level descriptions lowest-first for score, optional false/true
descriptions for noul).
Returns:
An HTML rendering of the per-option probabilities, and the System One JSON response
(answers with probabilities, confidence and usage) for every question.
"""
parsed_questions = _parse_questions(questions)
request = {"state": _parse_state(state), "questions": parsed_questions}
try:
validate(request)
except RequestError as error:
raise gr.Error(str(error))
try:
response, latency_ms = _forward(request)
except LengthError as error:
raise gr.Error(f"Request too long: {error}")
except RequestError as error:
raise gr.Error(f"Invalid request: {error}")
return _render(parsed_questions, response["answers"], response["usage"], latency_ms), response
CSS = """
#col-container { max-width: 1200px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
.karpo-wrap { display: flex; flex-direction: column; gap: 12px; }
.karpo-card { border: 1px solid var(--border-color-primary); border-radius: 10px; padding: 12px 14px; }
.karpo-head { display: flex; gap: 8px; align-items: center; margin-bottom: 2px; }
.karpo-id { font-weight: 700; font-family: var(--font-mono); }
.karpo-type { font-size: 0.75em; padding: 1px 8px; border-radius: 999px; background: var(--color-accent-soft); }
.karpo-instr { opacity: 0.8; font-size: 0.92em; }
.karpo-answer { margin: 6px 0 8px; }
.karpo-row { display: grid; grid-template-columns: minmax(120px, 40%) 1fr 56px; gap: 8px; align-items: center; margin: 3px 0; font-size: 0.9em; }
.karpo-label { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.karpo-bar { height: 12px; border-radius: 6px; background: var(--block-background-fill); border: 1px solid var(--border-color-primary); overflow: hidden; }
.karpo-fill { height: 100%; background: var(--neutral-400); }
.karpo-best .karpo-fill { background: var(--color-accent); }
.karpo-best .karpo-label { font-weight: 600; }
.karpo-val { text-align: right; font-family: var(--font-mono); }
.karpo-foot { font-size: 0.8em; opacity: 0.7; }
"""
with gr.Blocks(title="karpo-decision-31B") as demo:
with gr.Column(elem_id="col-container"):
gr.Markdown(
"# karpo-decision-31B — typed decisions from one forward pass\n"
"[MachinePulseAI/karpo-decision-31B](https://huggingface.co/MachinePulseAI/karpo-decision-31B) "
"is the #1 model on the [Decision Index](https://huggingface.co/spaces/multimodalart/jev-decision-index) "
"(Full 73.9). Give it a **state** — any text or JSON — and a **schema of typed questions**: "
"`noul` (true/false), `choice` (named options) or `score` (ordered levels). "
"It returns a probability for every option of every question in one forward pass; no text is generated. "
"Text only; up to 32,768 tokens per sequence."
)
with gr.Row():
with gr.Column(scale=1):
state = gr.Textbox(
label="State (plain text or JSON)",
lines=5,
placeholder="Describe the situation, paste a JSON record, a support ticket, a log line…",
)
questions = gr.Code(
label="Questions (JSON schema)",
language="json",
value=_dump(EX_RETURN_Q),
lines=18,
)
run = gr.Button("Decide", variant="primary")
with gr.Column(scale=1):
result_html = gr.HTML(label="Decisions")
with gr.Accordion("SystemOne JSON response", open=False):
result_json = gr.JSON(label="Response")
run.click(decide, inputs=[state, questions], outputs=[result_html, result_json])
gr.api(systemone, api_name="systemone")
gr.Examples(
label="Examples",
examples=[
[EX_RETURN_STATE, _dump(EX_RETURN_Q)],
[_dump(EX_INVOICE_STATE), _dump(EX_INVOICE_Q)],
[EX_INCIDENT_STATE, _dump(EX_INCIDENT_Q)],
[EX_REVIEW_STATE, _dump(EX_REVIEW_Q)],
],
inputs=[state, questions],
outputs=[result_html, result_json],
fn=decide,
cache_examples=True,
cache_mode="lazy",
)
if __name__ == "__main__":
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)