File size: 15,862 Bytes
4fecaf1 049e5e2 4fecaf1 1e5eafe 4fecaf1 1e5eafe 4fecaf1 049e5e2 4fecaf1 049e5e2 4fecaf1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 | import spaces # must be imported before torch
import html
import json
import sys
import time
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
import gradio as gr
import torch
from huggingface_hub import snapshot_download
MODEL_ID = "MachinePulseAI/karpo-decision-31B"
MODEL_PATH = Path(snapshot_download(MODEL_ID))
from karpo_decision.decider import Decider # noqa: E402
from karpo_decision.encoding import LengthError # noqa: E402
from karpo_decision.families.gemma import fused, triton_attention # noqa: E402
from karpo_decision.protocol import QUESTION_TYPES, RequestError, validate # noqa: E402
# The vendored package wraps six pointwise kernels in torch.compile ("fused by torch.compile on
# GPU and eager elsewhere" — the same math). ZeroGPU's forked worker cannot host the compile
# daemon, so point the wrapped names at their eager implementations before the Decider is built.
fused.rms_norm = fused._rms_norm
fused.add_norm = fused._add_norm
fused.add_scale = fused._add_scale
fused.gelu_mul = fused._gelu_mul
fused.rope = fused._rope
fused._fused_norm_rope = fused._norm_rope
decider = Decider.from_pretrained(MODEL_PATH, device="cuda")
# Gemma's global layers attend with head size 512, past flash attention's 256, and cuDNN's FROST kernels are
# sm_100-only; on this Space's sm_120 GPU the dense memory-efficient kernel also rejects GQA. The authors' kernel for
# exactly this hardware is triton_attention ("outputs within bf16 rounding of it" vs the memory-efficient path) —
# their serve engine wires it the same way (serve/engine.py).
triton_attention.enable(decider.model)
# The authors' own example from the model card (Order 1182).
EX_RETURN_STATE = "Order 1182: the customer asks to return a blender bought 45 days ago. Policy: returns within 30 days."
EX_RETURN_Q = {
"eligible": {"type": "noul", "instructions": "Is the return eligible under the policy?"},
"action": {
"type": "choice",
"instructions": "What should the agent do?",
"criteria": {
"refund": "Issue a full refund",
"decline": "Politely decline the return",
"escalate": "Escalate to a supervisor",
},
},
}
EX_INVOICE_STATE = {
"invoice": {
"vendor": "Acme Corp",
"total": 1250.0,
"currency": "USD",
"due_date": "2026-10-02",
"status": "overdue",
"collection_attempts": 2,
}
}
EX_INVOICE_Q = {
"status": {
"type": "choice",
"instructions": "What is the invoice status?",
"criteria": {
"paid": "Invoice is paid",
"overdue": "Invoice is past due",
"draft": "Not sent yet",
},
},
"large": {"type": "noul", "instructions": "Is the total above 1000 USD?"},
"follow_up": {
"type": "score",
"instructions": "How urgently should this invoice be followed up?",
"criteria": ["No follow-up needed", "This week", "Today"],
},
}
EX_INCIDENT_STATE = (
"Checkout has been failing for every customer for the last hour. Three support agents report the "
"same error at the payment step. The public status page still shows all systems operational."
)
EX_INCIDENT_Q = {
"outage": {"type": "noul", "instructions": "Is the service effectively down?"},
"severity": {
"type": "score",
"instructions": "How severe is the customer impact?",
"criteria": ["No impact", "Minor", "Major", "Critical"],
},
"owner": {
"type": "choice",
"instructions": "Which team should own this incident?",
"criteria": {
"technical": "Bugs, errors, outages",
"billing": "Payments and invoices",
"comms": "Status page and communications",
},
},
}
EX_REVIEW_STATE = (
"Review: \"The headphones arrived in two days and sound incredible, but the left earcup crackles "
"after about 20 minutes of use. Support sent a replacement no questions asked.\""
)
EX_REVIEW_Q = {
"sentiment": {
"type": "score",
"instructions": "How positive is the review overall?",
"criteria": ["Very negative", "Negative", "Mixed", "Positive", "Very positive"],
},
"defect": {"type": "noul", "instructions": "Does the review report a hardware defect?"},
"action": {
"type": "choice",
"instructions": "What should the store do next?",
"criteria": {
"restock": "Keep selling as is",
"inspect": "Check the affected batch",
"recall": "Recall the product",
},
},
}
def _dump(value) -> str:
return value if isinstance(value, str) else json.dumps(value, indent=2, ensure_ascii=False)
def _parse_state(state: str):
text = (state or "").strip()
if not text:
raise gr.Error("Please provide a state (plain text or JSON).")
try:
return json.loads(text)
except json.JSONDecodeError:
return text
def _parse_questions(questions: str) -> dict:
try:
parsed = json.loads(questions or "")
except json.JSONDecodeError as error:
raise gr.Error(f"Questions must be valid JSON: {error}")
if not isinstance(parsed, dict) or not parsed:
raise gr.Error("Questions must be a non-empty JSON object keyed by question ID.")
for qid, question in parsed.items():
if not isinstance(question, dict) or question.get("type") not in QUESTION_TYPES:
raise gr.Error(f"{qid}: type must be one of {', '.join(QUESTION_TYPES)}.")
if question["type"] == "choice" and not isinstance(question.get("criteria"), dict):
raise gr.Error(f"{qid}: choice criteria must be an object of option key -> description.")
if question["type"] == "score" and not (
isinstance(question.get("criteria"), list) and len(question["criteria"]) >= 2
):
raise gr.Error(f"{qid}: score criteria must list at least two level descriptions, lowest first.")
if question["type"] == "noul" and not isinstance(question.get("criteria") or {}, dict):
raise gr.Error(f"{qid}: noul criteria, when given, must map false and true to descriptions.")
return parsed
def _render(questions: dict, answers: dict, usage: dict, latency_ms: float) -> str:
cards = []
for qid, question in questions.items():
answer = answers[qid]
kind = answer["type"]
if kind == "noul":
p_true = answer["noul"]
best = "true" if p_true >= 0.5 else "false"
options = [("true", "true", p_true), ("false", "false", 1.0 - p_true)]
headline = f"→ <b>{best}</b> · P(true) = {p_true:.3f}"
elif kind == "choice":
best = answer["choice"]
options = []
for key, desc in question["criteria"].items():
label = f"{key} — {desc}" if desc else key
options.append((key, label, answer["probabilities"][key]))
headline = f"→ <b>{html.escape(str(best))}</b> · confidence {answer['confidence']:.2f}"
else: # score
probabilities = answer["probabilities"]
best = max(probabilities, key=probabilities.get)
options = [(key, f"{key} — {level}", probabilities[key]) for key, level in answer["legend"].items()]
headline = (
f"→ expected level <b>{answer['score']:.2f}</b> of {len(options) - 1} · "
f"most likely <b>{html.escape(str(answer['legend'][best]))}</b> · "
f"confidence {answer['confidence']:.2f}"
)
rows = []
for key, label, value in options:
highlight = "karpo-best" if key == best else ""
rows.append(
f'<div class="karpo-row {highlight}"><div class="karpo-label">{html.escape(str(label))}</div>'
f'<div class="karpo-bar"><div class="karpo-fill" style="width:{value * 100:.1f}%"></div></div>'
f'<div class="karpo-val">{value * 100:.1f}%</div></div>'
)
instructions = question.get("instructions") or qid
cards.append(
f'<div class="karpo-card"><div class="karpo-head"><span class="karpo-id">{html.escape(qid)}</span>'
f'<span class="karpo-type">{kind}</span></div>'
f'<div class="karpo-instr">{html.escape(str(instructions))}</div>'
f'<div class="karpo-answer">{headline}</div>{"".join(rows)}</div>'
)
footer = (
f'{usage["input_tokens"]} input tokens · {usage["sequences"]} sequence(s) · '
f"forward pass {latency_ms:.0f} ms · Powered by "
f'<a href="https://huggingface.co/MachinePulseAI/karpo-decision-31B">karpo-decision (MachinePulse)</a>'
)
return '<div class="karpo-wrap">' + "".join(cards) + f'<div class="karpo-foot">{footer}</div></div>'
def _gpu_duration(request: dict, *args, **kwargs) -> int:
# A cold worker streams ~62.5 GB of weights before the first forward pass; the reference
# forward itself is fast (the card reports a 94 ms median per request on one GPU).
chars = len(str(request.get("state") or "")) + len(str(request.get("questions") or ""))
return int(min(60, 20 + chars / 2000))
@spaces.GPU(duration=_gpu_duration, size="xlarge")
def _forward(request: dict):
torch.cuda.synchronize()
start = time.perf_counter()
response = decider.decide(request)
torch.cuda.synchronize()
latency_ms = (time.perf_counter() - start) * 1000
return response, latency_ms
# karpo.ai's playground calls systemone through Gradio's API: the model's name in its answers, and
# the title of an error about the request itself, whose message karpo-sites-api shows the reader.
SYSTEMONE_MODEL = "karpo-decision-31B"
REQUEST_ERROR_TITLE = "invalid_request"
def systemone(request: str) -> dict:
"""System One for karpo.ai's playground: a request's JSON in, the response's JSON out.
Args:
request: A System One request as JSON text, {"state", "questions"}.
Returns:
{"id", "model", "answers", "usage", "latency_ms"}; latency_ms is the forward pass on the GPU.
"""
try:
body = json.loads(request)
validate(body)
except (json.JSONDecodeError, RequestError) as error:
raise gr.Error(str(error), title=REQUEST_ERROR_TITLE)
try:
response, latency_ms = _forward({"state": body["state"], "questions": body["questions"]})
except (LengthError, RequestError) as error:
raise gr.Error(str(error), title=REQUEST_ERROR_TITLE)
return {
"id": f"decision-{uuid.uuid4().hex}",
"model": SYSTEMONE_MODEL,
"answers": response["answers"],
"usage": {"inputTokens": response["usage"]["input_tokens"], "outputTokens": 0},
"latency_ms": latency_ms,
}
def decide(state: str, questions: str):
"""Answer typed questions about a state with karpo-decision-31B.
Args:
state: The situation to decide on, as plain text or a JSON value.
questions: JSON object mapping question IDs to questions. Each question has a `type`
(`noul` = true/false, `choice` = named options, `score` = ordered levels),
optional `instructions`, and `criteria` (object of option key -> description for
choice, list of level descriptions lowest-first for score, optional false/true
descriptions for noul).
Returns:
An HTML rendering of the per-option probabilities, and the System One JSON response
(answers with probabilities, confidence and usage) for every question.
"""
parsed_questions = _parse_questions(questions)
request = {"state": _parse_state(state), "questions": parsed_questions}
try:
validate(request)
except RequestError as error:
raise gr.Error(str(error))
try:
response, latency_ms = _forward(request)
except LengthError as error:
raise gr.Error(f"Request too long: {error}")
except RequestError as error:
raise gr.Error(f"Invalid request: {error}")
return _render(parsed_questions, response["answers"], response["usage"], latency_ms), response
CSS = """
#col-container { max-width: 1200px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
.karpo-wrap { display: flex; flex-direction: column; gap: 12px; }
.karpo-card { border: 1px solid var(--border-color-primary); border-radius: 10px; padding: 12px 14px; }
.karpo-head { display: flex; gap: 8px; align-items: center; margin-bottom: 2px; }
.karpo-id { font-weight: 700; font-family: var(--font-mono); }
.karpo-type { font-size: 0.75em; padding: 1px 8px; border-radius: 999px; background: var(--color-accent-soft); }
.karpo-instr { opacity: 0.8; font-size: 0.92em; }
.karpo-answer { margin: 6px 0 8px; }
.karpo-row { display: grid; grid-template-columns: minmax(120px, 40%) 1fr 56px; gap: 8px; align-items: center; margin: 3px 0; font-size: 0.9em; }
.karpo-label { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.karpo-bar { height: 12px; border-radius: 6px; background: var(--block-background-fill); border: 1px solid var(--border-color-primary); overflow: hidden; }
.karpo-fill { height: 100%; background: var(--neutral-400); }
.karpo-best .karpo-fill { background: var(--color-accent); }
.karpo-best .karpo-label { font-weight: 600; }
.karpo-val { text-align: right; font-family: var(--font-mono); }
.karpo-foot { font-size: 0.8em; opacity: 0.7; }
"""
with gr.Blocks(title="karpo-decision-31B") as demo:
with gr.Column(elem_id="col-container"):
gr.Markdown(
"# karpo-decision-31B — typed decisions from one forward pass\n"
"[MachinePulseAI/karpo-decision-31B](https://huggingface.co/MachinePulseAI/karpo-decision-31B) "
"is the #1 model on the [Decision Index](https://huggingface.co/spaces/multimodalart/jev-decision-index) "
"(Full 73.9). Give it a **state** — any text or JSON — and a **schema of typed questions**: "
"`noul` (true/false), `choice` (named options) or `score` (ordered levels). "
"It returns a probability for every option of every question in one forward pass; no text is generated. "
"Text only; up to 32,768 tokens per sequence."
)
with gr.Row():
with gr.Column(scale=1):
state = gr.Textbox(
label="State (plain text or JSON)",
lines=5,
placeholder="Describe the situation, paste a JSON record, a support ticket, a log line…",
)
questions = gr.Code(
label="Questions (JSON schema)",
language="json",
value=_dump(EX_RETURN_Q),
lines=18,
)
run = gr.Button("Decide", variant="primary")
with gr.Column(scale=1):
result_html = gr.HTML(label="Decisions")
with gr.Accordion("SystemOne JSON response", open=False):
result_json = gr.JSON(label="Response")
run.click(decide, inputs=[state, questions], outputs=[result_html, result_json])
gr.api(systemone, api_name="systemone")
gr.Examples(
label="Examples",
examples=[
[EX_RETURN_STATE, _dump(EX_RETURN_Q)],
[_dump(EX_INVOICE_STATE), _dump(EX_INVOICE_Q)],
[EX_INCIDENT_STATE, _dump(EX_INCIDENT_Q)],
[EX_REVIEW_STATE, _dump(EX_REVIEW_Q)],
],
inputs=[state, questions],
outputs=[result_html, result_json],
fn=decide,
cache_examples=True,
cache_mode="lazy",
)
if __name__ == "__main__":
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)
|