${body}`;
}
// ---------- pages ----------
function viewOverview() {
const by = results.by_app;
const rows = Object.keys(APPS).filter((a) => by[a]).map((a) => {
const R = by[a].arms, n = Object.values(R).reduce((s, x) => s + x.n, 0);
return `
${esc(APPS[a].name)} ${esc(APPS[a].kind)}
${num(n)}
${pct(R.B.atomicity, 0)}
${pct(R.S.atomicity, 0)}
${R.B.IS} → ${R.S.IS}
`;
}).join("");
const body = `
How it works
Three moves wrapped around any agent that clicks through software.
1
Save
Before the agent starts, and before risky clicks like Save or Delete, Savepoint takes a checkpoint of the data.
2
Check
When the agent says “done”, Savepoint checks the result against a written rule: what must now be true, and what must not have changed.
3
Undo
If any check fails, Savepoint restores the checkpoint, then compares the data row by row to prove nothing was left behind.
What the tests show
600 recorded runs on three apps, with failures injected on purpose. “Ended clean” means the run either finished correctly or failed without changing anything.
App
Runs
Ended clean, no protection
Ended clean, with Savepoint
Said “done” but wrong (without → with)
${rows}
Source: Benchmark. Every one of the ${results.fidelity.restores} rollbacks restored the data exactly.
What this is, and what it is not
${icon("info")}
What it is
A safety layer that sits around an agent. It does not make the agent smarter. It makes failures safe and catches false “done” reports.
${icon("alert")}
What these numbers are not
The agent in every test is a script that fails on purpose, not an AI model. The numbers show what Savepoint does with known failures. They do not measure how reliable any AI model is.
A fuller explanation, with the technical detail for engineers.
`;
return pageShell({
title: "Undo for AI agents that work on old software",
lede: "Many companies run on old screens with no API. An AI agent can click through them, but when it fails halfway, or says “done” when it isn’t, the data is left broken and someone has to repair it by hand. Savepoint is a safety layer: it saves the data before risky steps, checks the result, and restores the data if anything is wrong.",
actions: `
`).join("");
const sel = B.runs.filter((r) => (resState.app === "all" || appOf(r.workflow_id) === resState.app) && (resState.arm === "all" || r.arm === resState.arm)
&& (resState.show === "all" || ["IS", "IF", "RF"].includes(outcome(r))));
const cnt = (o) => sel.filter((r) => outcome(r) === o).length;
let panels = "";
for (const app of apps) {
if (resState.app !== "all" && resState.app !== app) continue;
for (const arm of ["B", "S", "R"]) {
if (resState.arm !== "all" && resState.arm !== arm) continue;
const runs = B.runs.filter((r) => appOf(r.workflow_id) === app && r.arm === arm);
if (!runs.length) continue;
const T = Math.max(...runs.map((r) => r.trial));
const wfs = [...new Set(runs.map((r) => r.workflow_id))].sort();
let g = `` + [...Array(T).keys()].map((i) => `
Attempt ${i + 1}
`).join("");
for (const wf of wfs) {
g += `
${esc(taskTitle(wf))}
`;
for (let t = 1; t <= T; t++) {
const r = runs.find((x) => x.workflow_id === wf && x.trial === t);
const show = r && (resState.show === "all" || ["IS", "IF", "RF"].includes(outcome(r)));
g += `
${show ? tile(r) : ""}
`;
}
}
panels += `
${esc(APPS[app].name)} · ${esc(ARMS[arm].name)}
${esc(ARMS[arm].blurb)}
${g}
`;
}
}
const body = `
How to read this page
Each square is one test run: an agent doing one small task. Rows are tasks and columns are repeat attempts (up to 10 on ClaimDesk, 5 on the real apps). The same failures were injected in every protection level, so rows line up fairly. Click any square to see the full story of that run.
${legend}
Filter
App
${segBtn("app", "all", "All apps")}${apps.map((a) => segBtn("app", a, APPS[a].name)).join("")}
Protection
${segBtn("arm", "all", "All")}${["B", "S", "R"].map((a) => segBtn("arm", a, ARMS[a].name)).join("")}
`}`;
return pageShell({ title: "Test results", lede: "Every recorded run, one square each, grouped by app and protection level.", body });
}
function bindResults() {
$$("[data-res]").forEach((b) => b.addEventListener("click", () => { resState[b.dataset.res] = b.dataset.val; render(); }));
}
const STEP_WORDS = { "#save": "Click Save", "#ok": "Click OK on the confirmation", "#save-note": "Click Save note", "#delete": "Click Delete claim", "#next-button": "Click Next", "#submit": "Click Confirm", "#registration-submit": "Click Confirm", "#addAllergyBtn": "Click Save allergy", "#amount": "Type in the Amount box" };
function stepText(s) {
const t = s.target || "";
if (s.type === "goto") return "Open a page";
if (s.type === "wait") return "Wait for the screen to finish";
if (s.type === "fill") return STEP_WORDS[t] || "Type into a field";
if (s.type === "select") return "Choose an option from a list";
if (s.type === "click_until") return "Click Next until the confirm page appears";
if (s.type === "click") return STEP_WORDS[t] || "Click a button or link";
return s.type;
}
const CHECK_WORDS = { amount_under_cap: "The amount stays under 100,000", no_negative_amounts: "No amount is negative", no_orphan_notes: "No note is left without a claim", frame: "Nothing else changed" };
const checkText = (id) => CHECK_WORDS[id] || id.replace(/_/g, " ");
function viewRun(id) {
const r = B.runs.find((x) => x.id === id);
if (!r) return pageShell({ title: "Run not found", body: `
${r.arm === "B" ? "Run starts with no protection" : "Savepoint saves a checkpoint of the data"}${r.arm === "B" ? "Nothing is saved, so nothing can be undone." : "This is the state everything can be rolled back to."}
`;
for (const s of steps) {
for (const c of cps.filter((c) => c.step_idx === s.idx && c.kind !== "initial")) tl += `
${icon("diamond", "s")}
Savepoint saves a checkpointTaken before a risky click
No checks ran. ${r.arm === "B" ? "This protection level has no checks." : "The run ended before any data changed and before the agent said done, so Savepoint only restored the checkpoint."}
`;
let restore = "";
for (const rb of rbs) {
const why = { invariant: "a safety check failed", false_completion: "the agent said done but the result was wrong", agent_failed: "the agent could not finish", error: "the app produced an error", timeout: "the run took too long", step_budget: "the agent used too many steps" }[rb.reason] || rb.reason;
restore += `
The undo
Savepoint restored the checkpoint because ${esc(why)}. Rows different afterwards: ${rb.rows_differ}${rb.exact ? ", so the restore was exact" : ""}. Restore took ${rb.restore_ms} ms.
${rb.shot_before && rb.shot_after ? `
BeforeAfter
Drag the slider. The screen changed in ${rb.pixel_diff_pct}% of its pixels.
` : `
Screenshots are saved for featured runs only. Try the ones marked in the Try it page.
`,
});
}
function bindRun() { const sl = $("#slider"); if (sl) sl.addEventListener("input", () => $("#cmp").style.setProperty("--pos", sl.value + "%")); }
const benchState = { app: "claimdesk" };
function viewBenchmark() {
const apps = Object.keys(APPS).filter((a) => results.by_app[a]);
const R = results.by_app[benchState.app], arms = R.arms, W = 720, H = 300, L = 48, Rr = 140, T = 16, Bt = 36;
const K = Math.max(2, ...Object.values(arms).map((a) => Object.keys(a.pass_hat_k).length));
const x = (k) => L + ((k - 1) / (K - 1)) * (W - L - Rr), y = (v) => T + (1 - v) * (H - T - Bt);
const sty = { B: 'stroke="#101828" stroke-dasharray="6 4"', S: 'stroke="#1D4ED8"', R: 'stroke="#1D4ED8" stroke-dasharray="2 3"' };
let g = "";
for (let v = 0; v <= 1.001; v += 0.25) g += `${Math.round(v * 100)}%`;
for (let k = 1; k <= K; k++) g += `${k}`;
g += `Number of attempts in a row`;
const same = (a, b) => arms[a] && arms[b] && JSON.stringify(arms[a].pass_hat_k) === JSON.stringify(arms[b].pass_hat_k);
const labels = [];
for (const a of ["S", "R", "B"]) {
const ph = arms[a] && arms[a].pass_hat_k; if (!ph) continue;
const pts = Object.keys(ph).map((k) => [x(+k), y(ph[k])]);
g += `` + pts.map((p) => ``).join("");
const last = pts[pts.length - 1];
if (a === "B" && same("B", "S")) labels.push([`No protection(same as Savepoint)`]);
else if (!(a === "S" && same("B", "S"))) labels.push([`${esc(ARMS[a].name)}`]);
}
labels.forEach(([svgText]) => { g += svgText; });
let bars = "";
["B", "S", "R"].forEach((a, i) => {
const m = arms[a]; if (!m) return;
const yy = 12 + i * 52, w = W - 330, [lo, hi] = m.atomicity_ci95, col = a === "B" ? "#101828" : "#1D4ED8";
bars += `${esc(ARMS[a].name)}${pct(m.atomicity, 0)} (${pct(lo, 0)} to ${pct(hi, 0)})`;
});
const rows = ["B", "S", "R"].filter((a) => arms[a]).map((a) => { const m = arms[a]; return `
${esc(ARMS[a].name)}
${m.n}
${m.VS}
${m.VF}
${m.IS}
${m.IF}
${pct(m.valid_success, 0)}
${pct(m.atomicity, 0)}
`; }).join("");
const c = R.catch, t = R.timings, f = R.fidelity;
const seg = apps.map((a) => ``).join("");
const body = `
App
${seg}
${esc(APPS[benchState.app].kind)}. ${arms.B.workflows} tasks, ${arms.B.trials_per_workflow} attempts each, at three protection levels.
How often a run ended clean
A run ends clean when it either finishes correctly or fails without changing anything. Higher is better. The red line is the 95% confidence range (Wilson interval): with only ${arms.B.n} runs per level, the true rate could sit anywhere inside it.
Reliability over repeated attempts
If you ran the same task several times in a row, how likely is it that every run is correct? Real work needs this to stay high. ${same("B", "S") ? "The first two lines overlap because Savepoint (without retry) finishes the same tasks as the unprotected agent; what it changes is what the failed runs leave behind. Retry is what lets more runs finish." : ""} Technical name: pass^k.
No protectionSavepointSavepoint + retry
Outcome counts
How each run ended. See the glossary for the five outcomes.
Protection
Runs
Correct
Safe fail
False done
Mess left
Finished correctly
Ended clean
${rows}
Checks and restores
False “done” reports caught
${c.caught} of ${c.false_claims} (${pct(c.catch_rate, 0)}). When the agent said done and was wrong, Savepoint noticed.
Good results flagged by mistake
${c.false_alarms} of ${c.true_claims}. Savepoint never rolled back a run that was actually right.
Restores that were exact
${f.exact} of ${f.restores}. After each undo the data was compared row by row with the checkpoint.
Speed (95th percentile)
Taking a checkpoint: ${t.checkpoint_p95_ms} ms. Restoring: ${t.restore_p95_ms} ms.
Time cost
Checkpoints and restores took ${pct(R.overhead.checkpoint_restore_share_of_wall, 1)} of the run time.
${icon("chevronDown", "s")} Technical definitions and assumptions
Ended clean = atomicity = (correct + safe fail) / runs. False done = invalid success: the system reported done and the data differs from the expected change set. Mess left = invalid failure: reported not done, but data changed. Outcomes are judged by an independent oracle that never reads Savepoint’s checks.
pass^k for one task with n attempts and c correct is C(c,k)/C(n,k), averaged over tasks.
${esc(B.agent)}. Faults are injected on a fixed schedule and paired across protection levels. These figures describe runtime behaviour on a known failure mix, not model reliability.
`;
return pageShell({ title: "Benchmark", lede: "The same tasks and the same injected failures, at three protection levels. Pick an app.", body });
}
function bindBenchmark() { $$("[data-bapp]").forEach((b) => b.addEventListener("click", () => { benchState.app = b.dataset.bapp; render(); })); }
const taskState = { app: "all", q: "" };
function viewTasks() {
const list = B.workflows.filter((w) => (taskState.app === "all" || appOf(w.id) === taskState.app) && (w.title + " " + w.id).toLowerCase().includes(taskState.q.toLowerCase())).sort((a, b) => a.id.localeCompare(b.id));
const segBtn = (val, label) => ``;
const rows = list.map((w) => {
const c = parseContract(w), runs = B.runs.filter((r) => r.workflow_id === w.id);
return `
Before an agent starts a task, a rule file says how to judge the result. Safety checks must hold after every change. Goal checks say what must be true when the agent reports done. A frame lists everything that must not change. Rule files are plain YAML; the full text is under “Rule file”.
App
${segBtn("all", "All apps")}${Object.keys(APPS).map((a) => segBtn(a, APPS[a].name)).join("")}
Task
What the agent is told
Safety checks
Goal checks
Runs
Rules
${rows || `
No tasks match.
`}
`;
return pageShell({ title: "Tasks and rules", lede: `${B.workflows.length} tasks across three apps, each with a written rule file.`, body });
}
function bindTasks() {
$$("[data-tapp]").forEach((b) => b.addEventListener("click", () => { taskState.app = b.dataset.tapp; render(); }));
const q = $("#tq"); if (q) q.addEventListener("input", () => { taskState.q = q.value; const pos = q.selectionStart; render(); const n = $("#tq"); n.focus(); n.setSelectionRange(pos, pos); });
}
function viewHow() {
const body = `
The idea in one picture
A database transaction lets you undo a half-finished change. Old screens give agents no such thing. Savepoint adds it from the outside.
Step by step
1. Checkpoint
The data the app keeps (database rows and files), plus the browser session and the page you are on, is saved. This happens at the start, and again just before any click on a risky control such as Save, Delete or OK. If nothing changed since the last checkpoint, it is skipped.
2. Safety checks
After each step that changed data, rules that must always hold are tested (for example, “no amount is negative”). A failure stops the run immediately.
3. Goal and frame checks
When the agent reports done, Savepoint tests what must now be true (“the amount is 1250.00 and a note exists”) and that nothing else changed (a diff of the data against the start, minus the changes the task allows).
4. Restore
On any failure the checkpoint is restored. In the retry mode the agent is told why and tries again from the last good checkpoint, up to twice.
5. Proof
After the restore, every monitored row is fingerprinted column by column and compared with the checkpoint. Any difference marks the run “Restore failed”. None did in ${results.fidelity.restores} restores.
For engineers
${icon("chevronDown", "s")} What exactly is saved and compared
Per app adapter: the monitored tables (SQLite backup for ClaimDesk; a consistent read of the monitored MySQL and PostgreSQL tables for OpenMRS and iDempiere), a file-volume archive, the Playwright storage state and the URL. The digest is a per-row, per-column blake2b hash, with volatile columns (timestamps, uuids, generated ids) excluded.
${icon("chevronDown", "s")} How "false done" is judged
An oracle, separate from Savepoint’s own checks, replays the clean script once to get the expected change set, then compares each run’s final data with it. Oracle and validators never import each other (a test enforces it), so Savepoint cannot grade its own homework.
${icon("chevronDown", "s")} Protection levels
No protection: the agent runs alone. Savepoint (atomic): on any failure restore the first checkpoint. Savepoint + retry: on a safety-check failure restore the last checkpoint before the bad step, append the reason, and let the agent retry (at most twice); a false done is retried from the start.
${icon("chevronDown", "s")} Where to read more
The code includes the written spec, the 30 rule files and docs/validation.md, which lists every acceptance check as pass, adapted or partial.
`;
return pageShell({ title: "How it works", lede: "Savepoint wraps an agent in a transaction layer: save, check, undo, prove.", body });
}
function viewGlossary() {
const terms = [
["Agent", "Software that operates another program by looking at its screen and clicking and typing, like a person would. Here it is a script, not an AI model."],
["Checkpoint", "A saved copy of the data, taken so it can be restored."],
["Restore (rollback)", "Putting the data back to a checkpoint."],
["Task rules (contract)", "A short YAML file that says how to judge one task: safety checks, goal checks and what must not change."],
["Safety check (invariant)", "Something that must be true after every change, such as “no amount is negative”."],
["Goal check", "Something that must be true when the agent says done, such as “the amount is 1250.00”."],
["Frame check", "A diff of the data against the start. Everything the task is allowed to change is removed; anything left is an unexpected change."],
["False done (invalid success)", "The agent said done but the data is wrong. The most dangerous failure, because nobody is warned."],
["Safe fail (valid failure)", "The run did not finish, but the data is exactly as it was before."],
["Mess left (invalid failure)", "The run stopped partway and left half-finished changes."],
["Ended clean (atomicity)", "Correct or safe fail. All-or-nothing: the task either fully happened or did not happen at all."],
["pass^k", "The chance that k attempts in a row are all correct. It falls fast as k grows, which is why repeated reliability matters."],
["Digest", "A fingerprint of every monitored row and column, used to prove a restore was exact."],
["Oracle", "An independent judge that compares final data with the expected change set, without reading Savepoint’s checks."],
["Scripted agent", "A fixed click script that can be told to fail in four ways (stop, wrong value, skip save, say done early). It makes runs repeatable and free."],
].map(([t, d]) => `
${esc(t)}
${esc(d)}
`).join("");
const body = `
${terms}
Limits
${icon("alert")}
The agent in every test is a script that fails on purpose, not an AI model. The numbers show what Savepoint does with known failures; they do not measure any model.
The “live Claude run” was simulated through the real agent loop. No model was called and nothing was spent.
ClaimDesk 98 is a fake app built for safe testing. OpenMRS and iDempiere are real open-source apps run with their demo data, 5 attempts per task (10 on ClaimDesk). iDempiere’s screens were slow: 15 of its 150 runs ended in a UI timeout and were rolled back cleanly.
Rollback covers the tables a task’s rules monitor, not the whole database. Apps with outside effects (email, payments) are out of scope. Idea credit: LegacyWorld (arXiv 2608.14131).
Run it yourself
git clone https://github.com/MayaDispeler/savepoint
cd savepoint
uv sync && uv run playwright install chromium
uv run pytest -q # 60 tests, no model calls
uv run python -m savepoint.bench.cli run # ClaimDesk benchmark, about 10 minutes
`;
return pageShell({ title: "Glossary and limits", lede: "Every term used in this app, in plain words, and what the numbers do and do not show.", body });
}
// ---------- router ----------
function parseHash() {
const h = location.hash.replace(/^#\/?/, "");
const [path, qs] = h.split("?");
return { parts: path.split("/").filter(Boolean), query: Object.fromEntries(new URLSearchParams(qs || "")) };
}
function render() {
const { parts, query } = parseHash();
let route = parts[0] || "", html;
if (!B) { $("#main").innerHTML = `
${icon("alert")}
DATA_MISSING data/bundle.js did not load.
`; return; }
const views = { "": viewOverview, results: viewResults, benchmark: viewBenchmark, tasks: viewTasks, how: viewHow, glossary: viewGlossary };
if (route === "run") html = viewRun(parts[1]);
else if (route === "try") html = viewTry(query);
else html = (views[route] || viewOverview)();
const nav = route === "run" ? "results" : views[route] || route === "try" ? route : "";
renderNav(nav);
const main = $("#main"); main.innerHTML = html;
$("#side").classList.remove("open"); $("#menu").setAttribute("aria-expanded", "false");
if (route === "results") bindResults();
if (route === "run") bindRun();
if (route === "benchmark") bindBenchmark();
if (route === "tasks") bindTasks();
if (route === "try") bindTry(query);
document.title = ({ "": "Savepoint: undo for AI agents", try: "Try it", results: "Test results", run: "Run detail", benchmark: "Benchmark", tasks: "Tasks and rules", how: "How it works", glossary: "Glossary and limits" }[route] || "Savepoint") + (route ? " | Savepoint" : "");
}
let lastRoute = null;
window.addEventListener("hashchange", () => { const r = parseHash().parts[0] || ""; const changed = r !== lastRoute; lastRoute = r; render(); if (changed) window.scrollTo(0, 0); });
function boot() {
$("#mark").innerHTML = icon("undo", "l"); $("#mark-top").innerHTML = icon("undo", "l");
$("#l-gh").innerHTML = icon("code", "s") + " Source on GitHub"; $("#l-hf").innerHTML = icon("external", "s") + " Hugging Face Space";
$("#menu").addEventListener("click", () => { const o = $("#side").classList.toggle("open"); $("#menu").setAttribute("aria-expanded", String(o)); });
lastRoute = parseHash().parts[0] || "";
render();
}
boot();