| <!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width,initial-scale=1"> |
| <title>Agent Reliability Lab</title> |
|
|
| <style> |
| :root { |
| --bg: #070b14; |
| --surface: #0f1623; |
| --surface-2: #141d2d; |
| --border: rgba(255,255,255,.08); |
| --text: #f8fafc; |
| --muted: #98a2b3; |
| --blue: #004cff; |
| --green: #34d399; |
| --yellow: #fbbf24; |
| --red: #fb7185; |
| } |
| |
| * { |
| box-sizing: border-box; |
| } |
| |
| body { |
| margin: 0; |
| min-height: 100vh; |
| font-family: Inter, system-ui, -apple-system, BlinkMacSystemFont, |
| "Segoe UI", sans-serif; |
| background: |
| radial-gradient( |
| circle at top left, |
| rgba(0,76,255,.12), |
| transparent 30% |
| ), |
| var(--bg); |
| color: var(--text); |
| } |
| |
| .page { |
| width: min(1180px, calc(100% - 36px)); |
| margin: 0 auto; |
| padding-bottom: 60px; |
| } |
| |
| header { |
| height: 76px; |
| display: flex; |
| align-items: center; |
| justify-content: space-between; |
| border-bottom: 1px solid var(--border); |
| } |
| |
| .brand { |
| display: flex; |
| gap: 10px; |
| align-items: center; |
| font-weight: 700; |
| } |
| |
| .mark { |
| width: 24px; |
| height: 24px; |
| border-radius: 7px; |
| background: var(--blue); |
| box-shadow: 0 0 24px rgba(0,76,255,.45); |
| } |
| |
| .status { |
| font-size: 12px; |
| color: var(--muted); |
| border: 1px solid var(--border); |
| border-radius: 999px; |
| padding: 7px 11px; |
| } |
| |
| .hero { |
| padding: 76px 0 50px; |
| max-width: 880px; |
| } |
| |
| .eyebrow { |
| color: #7da4ff; |
| font-size: 12px; |
| font-weight: 700; |
| letter-spacing: .14em; |
| margin-bottom: 16px; |
| } |
| |
| h1 { |
| margin: 0; |
| font-size: clamp(44px, 7vw, 80px); |
| line-height: 1; |
| letter-spacing: -.05em; |
| } |
| |
| .hero p { |
| color: var(--muted); |
| font-size: 18px; |
| line-height: 1.7; |
| max-width: 720px; |
| margin-top: 24px; |
| } |
| |
| .grid { |
| display: grid; |
| grid-template-columns: 1fr 1fr; |
| gap: 18px; |
| } |
| |
| .panel { |
| background: var(--surface); |
| border: 1px solid var(--border); |
| border-radius: 18px; |
| padding: 26px; |
| } |
| |
| label { |
| display: block; |
| font-size: 12px; |
| color: var(--muted); |
| margin: 18px 0 8px; |
| } |
| |
| textarea { |
| width: 100%; |
| border: 1px solid var(--border); |
| border-radius: 12px; |
| background: #09111d; |
| color: var(--text); |
| padding: 14px; |
| resize: vertical; |
| outline: none; |
| line-height: 1.55; |
| } |
| |
| textarea:focus { |
| border-color: rgba(0,76,255,.7); |
| box-shadow: 0 0 0 4px rgba(0,76,255,.1); |
| } |
| |
| button { |
| width: 100%; |
| margin-top: 18px; |
| padding: 14px 16px; |
| border: 0; |
| border-radius: 11px; |
| background: var(--blue); |
| color: white; |
| font-weight: 700; |
| cursor: pointer; |
| } |
| |
| button.secondary { |
| background: transparent; |
| border: 1px solid var(--border); |
| color: var(--muted); |
| margin-top: 10px; |
| } |
| |
| .score { |
| font-size: 58px; |
| font-weight: 750; |
| letter-spacing: -.04em; |
| margin: 10px 0; |
| } |
| |
| .risk { |
| display: inline-block; |
| padding: 7px 10px; |
| border-radius: 999px; |
| font-size: 12px; |
| font-weight: 700; |
| } |
| |
| .low { |
| background: rgba(52,211,153,.12); |
| color: var(--green); |
| } |
| |
| .moderate { |
| background: rgba(251,191,36,.12); |
| color: var(--yellow); |
| } |
| |
| .elevated, |
| .high { |
| background: rgba(251,113,133,.12); |
| color: var(--red); |
| } |
| |
| .metrics { |
| display: grid; |
| grid-template-columns: 1fr 1fr; |
| gap: 10px; |
| margin-top: 22px; |
| } |
| |
| .metric { |
| border: 1px solid var(--border); |
| border-radius: 10px; |
| padding: 12px; |
| background: var(--surface-2); |
| } |
| |
| .metric span { |
| display: block; |
| color: var(--muted); |
| font-size: 11px; |
| margin-bottom: 4px; |
| } |
| |
| .section { |
| margin-top: 24px; |
| } |
| |
| .section h3 { |
| margin-bottom: 10px; |
| } |
| |
| ul { |
| color: #d6dbe5; |
| line-height: 1.6; |
| padding-left: 20px; |
| } |
| |
| .muted { |
| color: var(--muted); |
| font-size: 13px; |
| line-height: 1.65; |
| } |
| |
| .hidden { |
| display: none; |
| } |
| |
| footer { |
| margin-top: 56px; |
| padding-top: 24px; |
| border-top: 1px solid var(--border); |
| color: var(--muted); |
| font-size: 12px; |
| } |
| |
| @media (max-width: 850px) { |
| .grid { |
| grid-template-columns: 1fr; |
| } |
| } |
| </style> |
| </head> |
|
|
| <body> |
| <div class="page"> |
|
|
| <header> |
| <div class="brand"> |
| <div class="mark"></div> |
| Agent Reliability Lab |
| </div> |
|
|
| <div class="status"> |
| Static · deterministic analysis |
| </div> |
| </header> |
|
|
| <section class="hero"> |
| <div class="eyebrow">AGENT EVALUATION</div> |
|
|
| <h1> |
| Analyze observable reliability signals in agent traces. |
| </h1> |
|
|
| <p> |
| Detect repeated tool calls, execution failures, missing observations, |
| unsupported answers, weak verification and overconfident behaviour. |
| </p> |
| </section> |
|
|
| <section class="grid"> |
|
|
| <div class="panel"> |
| <h2>Execution trace</h2> |
|
|
| <label for="goal">Agent goal</label> |
|
|
| <textarea |
| id="goal" |
| rows="4" |
| placeholder="Find the current release version and verify it using an official source." |
| ></textarea> |
|
|
| <label for="trace">Agent execution trace</label> |
|
|
| <textarea |
| id="trace" |
| rows="16" |
| placeholder="Plan: Search official documentation. |
| Tool: Search release documentation |
| Observation: Found official release page. |
| Tool: Open release page |
| Observation: Version 4.2 is listed. |
| Action: Verify against changelog. |
| Observation: Changelog confirms version 4.2. |
| Final: Current release is 4.2." |
| ></textarea> |
|
|
| <button id="analyze"> |
| Analyze agent trace |
| </button> |
|
|
| <button id="reliable" class="secondary"> |
| Load reliable example |
| </button> |
|
|
| <button id="problematic" class="secondary"> |
| Load problematic example |
| </button> |
| </div> |
|
|
| <div class="panel"> |
|
|
| <div id="empty"> |
| <h2>Assessment ready</h2> |
| <p class="muted"> |
| Add an agent goal and execution trace, then run the analysis. |
| </p> |
| </div> |
|
|
| <div id="result" class="hidden"> |
|
|
| <h2>Reliability assessment</h2> |
|
|
| <div id="riskBadge" class="risk"></div> |
|
|
| <div id="score" class="score"></div> |
|
|
| <div class="metrics" id="metrics"></div> |
|
|
| <div class="section"> |
| <h3>Reliability findings</h3> |
| <ul id="findings"></ul> |
| </div> |
|
|
| <div class="section"> |
| <h3>Positive signals</h3> |
| <ul id="positives"></ul> |
| </div> |
|
|
| <div class="section"> |
| <h3>Interpretation</h3> |
| <p class="muted"> |
| This tool performs transparent heuristic analysis of an |
| observable execution trace. It does not inspect hidden |
| chain-of-thought and does not determine whether the final |
| answer is factually correct. |
| </p> |
| </div> |
|
|
| </div> |
| </div> |
|
|
| </section> |
|
|
| <footer> |
| Experimental portfolio project for observable AI-agent reliability analysis. |
| </footer> |
|
|
| </div> |
|
|
| <script> |
| const normalize = text => |
| text |
| .toLowerCase() |
| .trim() |
| .replace(/\s+/g, " ") |
| .replace(/\d+/g, "<n>"); |
| |
| function analyzeTrace(goal, trace) { |
| const lines = trace |
| .split("\n") |
| .map(x => x.trim()) |
| .filter(Boolean); |
| |
| const lower = trace.toLowerCase(); |
| |
| const toolPrefixes = [ |
| "tool:", |
| "tool call:", |
| "call:", |
| "search:", |
| "browse:", |
| "execute:", |
| "query:", |
| "action:" |
| ]; |
| |
| const observationPrefixes = [ |
| "observation:", |
| "result:", |
| "evidence:", |
| "output:", |
| "response:" |
| ]; |
| |
| const planningPrefixes = [ |
| "plan:", |
| "step:", |
| "next:", |
| "reasoning:" |
| ]; |
| |
| const finalPrefixes = [ |
| "final:", |
| "final answer:", |
| "answer:" |
| ]; |
| |
| const starts = (line, prefixes) => |
| prefixes.some(prefix => |
| line.toLowerCase().startsWith(prefix) |
| ); |
| |
| const toolLines = |
| lines.filter(x => starts(x, toolPrefixes)); |
| |
| const observationLines = |
| lines.filter(x => starts(x, observationPrefixes)); |
| |
| const planningLines = |
| lines.filter(x => starts(x, planningPrefixes)); |
| |
| const finalLines = |
| lines.filter(x => starts(x, finalPrefixes)); |
| |
| let penalty = 0; |
| |
| const findings = []; |
| const positives = []; |
| |
| const failureMarkers = [ |
| "error", |
| "failed", |
| "failure", |
| "timeout", |
| "timed out", |
| "not found", |
| "permission denied", |
| "unauthorized", |
| "invalid", |
| "exception" |
| ]; |
| |
| const failedSteps = lines.filter(line => |
| failureMarkers.some(marker => |
| line.toLowerCase().includes(marker) |
| ) |
| ); |
| |
| if (failedSteps.length) { |
| penalty += Math.min( |
| 25, |
| failedSteps.length * 8 |
| ); |
| |
| findings.push( |
| `Detected ${failedSteps.length} possible tool or execution failure signal(s).` |
| ); |
| } else { |
| positives.push( |
| "No explicit tool failure markers detected." |
| ); |
| } |
| |
| const counts = {}; |
| |
| toolLines |
| .map(normalize) |
| .forEach(item => { |
| counts[item] = (counts[item] || 0) + 1; |
| }); |
| |
| const repeated = |
| Object.entries(counts) |
| .filter(([, count]) => count >= 3); |
| |
| if (repeated.length) { |
| penalty += Math.min( |
| 30, |
| repeated.length * 12 |
| ); |
| |
| findings.push( |
| "Possible agent loop detected: one or more tool actions were repeated at least three times." |
| ); |
| } else { |
| positives.push( |
| "No obvious repeated-tool loop detected." |
| ); |
| } |
| |
| const observationRatio = |
| toolLines.length > 0 |
| ? observationLines.length / toolLines.length |
| : 0; |
| |
| if ( |
| toolLines.length > 0 && |
| observationRatio < 0.5 |
| ) { |
| penalty += 18; |
| |
| findings.push( |
| "Tool calls are not consistently followed by explicit observations or evidence." |
| ); |
| } else if (toolLines.length > 0) { |
| positives.push( |
| "Most tool activity is accompanied by explicit observations." |
| ); |
| } |
| |
| if ( |
| finalLines.length && |
| observationLines.length === 0 |
| ) { |
| penalty += 22; |
| |
| findings.push( |
| "A final answer is present without explicit observations or evidence." |
| ); |
| } |
| |
| if (!finalLines.length) { |
| penalty += 8; |
| |
| findings.push( |
| "No explicit final answer was detected." |
| ); |
| } else { |
| positives.push( |
| "An explicit final answer is present." |
| ); |
| } |
| |
| const overconfidenceMarkers = [ |
| "definitely", |
| "guaranteed", |
| "without doubt", |
| "certainly", |
| "100%", |
| "always correct", |
| "cannot be wrong", |
| "no need to verify", |
| "does not need to be verified" |
| ]; |
| |
| const overconfidence = |
| overconfidenceMarkers.filter( |
| marker => lower.includes(marker) |
| ); |
| |
| if (overconfidence.length) { |
| penalty += Math.min( |
| 20, |
| overconfidence.length * 7 |
| ); |
| |
| findings.push( |
| `Potential overconfidence detected: ${overconfidence.join(", ")}.` |
| ); |
| } else { |
| positives.push( |
| "No explicit high-confidence language detected." |
| ); |
| } |
| |
| const verificationMarkers = [ |
| "verify", |
| "verified", |
| "verification", |
| "cross-check", |
| "cross check", |
| "validate", |
| "validated", |
| "confirm", |
| "confirmed", |
| "source", |
| "evidence", |
| "changelog" |
| ]; |
| |
| const verification = |
| verificationMarkers.some( |
| marker => lower.includes(marker) |
| ); |
| |
| if ( |
| toolLines.length >= 2 && |
| !verification |
| ) { |
| penalty += 12; |
| |
| findings.push( |
| "Multiple actions were performed without an explicit verification or validation step." |
| ); |
| } else if (verification) { |
| positives.push( |
| "Verification or evidence-checking behaviour detected." |
| ); |
| } |
| |
| if (planningLines.length) { |
| positives.push( |
| "Explicit planning or next-step structure detected." |
| ); |
| } else { |
| penalty += 5; |
| |
| findings.push( |
| "No explicit planning step was detected." |
| ); |
| } |
| |
| const score = |
| Math.max( |
| 0, |
| Math.min(100, 100 - penalty) |
| ); |
| |
| let risk = "High"; |
| |
| if (score >= 85) { |
| risk = "Low"; |
| } else if (score >= 70) { |
| risk = "Moderate"; |
| } else if (score >= 50) { |
| risk = "Elevated"; |
| } |
| |
| return { |
| score, |
| risk, |
| findings, |
| positives, |
| metrics: { |
| "Trace steps": lines.length, |
| "Tool calls": toolLines.length, |
| "Observations": observationLines.length, |
| "Planning steps": planningLines.length, |
| "Final answers": finalLines.length, |
| "Failure signals": failedSteps.length, |
| "Repeated patterns": repeated.length, |
| "Verification": verification |
| ? "Detected" |
| : "Not detected" |
| } |
| }; |
| } |
| |
| function render(result) { |
| document |
| .getElementById("empty") |
| .classList.add("hidden"); |
| |
| document |
| .getElementById("result") |
| .classList.remove("hidden"); |
| |
| const badge = |
| document.getElementById("riskBadge"); |
| |
| badge.textContent = |
| `${result.risk} risk`; |
| |
| badge.className = |
| `risk ${result.risk.toLowerCase()}`; |
| |
| document.getElementById("score") |
| .textContent = |
| `${result.score}/100`; |
| |
| const metrics = |
| document.getElementById("metrics"); |
| |
| metrics.innerHTML = ""; |
| |
| Object.entries(result.metrics) |
| .forEach(([key, value]) => { |
| metrics.innerHTML += ` |
| <div class="metric"> |
| <span>${key}</span> |
| <strong>${value}</strong> |
| </div> |
| `; |
| }); |
| |
| const findings = |
| document.getElementById("findings"); |
| |
| findings.innerHTML = ""; |
| |
| ( |
| result.findings.length |
| ? result.findings |
| : ["No major heuristic reliability issues detected."] |
| ).forEach(item => { |
| findings.innerHTML += |
| `<li>${item}</li>`; |
| }); |
| |
| const positives = |
| document.getElementById("positives"); |
| |
| positives.innerHTML = ""; |
| |
| ( |
| result.positives.length |
| ? result.positives |
| : ["No strong positive reliability signals detected."] |
| ).forEach(item => { |
| positives.innerHTML += |
| `<li>${item}</li>`; |
| }); |
| } |
| |
| document |
| .getElementById("analyze") |
| .onclick = () => { |
| |
| const goal = |
| document.getElementById("goal") |
| .value.trim(); |
| |
| const trace = |
| document.getElementById("trace") |
| .value.trim(); |
| |
| if ( |
| goal.length < 5 || |
| trace.length < 20 |
| ) { |
| alert( |
| "Please provide both an agent goal and a sufficiently detailed trace." |
| ); |
| return; |
| } |
| |
| render( |
| analyzeTrace(goal, trace) |
| ); |
| }; |
| |
| document |
| .getElementById("reliable") |
| .onclick = () => { |
| |
| document.getElementById("goal").value = |
| "Find the current release version of a software library and verify it using an official source."; |
| |
| document.getElementById("trace").value = |
| `Plan: Search the official documentation. |
| Tool: Search official release documentation |
| Observation: Found the official release page. |
| Tool: Open the release page |
| Observation: Version 4.2 is listed. |
| Action: Verify against the official changelog. |
| Observation: The changelog confirms version 4.2. |
| Final: The current release is version 4.2.`; |
| }; |
| |
| document |
| .getElementById("problematic") |
| .onclick = () => { |
| |
| document.getElementById("goal").value = |
| "Find the latest release version of a software library."; |
| |
| document.getElementById("trace").value = |
| `Tool: Search web for latest version |
| Error: Search request timed out. |
| Tool: Search web for latest version |
| Error: Search request timed out. |
| Tool: Search web for latest version |
| Error: Search request timed out. |
| Final: The version is definitely 4.2 and does not need to be verified.`; |
| }; |
| </script> |
|
|
| </body> |
| </html> |
|
|
