Eric Xu Claude Opus 4.6 (1M context) commited on
Commit ·
5589a80
1
Parent(s): cc9ff58
Add downloadable report + switch default model to gpt-oss-120b
Browse filesAfter any phase (eval, counterfactual, bias audit), users can download
a comprehensive markdown report with all results. Default eval model
changed from gpt-4o-mini to gpt-oss-120b for better quality.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
- web/app.py +151 -2
- web/static/index.html +35 -1
web/app.py
CHANGED
|
@@ -25,7 +25,7 @@ from pathlib import Path
|
|
| 25 |
from dotenv import load_dotenv
|
| 26 |
from fastapi import FastAPI, HTTPException, Query, Request
|
| 27 |
from fastapi.staticfiles import StaticFiles
|
| 28 |
-
from fastapi.responses import FileResponse
|
| 29 |
from pydantic import BaseModel
|
| 30 |
from sse_starlette.sse import EventSourceResponse
|
| 31 |
|
|
@@ -129,7 +129,7 @@ def get_client(api_key=None, base_url=None):
|
|
| 129 |
|
| 130 |
|
| 131 |
def get_model(model=None):
|
| 132 |
-
return model or os.getenv("LLM_MODEL_NAME", "openai/gpt-
|
| 133 |
|
| 134 |
|
| 135 |
def get_fast_model():
|
|
@@ -304,6 +304,23 @@ async def create_session(entity: EntityInput):
|
|
| 304 |
return {"session_id": sid}
|
| 305 |
|
| 306 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 307 |
@app.get("/api/session/{sid}")
|
| 308 |
async def get_session(sid: str):
|
| 309 |
if sid not in sessions:
|
|
@@ -892,6 +909,138 @@ async def get_results(sid: str):
|
|
| 892 |
}
|
| 893 |
|
| 894 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 895 |
|
| 896 |
if __name__ == "__main__":
|
| 897 |
import uvicorn
|
|
|
|
| 25 |
from dotenv import load_dotenv
|
| 26 |
from fastapi import FastAPI, HTTPException, Query, Request
|
| 27 |
from fastapi.staticfiles import StaticFiles
|
| 28 |
+
from fastapi.responses import FileResponse, Response
|
| 29 |
from pydantic import BaseModel
|
| 30 |
from sse_starlette.sse import EventSourceResponse
|
| 31 |
|
|
|
|
| 129 |
|
| 130 |
|
| 131 |
def get_model(model=None):
|
| 132 |
+
return model or os.getenv("LLM_MODEL_NAME", "openai/gpt-oss-120b")
|
| 133 |
|
| 134 |
|
| 135 |
def get_fast_model():
|
|
|
|
| 304 |
return {"session_id": sid}
|
| 305 |
|
| 306 |
|
| 307 |
+
class SessionMetaUpdate(BaseModel):
|
| 308 |
+
goal: str = ""
|
| 309 |
+
audience: str = ""
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
@app.patch("/api/session/{sid}")
|
| 313 |
+
async def update_session_meta(sid: str, meta: SessionMetaUpdate):
|
| 314 |
+
"""Update session metadata (goal, audience)."""
|
| 315 |
+
if sid not in sessions:
|
| 316 |
+
raise HTTPException(404, "Session not found")
|
| 317 |
+
if meta.goal:
|
| 318 |
+
sessions[sid]["goal"] = meta.goal
|
| 319 |
+
if meta.audience:
|
| 320 |
+
sessions[sid]["audience"] = meta.audience
|
| 321 |
+
return {"ok": True}
|
| 322 |
+
|
| 323 |
+
|
| 324 |
@app.get("/api/session/{sid}")
|
| 325 |
async def get_session(sid: str):
|
| 326 |
if sid not in sessions:
|
|
|
|
| 909 |
}
|
| 910 |
|
| 911 |
|
| 912 |
+
@app.get("/api/report/{sid}")
|
| 913 |
+
async def download_report(sid: str):
|
| 914 |
+
"""Generate and download a comprehensive markdown report for this session."""
|
| 915 |
+
if sid not in sessions:
|
| 916 |
+
raise HTTPException(404, "Session not found")
|
| 917 |
+
s = sessions[sid]
|
| 918 |
+
if not s["eval_results"]:
|
| 919 |
+
raise HTTPException(400, "No evaluation results yet")
|
| 920 |
+
|
| 921 |
+
lines = []
|
| 922 |
+
lines.append("# SGO Evaluation Report")
|
| 923 |
+
lines.append(f"*Generated {datetime.now().strftime('%Y-%m-%d %H:%M')}*\n")
|
| 924 |
+
|
| 925 |
+
# Entity
|
| 926 |
+
lines.append("---\n")
|
| 927 |
+
lines.append("## Entity Evaluated\n")
|
| 928 |
+
lines.append(s["entity_text"])
|
| 929 |
+
lines.append("")
|
| 930 |
+
|
| 931 |
+
if s.get("goal"):
|
| 932 |
+
lines.append(f"**Goal:** {s['goal']}\n")
|
| 933 |
+
if s.get("audience"):
|
| 934 |
+
lines.append(f"**Audience:** {s['audience']}\n")
|
| 935 |
+
|
| 936 |
+
# Cohort summary
|
| 937 |
+
cohort = s.get("cohort") or []
|
| 938 |
+
if cohort:
|
| 939 |
+
lines.append("---\n")
|
| 940 |
+
lines.append(f"## Panel ({len(cohort)} evaluators)\n")
|
| 941 |
+
lines.append("| # | Name | Age | Occupation | Location |")
|
| 942 |
+
lines.append("|---|------|-----|------------|----------|")
|
| 943 |
+
for i, p in enumerate(cohort, 1):
|
| 944 |
+
name = p.get("name", "?")
|
| 945 |
+
age = p.get("age", "")
|
| 946 |
+
occ = p.get("occupation", "")
|
| 947 |
+
loc = p.get("city", p.get("location", ""))
|
| 948 |
+
if p.get("state"):
|
| 949 |
+
loc = f"{loc}, {p['state']}" if loc else p["state"]
|
| 950 |
+
lines.append(f"| {i} | {name} | {age} | {occ} | {loc} |")
|
| 951 |
+
lines.append("")
|
| 952 |
+
|
| 953 |
+
# Evaluation results
|
| 954 |
+
results = s["eval_results"]
|
| 955 |
+
valid = [r for r in results if r and "score" in r]
|
| 956 |
+
scores = [r["score"] for r in valid]
|
| 957 |
+
avg = sum(scores) / len(scores) if scores else 0
|
| 958 |
+
|
| 959 |
+
lines.append("---\n")
|
| 960 |
+
lines.append("## Evaluation Results\n")
|
| 961 |
+
lines.append(f"**Average Score: {avg:.1f}/10** ({len(valid)} evaluators)\n")
|
| 962 |
+
|
| 963 |
+
pos = sum(1 for r in valid if r.get("action") == "positive")
|
| 964 |
+
neu = sum(1 for r in valid if r.get("action") == "neutral")
|
| 965 |
+
neg = sum(1 for r in valid if r.get("action") == "negative")
|
| 966 |
+
lines.append(f"- Would say yes: {pos}")
|
| 967 |
+
lines.append(f"- Unsure: {neu}")
|
| 968 |
+
lines.append(f"- Would say no: {neg}\n")
|
| 969 |
+
|
| 970 |
+
# Full analysis from evaluate.py
|
| 971 |
+
analysis = analyze_eval(results)
|
| 972 |
+
lines.append(analysis)
|
| 973 |
+
lines.append("")
|
| 974 |
+
|
| 975 |
+
# Individual evaluator details
|
| 976 |
+
lines.append("### All Evaluator Responses\n")
|
| 977 |
+
lines.append("| Name | Age | Occupation | Score | Action | Summary |")
|
| 978 |
+
lines.append("|------|-----|------------|-------|--------|---------|")
|
| 979 |
+
sorted_results = sorted(valid, key=lambda r: r["score"], reverse=True)
|
| 980 |
+
for r in sorted_results:
|
| 981 |
+
ev = r.get("_evaluator", {})
|
| 982 |
+
name = ev.get("name", "?")
|
| 983 |
+
age = ev.get("age", "")
|
| 984 |
+
occ = ev.get("occupation", "")
|
| 985 |
+
score = r["score"]
|
| 986 |
+
action = r.get("action", "")
|
| 987 |
+
summary = r.get("summary", "").replace("|", "/").replace("\n", " ")
|
| 988 |
+
lines.append(f"| {name} | {age} | {occ} | {score}/10 | {action} | {summary} |")
|
| 989 |
+
lines.append("")
|
| 990 |
+
|
| 991 |
+
# Counterfactual gradient
|
| 992 |
+
if s.get("gradient"):
|
| 993 |
+
lines.append("---\n")
|
| 994 |
+
lines.append("## Priority Actions (Counterfactual Gradient)\n")
|
| 995 |
+
lines.append(s["gradient"])
|
| 996 |
+
lines.append("")
|
| 997 |
+
|
| 998 |
+
# Bias audit
|
| 999 |
+
if s.get("bias_audit"):
|
| 1000 |
+
audit = s["bias_audit"]
|
| 1001 |
+
lines.append("---\n")
|
| 1002 |
+
lines.append("## Panel Realism Check (Bias Audit)\n")
|
| 1003 |
+
if audit.get("report"):
|
| 1004 |
+
lines.append(audit["report"])
|
| 1005 |
+
lines.append("")
|
| 1006 |
+
if audit.get("analyses"):
|
| 1007 |
+
lines.append("| Probe | Shifted % | Avg Score Change | Human Baseline | Assessment |")
|
| 1008 |
+
lines.append("|-------|-----------|------------------|----------------|------------|")
|
| 1009 |
+
baselines = {"framing": 30, "authority": 20, "order": 0}
|
| 1010 |
+
for a in audit["analyses"]:
|
| 1011 |
+
if a.get("error"):
|
| 1012 |
+
continue
|
| 1013 |
+
expected = baselines.get(a["probe"])
|
| 1014 |
+
gap = a["shifted_pct"] - (expected or 0)
|
| 1015 |
+
if expected is not None:
|
| 1016 |
+
if gap > 10:
|
| 1017 |
+
assessment = "Over-biased"
|
| 1018 |
+
elif gap < -10:
|
| 1019 |
+
assessment = "Under-biased"
|
| 1020 |
+
else:
|
| 1021 |
+
assessment = "Well-calibrated"
|
| 1022 |
+
else:
|
| 1023 |
+
assessment = "—"
|
| 1024 |
+
lines.append(
|
| 1025 |
+
f"| {a['probe']} | {a['shifted_pct']:.1f}% | "
|
| 1026 |
+
f"{a['avg_abs_delta']:.2f} | "
|
| 1027 |
+
f"{str(expected) + '%' if expected is not None else '—'} | "
|
| 1028 |
+
f"{assessment} |"
|
| 1029 |
+
)
|
| 1030 |
+
lines.append("")
|
| 1031 |
+
|
| 1032 |
+
lines.append("---\n")
|
| 1033 |
+
lines.append("*Report generated by [SGO — Semantic Gradient Optimization](https://github.com/anthropics/sgo)*")
|
| 1034 |
+
|
| 1035 |
+
report_md = "\n".join(lines)
|
| 1036 |
+
filename = f"sgo-report-{sid}.md"
|
| 1037 |
+
return Response(
|
| 1038 |
+
content=report_md,
|
| 1039 |
+
media_type="text/markdown",
|
| 1040 |
+
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
| 1041 |
+
)
|
| 1042 |
+
|
| 1043 |
+
|
| 1044 |
|
| 1045 |
if __name__ == "__main__":
|
| 1046 |
import uvicorn
|
web/static/index.html
CHANGED
|
@@ -292,6 +292,17 @@
|
|
| 292 |
}
|
| 293 |
.template-chip:hover { border-color: var(--accent); color: var(--text); }
|
| 294 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 295 |
/* Responsive */
|
| 296 |
@media (max-width: 600px) {
|
| 297 |
.container { padding: 16px 12px; }
|
|
@@ -448,6 +459,7 @@
|
|
| 448 |
<div class="btn-row mt-16">
|
| 449 |
<button onclick="runDirections()">Test what to change next</button>
|
| 450 |
<button class="secondary" onclick="goToStep(3)">Check panel realism</button>
|
|
|
|
| 451 |
</div>
|
| 452 |
</div>
|
| 453 |
</div>
|
|
@@ -476,6 +488,9 @@
|
|
| 476 |
</table>
|
| 477 |
<div id="gradientText" class="hidden"></div>
|
| 478 |
<div id="changesTested" class="hidden"></div>
|
|
|
|
|
|
|
|
|
|
| 479 |
</div>
|
| 480 |
</div>
|
| 481 |
|
|
@@ -535,6 +550,7 @@
|
|
| 535 |
<div class="btn-row mt-16">
|
| 536 |
<button class="secondary" onclick="rerunWithCalibration()">Run again with realism tuning</button>
|
| 537 |
<button class="secondary" onclick="goToStep(2)">Test what to change next</button>
|
|
|
|
| 538 |
</div>
|
| 539 |
</div>
|
| 540 |
</div>
|
|
@@ -965,7 +981,13 @@ async function runFullPipeline() {
|
|
| 965 |
});
|
| 966 |
const sessData = await sessResp.json();
|
| 967 |
sessionId = sessData.session_id;
|
| 968 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 969 |
|
| 970 |
// Start elapsed timer
|
| 971 |
const startTime = Date.now();
|
|
@@ -1437,6 +1459,18 @@ function runBiasAudit() {
|
|
| 1437 |
};
|
| 1438 |
}
|
| 1439 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1440 |
// Boot
|
| 1441 |
init();
|
| 1442 |
</script>
|
|
|
|
| 292 |
}
|
| 293 |
.template-chip:hover { border-color: var(--accent); color: var(--text); }
|
| 294 |
|
| 295 |
+
/* Download button */
|
| 296 |
+
.btn-download {
|
| 297 |
+
background: var(--surface2);
|
| 298 |
+
border: 1px solid var(--green);
|
| 299 |
+
color: var(--green);
|
| 300 |
+
display: inline-flex;
|
| 301 |
+
align-items: center;
|
| 302 |
+
gap: 6px;
|
| 303 |
+
}
|
| 304 |
+
.btn-download:hover { background: color-mix(in srgb, var(--green) 15%, var(--surface2)); }
|
| 305 |
+
|
| 306 |
/* Responsive */
|
| 307 |
@media (max-width: 600px) {
|
| 308 |
.container { padding: 16px 12px; }
|
|
|
|
| 459 |
<div class="btn-row mt-16">
|
| 460 |
<button onclick="runDirections()">Test what to change next</button>
|
| 461 |
<button class="secondary" onclick="goToStep(3)">Check panel realism</button>
|
| 462 |
+
<button class="btn-download" onclick="downloadReport()">⤓ Download report</button>
|
| 463 |
</div>
|
| 464 |
</div>
|
| 465 |
</div>
|
|
|
|
| 488 |
</table>
|
| 489 |
<div id="gradientText" class="hidden"></div>
|
| 490 |
<div id="changesTested" class="hidden"></div>
|
| 491 |
+
<div class="btn-row mt-16">
|
| 492 |
+
<button class="btn-download" onclick="downloadReport()">⤓ Download full report</button>
|
| 493 |
+
</div>
|
| 494 |
</div>
|
| 495 |
</div>
|
| 496 |
|
|
|
|
| 550 |
<div class="btn-row mt-16">
|
| 551 |
<button class="secondary" onclick="rerunWithCalibration()">Run again with realism tuning</button>
|
| 552 |
<button class="secondary" onclick="goToStep(2)">Test what to change next</button>
|
| 553 |
+
<button class="btn-download" onclick="downloadReport()">⤓ Download full report</button>
|
| 554 |
</div>
|
| 555 |
</div>
|
| 556 |
</div>
|
|
|
|
| 981 |
});
|
| 982 |
const sessData = await sessResp.json();
|
| 983 |
sessionId = sessData.session_id;
|
| 984 |
+
|
| 985 |
+
// Store goal/audience in session for report generation
|
| 986 |
+
fetch(`/api/session/${sessionId}`, {
|
| 987 |
+
method: 'PATCH',
|
| 988 |
+
headers: llmHeaders(),
|
| 989 |
+
body: JSON.stringify({goal: goalField.value.trim(), audience: audienceCtx}),
|
| 990 |
+
}).catch(() => {});
|
| 991 |
|
| 992 |
// Start elapsed timer
|
| 993 |
const startTime = Date.now();
|
|
|
|
| 1459 |
};
|
| 1460 |
}
|
| 1461 |
|
| 1462 |
+
// ── Download report ──
|
| 1463 |
+
|
| 1464 |
+
function downloadReport() {
|
| 1465 |
+
if (!sessionId) return alert('Run an evaluation first.');
|
| 1466 |
+
const a = document.createElement('a');
|
| 1467 |
+
a.href = `/api/report/${sessionId}`;
|
| 1468 |
+
a.download = `sgo-report-${sessionId}.md`;
|
| 1469 |
+
document.body.appendChild(a);
|
| 1470 |
+
a.click();
|
| 1471 |
+
document.body.removeChild(a);
|
| 1472 |
+
}
|
| 1473 |
+
|
| 1474 |
// Boot
|
| 1475 |
init();
|
| 1476 |
</script>
|