| """Summarize completed run artifacts without fitting or changing a policy.""" |
|
|
| import collections |
| import csv |
| import json |
| from pathlib import Path |
|
|
| import numpy as np |
|
|
| root = Path(__file__).resolve().parents[1] |
| reports = root / "reports" |
| primary = {} |
| for split in ["development", "final", "ood"]: |
| folder = reports / "runs" / f"{split}-primary" |
| primary[split] = json.loads((folder / "summary.json").read_text()) |
| manifest = json.loads((reports / "runs/final-primary/manifest.json").read_text()) |
| metadata = manifest["model"] |
| hardware = json.loads((reports / "hardware.json").read_text()) |
| lines = [ |
| "# Seal Cell: measured zero-shot evaluation", |
| "", |
| "This release implements the local simulator, bounded supervisor, three main policies, " |
| "two control/ablation policies, evaluation CLI and Gradio dashboard. " |
| "The results below are actual local CPU inference on synthetic episodes. " |
| "No threshold, label, schema or policy was tuned on the final split.", |
| "", |
| f"Checkpoint: `{metadata['model_id']}@{metadata['revision']}`. Device/dtype: CPU / float32. " |
| f"Exact input limit: {metadata['input_limit']} tokens including schema. " |
| f"Measured model construction/weight load: {metadata['load_ms']:.0f} ms (Python imports and warm-up excluded).", |
| "", |
| f"Host: {hardware.get('cpu', hardware['machine'])}, " |
| f"{hardware.get('logical_cpus', 'unknown')} logical CPUs, " |
| f"{hardware.get('memory_bytes', 0) / 2**30:.0f} GiB RAM, " |
| f"{hardware['torch_threads']} PyTorch CPU threads, Python {hardware['python']}. " |
| f"Versions: {', '.join(f'{k} {v}' for k, v in metadata['versions'].items())}. " |
| "Wall-clock measurements were taken on an interactive workstation, not an isolated benchmark host.", |
| "", |
| "## Held-out physical outcomes", |
| "", |
| "1,000 paired scenario seeds per policy; recipe C and a distinct inspection-text family. " |
| "The 13 fault presets have fixed magnitudes with bounded observation/quality noise. " |
| "This measures variability across seeds, not broad process robustness.", |
| "", |
| "| Policy | Automated batches | Recovery on recoverable cases (95% CI) | Bad / released pouches | Good / simulated hour | Review |", |
| "|---|---:|---:|---:|---:|---:|", |
| ] |
| for name, m in primary["final"]["policies"].items(): |
| ci = m["recovery_ci95"] |
| lines.append( |
| f"| {name} | {m['automation_rate']:.1%} | {m['recovery_rate_recoverable']:.1%} ({ci[0]:.1%}–{ci[1]:.1%}) | {m['bad_released']} / {m['released_denominator']} | {m['good_per_simulated_hour']:.1f} | {m['review_rate']:.1%} |" |
| ) |
| lines += [ |
| "", |
| "Recovery means a completed batch with no bad pouch. Automation counts completed batches. " |
| "Intervals use 1,000 episode-bootstrap replicates, never independent-pouch sampling. " |
| "The hold policy has zero coverage and throughput; its defect rate is undefined, not a perfect score.", |
| "", |
| "## Proposals, gates and latency", |
| "", |
| "| Policy | Invalid proposal / gate rejection | Model errors | Scrap | Mean corrections | Warm decision p50 / p95 ms |", |
| "|---|---:|---:|---:|---:|---:|", |
| ] |
| for name, m in primary["final"]["policies"].items(): |
| latency = m["warm_latency_ms"] |
| lines.append( |
| f"| {name} | {m['gate_rejection_rate']:.1%} | {m['model_errors']} | {m['scrap']} | {m['mean_corrections']:.2f} | {latency['p50']:.2f} / {latency['p95']:.2f} |" |
| ) |
| lines += [ |
| "", |
| "Every invalid top proposal ends in HOLD, so the invalid-proposal and gate-rejection rates coincide. " |
| "Warm decision latency includes serialization, budget checks, inference and policy mapping; rules " |
| "use the same timed decision boundary. Cold construction and warm-up are excluded. " |
| "The simulator does not advance while inference is running.", |
| "", |
| ] |
| counts = collections.defaultdict(collections.Counter) |
| with (reports / "runs/final-primary/episodes.csv").open() as f: |
| for row in csv.DictReader(f): |
| if row["reason"]: |
| counts[row["policy"]][row["reason"]] += 1 |
| for name in ["hybrid", "direct"]: |
| lines.append( |
| f"**{name} terminal reasons:** " |
| + "; ".join(f"{reason} ({n} episodes)" for reason, n in counts[name].most_common()) |
| + "." |
| ) |
| trimmed = collections.Counter() |
| inference = collections.defaultdict(list) |
| invalid_mass = [] |
| with (reports / "runs/final-primary/decisions.jsonl").open() as f: |
| for line in f: |
| row = json.loads(line) |
| record = row["proposal"]["model"] |
| if record: |
| trimmed[row["policy"]] += bool(record.get("removed_oldest_history")) |
| if not record.get("error"): |
| inference[row["policy"]].append(record["inference_ms"]) |
| if row["invalid_action_probability"] is not None: |
| invalid_mass.append(row["invalid_action_probability"]) |
| lines += [ |
| "", |
| "Successful model calls, inference-only p50/p95: " |
| + "; ".join( |
| f"{name} {np.quantile(values, 0.5):.1f}/{np.quantile(values, 0.95):.1f} ms" |
| for name, values in inference.items() |
| ) |
| + ".", |
| f"Calls with explicitly logged oldest-history removal: {dict(trimmed)}. " |
| f"Mean probability mass assigned to invalid direct actions: {np.mean(invalid_mass):.1%}.", |
| "", |
| "## Fixture diagnosis and action agreement", |
| "", |
| "Five original three-head probes were rerun verbatim. The selected labels match the " |
| "playground observations; full precision local scores, raw logits and timings are preserved. " |
| "The original hosted scores were rounded and its checkpoint revision was not exposed.", |
| "", |
| "| Probe | Local issue | Local action | Reference-action agreement |", |
| "|---|---|---|---|", |
| ] |
| probes = [json.loads(line) for line in (reports / "local-probes.jsonl").read_text().splitlines()] |
| for r in probes: |
| h = r["heads"] |
| lines.append( |
| f"| {r['fixture_id']} | {h['suspected_issue']['selected_label']} | {h['next_action']['selected_label']} | {r['action_correct']} |" |
| ) |
| fixtures = [ |
| json.loads(line) for line in (reports / "development-fixtures.jsonl").read_text().splitlines() |
| ] |
| lines += [ |
| "", |
| "The 24 frozen development fixtures are 12 matched raw/derived pairs, including missing/stale " |
| "sensors, negation, resolved history, repeated ineffective correction and contradictory text. " |
| "Expected diagnoses can be sets. These are development checks, not an estimate of held-out accuracy.", |
| "", |
| ] |
| for condition in ["raw", "derived"]: |
| selected = [r for r in fixtures if r["fixture_id"].endswith(condition)] |
| lines.append( |
| f"- {condition}: {sum(r['diagnosis_correct'] is True for r in selected)}/{len(selected)} diagnosis agreements; {sum(bool(r['error']) for r in selected)} model errors." |
| ) |
| lines += [ |
| "", |
| "The exported development error-versus-coverage curve is descriptive. The 0.5 issue/evidence " |
| "thresholds were frozen prototype operating points. No temperature scaling was fitted and no " |
| "score is advertised as calibrated confidence. Separate calibrated policy selection remains future work.", |
| "", |
| "## Development robustness and OOD", |
| "", |
| "| Development condition | Episodes / policy | Hybrid automation | Direct automation | Model errors (hybrid / direct) |", |
| "|---|---:|---:|---:|---:|", |
| ] |
| for condition in [ |
| "primary", |
| "raw", |
| "three-heads", |
| "no-maintenance", |
| "reverse-labels", |
| "descriptions", |
| "paraphrase", |
| ]: |
| result = json.loads((reports / f"runs/development-{condition}/summary.json").read_text())[ |
| "policies" |
| ] |
| h, d = result["hybrid"], result["direct"] |
| lines.append( |
| f"| {condition} | {h['episodes']} | {h['automation_rate']:.1%} | {d['automation_rate']:.1%} | {h['model_errors']} / {d['model_errors']} |" |
| ) |
| lines += [ |
| "", |
| "The primary development run has 200 episodes; each robustness check has 26 matched development " |
| "episodes. No condition was selected based on final results. Descriptions use the library-supported " |
| "label-description schema; reversed order retains identical meanings. The paraphrase condition " |
| "changes the introductory observation wording and is only a narrow robustness check.", |
| "", |
| "The separate OOD run uses 100 episodes (recipe B, reserved combined-fault and unseen-material scenarios):", |
| "", |
| ] |
| for name, m in primary["ood"]["policies"].items(): |
| lines.append( |
| f"- {name}: {m['automation_rate']:.1%} automation, {m['bad_released']}/{m['released_denominator']} bad/released pouches." |
| ) |
| lines += [ |
| "", |
| "## Interpretation and scope", |
| "", |
| "The competent rules controller is the stronger controller in this toy domain. " |
| "These zero-shot GLiNER policies do not establish learned planning or an advantage over rules. " |
| "Their zero defect release must be read together with automation and throughput. " |
| "The proposed research target (90% recoverable recovery and 70% automation, with no worse " |
| "defect release than rules) is not achieved by the model policies. The current balanced fault " |
| "mix contains five unrecoverable presets out of thirteen, so even perfect recovery cannot " |
| "reach 70% batch automation across this entire population; that research target needs a " |
| "separately predefined operational scenario mix.", |
| "", |
| "Engineering validation covers deterministic replay, causal recovery, unrecoverable faults, " |
| "observation isolation, units, stale/missing sensors, fresh sample qualification, all budgets, " |
| "invalid model proposals, model failure and stale in-flight UI results. " |
| "The real-model integration check verifies full distributions and overlength rejection. " |
| "The dashboard was inspected in native Chrome; live Gradio endpoint smoke tests cover " |
| "load/run/step/reset/pause, exported traces and two-session isolation.", |
| "", |
| "Known limits: fixed toy fault severities; perfect threshold-based inspection of noisy synthetic " |
| "quality; narrow lexical baseline; no trained TF-IDF/tabular baseline, no fine-tuning, no fitted " |
| "calibration; no hard cancellation of a stuck native inference call. " |
| "All coefficients are synthetic, and confidence intervals do not imply industrial assurance.", |
| "", |
| "## Reproduce and inspect", |
| "", |
| "An earlier engineering run was stopped during an input-parity audit: the renderer omitted " |
| "material thickness, recipe targets, approved alternate profiles and sample freshness context. " |
| "That partial run is archived under `runs/superseded-input-v1/` and excluded here. " |
| "The delivered v2 evaluation includes those fields; labels and thresholds were unchanged.", |
| "", |
| "- [README and uv commands](../README.md)", |
| "- [Separate MPS acceleration study](mps.md)", |
| "- [Final full report](runs/final-primary/report.md), [summary](runs/final-primary/summary.json), [manifest](runs/final-primary/manifest.json)", |
| "- [Development report](runs/development-primary/report.md)", |
| "- [OOD report](runs/ood-primary/report.md)", |
| "- [Local probes](local-probes.jsonl), [development fixture outputs](development-fixtures.jsonl), [coverage curve](development-fixtures.jsonl.coverage.json)", |
| "- Each run directory also contains complete `decisions.jsonl` and `episodes.csv`.", |
| "", |
| ] |
| (reports / "zero-shot.md").write_text("\n".join(lines)) |
| print(reports / "zero-shot.md") |
|
|
