#!/usr/bin/env python3 """Build a judge-facing award readiness scorecard for ProofFrame.""" from __future__ import annotations import argparse import importlib.util import json from pathlib import Path from typing import Any from proofframe.submission_gate import build_submission_gate ROOT = Path(__file__).resolve().parents[1] DEFAULT_JSON = ROOT / "docs" / "assets" / "award-readiness-report.json" DEFAULT_MD = ROOT / "docs" / "assets" / "award-readiness-report.md" def load_script(module_name: str, relative_path: str): spec = importlib.util.spec_from_file_location(module_name, ROOT / relative_path) module = importlib.util.module_from_spec(spec) assert spec.loader is not None spec.loader.exec_module(module) return module claim_lint = load_script("claim_lint", "scripts/claim_lint.py") demo_readiness = load_script("demo_readiness", "scripts/demo_readiness.py") secret_scan = load_script("secret_scan", "scripts/secret_scan.py") def load_json(path: Path) -> dict[str, Any] | None: try: return json.loads(path.read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError): return None def read_text(path: Path) -> str: try: return path.read_text(encoding="utf-8") except FileNotFoundError: return "" def task_statuses(root: Path) -> dict[str, str]: tasks = load_json(root / "tasks.json") or {} return { str(task.get("id", "")).upper(): str(task.get("status", "missing")) for task in tasks.get("tasks", []) } def has_text(root: Path, relative_path: str, needle: str) -> bool: return needle in read_text(root / relative_path) def has_any_text(root: Path, relative_path: str, needles: list[str]) -> bool: text = read_text(root / relative_path).lower() return any(needle.lower() in text for needle in needles) def path_present(root: Path, relative_path: str) -> bool: return (root / relative_path).exists() def run_secret_scan(root: Path) -> list[str]: original_root = secret_scan.ROOT secret_scan.ROOT = root try: return secret_scan.run_scan(root) finally: secret_scan.ROOT = original_root def signal(signal_id: str, label: str, ok: bool, points: int, detail: str) -> dict[str, Any]: return { "id": signal_id, "label": label, "ok": ok, "points": points if ok else 0, "max_points": points, "detail": detail, } def criterion(criterion_id: str, label: str, signals: list[dict[str, Any]]) -> dict[str, Any]: earned = sum(item["points"] for item in signals) maximum = sum(item["max_points"] for item in signals) return { "id": criterion_id, "label": label, "score": earned, "max_score": maximum, "percent": round(earned / maximum * 100, 1) if maximum else 0, "signals": signals, } def b2_setup_ready(root: Path) -> bool: setup = load_json(root / "docs" / "assets" / "b2-live-setup.json") or {} bucket_name = setup.get("bucket_name") or setup.get("bucket") return bool(bucket_name and setup.get("endpoint") and setup.get("bucket_type") == "private") def packet_has_public_demo(root: Path) -> bool: packet = load_json(root / "docs" / "assets" / "devpost-submission-packet.json") or {} return str(packet.get("demo_url", "")).endswith("/?judge=1") def b2_evidence_ready(root: Path) -> bool: evidence = load_json(root / "docs" / "assets" / "b2-live-proof-evidence.json") or {} return bool( evidence.get("ok") and evidence.get("storage_backend") == "b2" and evidence.get("asset_storage_backend") == "b2" and evidence.get("asset_sha256") and evidence.get("manifest_sha256") ) def sponsor_fit_audit_ready(root: Path) -> bool: report = load_json(root / "docs" / "assets" / "sponsor-fit-audit.json") or {} return bool(report.get("ok") and report.get("mode") == "sponsor_fit_ready") def event_snapshot_ready(root: Path) -> bool: report = load_json(root / "docs" / "assets" / "devpost-event-snapshot.json") or {} criteria = report.get("rules", {}).get("judging_criteria", []) requirements = report.get("rules", {}).get("requirements", {}) return bool( report.get("schema") == "proofframe.devpost_event_snapshot.v1" and report.get("validation", {}).get("ok") and report.get("validation", {}).get("submission_open") and all(item.get("present") for item in criteria) and all(requirements.get(key) for key in ["working_app_url", "github_repo_url", "demo_video"]) ) def build_criteria(root: Path, context: dict[str, Any]) -> list[dict[str, Any]]: statuses = context["task_statuses"] demo = context["demo_readiness"] claim = context["claim_lint"] secret_findings = context["secret_findings"] gate = context["submission_gate"] return [ criterion( "sponsor_fit", "Sponsor integration fit", [ signal( "b2_backend_code", "B2 storage backend is implemented", has_text(root, "src/proofframe/storage.py", "class B2StorageBackend"), 5, "Backblaze B2 has a dedicated S3-compatible storage adapter.", ), signal( "b2_bucket_setup", "Dedicated private B2 bucket is recorded", b2_setup_ready(root), 4, "The non-secret B2 bucket setup record exists for final proof.", ), signal( "b2_live_runner", "B2-only live proof runner exists", path_present(root, "scripts/run_b2_live_proof.py"), 4, "B2 storage can be verified independently before Genblaze is ready.", ), signal( "genblaze_provider_code", "Genblaze provider path is implemented", has_text(root, "src/proofframe/providers.py", "class GenblazeMediaProvider"), 5, "The app has Genblaze provider adapters for GMICloud, OpenAI, and credential-free local Pipeline proof.", ), signal( "final_live_runner", "Final B2 plus Genblaze runner exists", path_present(root, "scripts/run_final_live_proof.py"), 4, "A one-command runner can produce sanitized final evidence with the configured Genblaze provider.", ), signal( "sponsor_fit_matrix", "Sponsor fit matrix is documented", path_present(root, "docs/sponsor_fit_matrix.md"), 4, "Judging angles are mapped to current evidence, safe claims, final gates, and demo shots.", ), signal( "sponsor_fit_audit", "Sponsor fit audit is green", sponsor_fit_audit_ready(root), 4, "Devpost B2/Genblaze copy is specific and the demo introduces B2 early.", ), signal( "b2_live_evidence", "Live B2 evidence is captured", statuses.get("T020") == "done" and b2_evidence_ready(root), 4, "T020 requires a real B2 media and manifest proof.", ), signal( "genblaze_live_evidence", "Live Genblaze evidence is captured", statuses.get("T021") == "done" and gate["evidence_gate"]["ok"], 4, "T021 requires provider/model metadata from a live Genblaze run.", ), ], ), criterion( "product_depth", "Provenance product depth", [ signal( "manifest_model", "Manifest model captures provenance", has_text(root, "src/proofframe/models.py", "CampaignManifest"), 4, "Assets, prompts, provider/model metadata, hashes, and review state are modeled.", ), signal( "packet_zip", "Evidence packet ZIP endpoint exists", has_text(root, "src/proofframe/app.py", "packet.zip"), 4, "Judges can inspect a packaged manifest and local media evidence.", ), signal( "judge_packet", "One-click judge packet exists", has_text(root, "src/proofframe/app.py", "/api/demo/judge-packet"), 4, "The demo can be loaded quickly into a believable reviewer workflow.", ), signal( "review_console", "Review console UI is present", has_any_text( root, "apps/web/index.html", ["review console", "review-console", 'aria-label="Review console"'], ), 4, "The app is positioned as an operations desk, not a generic generator.", ), signal( "sponsor_model_ui", "Sponsor evidence model is visible in the UI", has_text(root, "apps/web/index.html", "Sponsor Evidence Model"), 3, "Judge mode foregrounds Genblaze, B2 object route, manifest proof, and claim mode.", ), signal( "prd_spec", "PRD and technical spec are present", path_present(root, "docs/prd.md") and path_present(root, "docs/spec.md"), 4, "The product and implementation story are documented for judges and maintainers.", ), ], ), criterion( "demo_and_submission", "Demo and Devpost readiness", [ signal( "public_judge_demo", "Public judge-mode demo URL is packaged", packet_has_public_demo(root), 5, "The Devpost packet points judges to a credential-free public demo.", ), signal( "devpost_form_kit", "Devpost form kit is generated", path_present(root, "docs/assets/devpost-form-kit.json") and path_present(root, "docs/assets/devpost-form-kit.md"), 4, "Field-by-field copy is ready and length checked.", ), signal( "official_event_snapshot", "Official Devpost snapshot is fresh", event_snapshot_ready(root), 4, "Deadline, participants, requirements, and judging criteria are refreshed from Devpost.", ), signal( "storyboard", "Demo storyboard is machine-checked", path_present(root, "docs/assets/demo-storyboard.json") and path_present(root, "docs/assets/demo-storyboard.md"), 4, "The video can be recorded around a clear judge story.", ), signal( "mock_recording_ready", "Mock recording package is ready", bool(demo["mock_recording_ready"]), 4, "The current public demo can be recorded safely before live proof.", ), signal( "screenshots", "Key screenshots are captured", all( path_present(root, path) for path in [ "docs/assets/proofframe-local-ui-smoke.png", "docs/assets/proofframe-review-console-smoke.png", "docs/assets/proofframe-hf-public-smoke.png", ] ), 4, "Local, review-console, and public demo screenshots are available.", ), signal( "devpost_registered", "Devpost registration is done", statuses.get("T040") == "done", 4, "The project is through the registration gate.", ), ], ), criterion( "trust_and_compliance", "Trust, safety, and claim discipline", [ signal( "secret_scan", "Secret scan is clean", not secret_findings, 4, "No obvious API keys, cookies, signed URLs, or tokens were found in public files.", ), signal( "claim_lint", "Public claim lint is clean", bool(claim["ok"]), 4, "Public copy only claims B2 or Genblaze proof when sanitized evidence is present.", ), signal( "env_ignored", "Local final env file is ignored", has_text(root, ".gitignore", ".env.final.local"), 3, "Credential-bearing local env files stay out of Git.", ), signal( "safe_evidence_writer", "Evidence writer rejects secret-like values", has_text(root, "scripts/api_smoke.py", "assert_safe_evidence"), 2, "Live evidence cannot silently include key-like fields or signed URLs.", ), signal( "claim_freeze_doc", "Public claim freeze is documented", path_present(root, "docs/public_claim_freeze.md"), 2, "The team has a written boundary for what can be said before final proof.", ), ], ), criterion( "final_closure", "Final submission closure", [ signal( "final_gate", "Final live proof gate is green", bool(gate["ok"]), 6, "Requires T020, T021, T040, T041, T041A, T042, final live evidence, and final reports.", ), signal( "final_audit_done", "Final audit task is done", statuses.get("T041") == "done", 2, "The reviewer audit must pass after live evidence exists.", ), signal( "final_secret_scan_done", "Final secret scan task is done", statuses.get("T041A") == "done", 1, "The final scan should run after live proof and final public submission artifacts are generated.", ), signal( "devpost_submitted", "Devpost project is submitted", statuses.get("T042") == "done", 1, "The submission is not complete until the project page is submitted.", ), ], ), ] def build_next_actions(report: dict[str, Any]) -> list[str]: statuses = report["task_statuses"] actions: list[str] = [] if statuses.get("T020") != "done": actions.append( "After explicit key-creation confirmation, create the scoped B2 key and run " "python scripts/run_b2_live_proof.py --env-file .env.final.local." ) if statuses.get("T021") != "done": actions.append( "Run the credential-free local Genblaze Pipeline proof with python scripts/run_final_live_proof.py " "--env-file .env.final.local --genblaze-provider local --genblaze-image-model local-svg-v1." ) if statuses.get("T041A") != "done": if statuses.get("T020") == "done" and statuses.get("T021") == "done": actions.append("Run python scripts/secret_scan.py after the public video and Devpost-safe artifacts are staged.") else: actions.append("Run python scripts/secret_scan.py after live evidence is generated.") if statuses.get("T041") != "done": actions.append( "Run python scripts/submission_audit.py --strict-final after live proof, public video URL, and T041A are done." ) if statuses.get("T042") != "done": actions.append("Submit the Devpost project only after the final gate turns green.") return actions[:5] def readiness_mode(score: int, gate_ok: bool, evidence_status: str | None = None) -> str: if gate_ok and score >= 90: return "final_award_ready" if evidence_status == "verified" and score >= 75: return "proof_verified_video_pending" if score >= 75: return "pre_live_competitive" return "needs_polish" def readiness_interpretation(criteria: list[dict[str, Any]], gate_ok: bool, mode: str) -> dict[str, Any]: by_id = {item["id"]: item for item in criteria} final_closure = by_id.get("final_closure", {}) demo_submission = by_id.get("demo_and_submission", {}) final_closure_score = int(final_closure.get("score") or 0) final_closure_max = int(final_closure.get("max_score") or 0) return { "mode": mode, "pre_live_competitive": mode == "pre_live_competitive", "final_award_ready": mode == "final_award_ready", "submission_gate_ok": gate_ok, "demo_and_submission_score": demo_submission.get("score"), "demo_and_submission_max_score": demo_submission.get("max_score"), "final_closure_score": final_closure_score, "final_closure_max_score": final_closure_max, "boundary": ( "Final award readiness is complete." if gate_ok else "B2 and Genblaze proof are verified; final award readiness still requires the public video, final audit, final secret scan, and Devpost receipt." if mode == "proof_verified_video_pending" else "Pre-live score reflects product, demo, and documentation strength; final award readiness still requires live proof, final audit, and Devpost receipt." ), } def build_report(root: Path = ROOT) -> dict[str, Any]: root = root.resolve() context = { "task_statuses": task_statuses(root), "submission_gate": build_submission_gate(root), "demo_readiness": demo_readiness.build_report(root), "claim_lint": claim_lint.build_claim_report(root), "secret_findings": run_secret_scan(root), } criteria = build_criteria(root, context) score = sum(item["score"] for item in criteria) max_score = sum(item["max_score"] for item in criteria) mode = readiness_mode( score, context["submission_gate"]["ok"], context["submission_gate"]["evidence_gate"]["status"], ) report = { "schema": "proofframe.award_readiness.v1", "project": "ProofFrame", "score": score, "max_score": max_score, "percent": round(score / max_score * 100, 1) if max_score else 0, "mode": mode, "public_demo_url": ( load_json(root / "docs" / "assets" / "devpost-submission-packet.json") or {} ).get("demo_url"), "task_statuses": { task: context["task_statuses"].get(task, "missing") for task in ["T020", "T021", "T040", "T041", "T041A", "T042"] }, "submission_gate": { "ok": context["submission_gate"]["ok"], "mode": context["submission_gate"]["mode"], "summary": context["submission_gate"]["summary"], "evidence_status": context["submission_gate"]["evidence_gate"]["status"], "report_status": context["submission_gate"]["report_gate"]["status"], }, "demo_readiness": { "mode": context["demo_readiness"]["mode"], "mock_recording_ready": context["demo_readiness"]["mock_recording_ready"], "final_recording_ready": context["demo_readiness"]["final_recording_ready"], }, "claim_lint": { "ok": context["claim_lint"]["ok"], "mode": context["claim_lint"]["mode"], "finding_count": len(context["claim_lint"]["findings"]), }, "secret_scan": { "ok": not context["secret_findings"], "finding_count": len(context["secret_findings"]), }, "criteria": criteria, "readiness_interpretation": readiness_interpretation( criteria, context["submission_gate"]["ok"], mode, ), } report["next_actions"] = build_next_actions(report) return report def render_markdown(report: dict[str, Any]) -> str: lines = [ "# ProofFrame Award Readiness", "", f"Mode: `{report['mode']}`", f"Score: `{report['score']}/{report['max_score']}` ({report['percent']}%)", f"Public demo: {report['public_demo_url']}", "", "## Gate Snapshot", "", f"- Submission gate: `{report['submission_gate']['mode']}`", f"- Live evidence: `{report['submission_gate']['evidence_status']}`", f"- Final reports: `{report['submission_gate']['report_status']}`", f"- Mock recording ready: `{str(report['demo_readiness']['mock_recording_ready']).lower()}`", f"- Final recording ready: `{str(report['demo_readiness']['final_recording_ready']).lower()}`", f"- Claim lint: `{str(report['claim_lint']['ok']).lower()}`", f"- Secret scan: `{str(report['secret_scan']['ok']).lower()}`", f"- Final closure: `{report['readiness_interpretation']['final_closure_score']}/{report['readiness_interpretation']['final_closure_max_score']}`", f"- Interpretation: {report['readiness_interpretation']['boundary']}", "", "## Criteria", "", ] for item in report["criteria"]: lines.append( f"### {item['label']} - {item['score']}/{item['max_score']} ({item['percent']}%)" ) for sig in item["signals"]: marker = "OK" if sig["ok"] else "TODO" lines.append( f"- {marker} `{sig['id']}` ({sig['points']}/{sig['max_points']}): {sig['detail']}" ) lines.append("") lines.extend(["## Next Actions", ""]) if report["next_actions"]: lines.extend(f"- {action}" for action in report["next_actions"]) else: lines.append("- Ready to submit and defend with verified live evidence.") return "\n".join(lines) + "\n" def write_outputs(report: dict[str, Any], json_path: Path, markdown_path: Path) -> None: json_path.parent.mkdir(parents=True, exist_ok=True) markdown_path.parent.mkdir(parents=True, exist_ok=True) json_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") markdown_path.write_text(render_markdown(report), encoding="utf-8") def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Build a ProofFrame award readiness scorecard.") parser.add_argument("--root", type=Path, default=ROOT) parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON) parser.add_argument("--markdown-out", type=Path, default=DEFAULT_MD) parser.add_argument("--min-score", type=int, default=75) parser.add_argument( "--strict-final", action="store_true", help="Fail unless the final B2 plus Genblaze gate is award-ready.", ) return parser def main() -> None: args = build_parser().parse_args() report = build_report(args.root) write_outputs(report, args.json_out, args.markdown_out) ok = report["mode"] == "final_award_ready" if args.strict_final else report["score"] >= args.min_score print( json.dumps( { "ok": ok, "mode": report["mode"], "score": report["score"], "max_score": report["max_score"], "json": str(args.json_out), "markdown": str(args.markdown_out), "next_actions": report["next_actions"], }, indent=2, ) ) raise SystemExit(0 if ok else 2) if __name__ == "__main__": main()