"""Evidence completeness/safety checker for the single-device smoke folder. Read-only and offline. Does NOT call Gemini/ComfyUI/S3, does not read customer images, and never modifies evidence. Tri+ state result: evidence_check: OK | WARN | WARN_WITH_FAILURE_EVIDENCE | FAIL Meaning: OK - all required files present and summary status lines are safe. WARN - a non-critical evidence file is missing, or an expected status line could not be located. WARN_WITH_FAILURE_EVIDENCE - a real Gemini call was attempted and recorded as RUN_FAILED (honest failure evidence). This is NOT a safety violation. FAIL - a critical file is missing, an inconsistent/unsafe claim is present, or an API key value appears to be exposed. Gemini real-call lifecycle (all of these keep Pilot Ready NOT CONFIRMED): NOT RUN -> safe pre-call state (human QA PENDING) RUN_FAILED -> call attempted but failed; output may be MISSING -> WARN_WITH_FAILURE_EVIDENCE RUN -> call succeeded; output image MUST exist; human QA PENDING/PARTIAL FAIL conditions (unsafe / inconsistent): - Pilot Ready: CONFIRMED - single-device smoke: PASS - gemini_human_visual_qa PASS while real call is NOT RUN or RUN_FAILED - gemini_real_call RUN but output image missing - gemini_output_image PRESENT but the actual file is missing - "actual Gemini result verified" claimed but no output image - an API-key-like string exposed in the summary or in the Gemini log """ from __future__ import annotations import re import sys from pathlib import Path DEFAULT_EVIDENCE_DIR = Path( r"C:\Users\Admin\Documents\헤어\field-test-evidence" r"\v0.1.0-staging\2026-05-31-single-device-smoke-01" ) # (relative path, criticality): critical missing -> FAIL, otherwise WARN. REQUIRED_FILES = [ ("summary_report.md", "critical"), ("api-evidence/pytest-backend-final-with-gemini.txt", "warn"), ("api-evidence/pytest-gemini-only-final.txt", "warn"), ("api-evidence/gemini-provider-commit-readiness.md", "warn"), ("localStorage-evidence/localStorage-browser-screenshot.png", "warn"), ("face-only-qa/human-visual-qa-checklist.md", "warn"), ] _SEVERITY = {"OK": 0, "WARN": 1, "WARN_WITH_FAILURE_EVIDENCE": 2, "FAIL": 3} # Google AI Studio keys look like "AIza" + ~35 url-safe chars. Detect the prefix # pattern; never echo the matched value back. _API_KEY_RE = re.compile(r"AIza[0-9A-Za-z_\-]{20,}") def _worst(a: str, b: str) -> str: return a if _SEVERITY[a] >= _SEVERITY[b] else b def _values(text: str, key: str) -> list[str]: prefix = key.lower() + ":" out = [] for line in text.splitlines(): stripped = line.strip().lstrip("-* ").strip() if stripped.lower().startswith(prefix): out.append(stripped.split(":", 1)[1].strip()) return out def _first(text: str, key: str) -> str | None: vals = _values(text, key) return vals[0] if vals else None def _find_api_key_like(text: str) -> str | None: """Return a REDACTED indicator if the text looks like it contains an API key. The actual matched value is never returned, to avoid leaking it further. """ return "AIza...(redacted)" if _API_KEY_RE.search(text) else None def _output_png_exists(evidence_dir: Path) -> bool: out = evidence_dir / "gemini-smoke" / "output" return out.exists() and any(out.glob("gemini-output-*.png")) def evaluate_summary(text: str, *, output_png_exists: bool) -> tuple[str, list[str], dict]: """Evaluate gemini lifecycle + safety claims in a summary_report body.""" result = "OK" reasons: list[str] = [] real_call = _first(text, "gemini_real_call") human_qa = _first(text, "gemini_human_visual_qa") output_line = _first(text, "gemini_output_image") field_test = _first(text, "field_test") rc = (real_call or "").strip().upper() hq = (human_qa or "").strip().upper() oi = (output_line or "").strip().upper() is_not_run = rc.startswith("NOT RUN") is_run_failed = rc.startswith("RUN_FAILED") is_run = rc.startswith("RUN") and not is_run_failed hq_pass = hq.startswith("PASS") or "CONFIRMED" in hq output_present_claim = oi.startswith("PRESENT") output_missing_claim = oi.startswith("MISSING") # --- always-FAIL danger rules ------------------------------------- pilot = _values(text, "Pilot Ready") if not pilot: result = _worst(result, "WARN") reasons.append("WARN: Pilot Ready line not found") for val in pilot: u = val.upper() if "CONFIRMED" in u and "NOT CONFIRMED" not in u: result = _worst(result, "FAIL") reasons.append(f"FAIL: Pilot Ready is '{val}' (must be NOT CONFIRMED)") for val in _values(text, "single-device smoke"): if val.upper().startswith("PASS"): result = _worst(result, "FAIL") reasons.append("FAIL: single-device smoke is marked PASS") if field_test and field_test.upper().startswith("PASS"): result = _worst(result, "FAIL") reasons.append("FAIL: field_test is marked PASS") leak = _find_api_key_like(text) if leak: result = _worst(result, "FAIL") reasons.append(f"FAIL: possible API key value exposed in summary: {leak}") # output PRESENT claim but no actual file if output_present_claim and not output_png_exists: result = _worst(result, "FAIL") reasons.append("FAIL: gemini_output_image PRESENT but no output file found on disk") if "actual gemini result verified" in text.lower() and not output_png_exists: result = _worst(result, "FAIL") reasons.append("FAIL: 'actual Gemini result verified' claimed but no output image") # --- gemini_real_call lifecycle ----------------------------------- if real_call is None: result = _worst(result, "WARN") reasons.append("WARN: gemini_real_call line not found") elif is_not_run: if hq_pass: result = _worst(result, "FAIL") reasons.append("FAIL: human QA PASS while gemini_real_call is NOT RUN") elif is_run_failed: if hq_pass: result = _worst(result, "FAIL") reasons.append("FAIL: human QA PASS while gemini_real_call is RUN_FAILED") else: result = _worst(result, "WARN_WITH_FAILURE_EVIDENCE") reasons.append("INFO: gemini_real_call RUN_FAILED - failure evidence recorded (not a safety violation)") elif is_run: if output_missing_claim or not output_png_exists: result = _worst(result, "FAIL") reasons.append("FAIL: gemini_real_call RUN but output image is missing") else: result = _worst(result, "WARN") reasons.append(f"WARN: unrecognized gemini_real_call value '{real_call}'") if human_qa is None: result = _worst(result, "WARN") reasons.append("WARN: gemini_human_visual_qa line not found") observed = { "gemini_real_call": real_call, "gemini_human_visual_qa": human_qa, "gemini_output_image": output_line, "output_png_on_disk": output_png_exists, "field_test": field_test, "pilot_ready": pilot[0] if pilot else None, } return result, reasons, observed def _scan_logs_for_key(evidence_dir: Path) -> list[str]: reasons: list[str] = [] log_dir = evidence_dir / "gemini-smoke" / "logs" if not log_dir.exists(): return reasons for log in log_dir.glob("gemini-real-call-*.txt"): try: if _find_api_key_like(log.read_text(encoding="utf-8", errors="replace")): reasons.append(f"FAIL: possible API key value exposed in log: {log.name}") except OSError: continue return reasons def verify(evidence_dir: Path | str = DEFAULT_EVIDENCE_DIR) -> dict: evidence_dir = Path(evidence_dir) result = "OK" reasons: list[str] = [] files = [] for rel, crit in REQUIRED_FILES: path = evidence_dir / rel exists = path.exists() files.append({"path": rel, "exists": exists, "criticality": crit}) if not exists: if crit == "critical": result = _worst(result, "FAIL") reasons.append(f"FAIL: missing critical file: {rel}") else: result = _worst(result, "WARN") reasons.append(f"WARN: missing evidence file: {rel}") # API key leak in Gemini logs for r in _scan_logs_for_key(evidence_dir): result = _worst(result, "FAIL") reasons.append(r) observed: dict = {} summary = evidence_dir / "summary_report.md" if summary.exists(): text = summary.read_text(encoding="utf-8", errors="replace") s_result, s_reasons, observed = evaluate_summary( text, output_png_exists=_output_png_exists(evidence_dir) ) result = _worst(result, s_result) reasons.extend(s_reasons) return { "evidence_dir": str(evidence_dir), "files": files, "observed": observed, "reasons": reasons, "result": result, } def main(argv: list[str] | None = None) -> int: argv = list(sys.argv[1:] if argv is None else argv) evidence_dir = argv[0] if argv else DEFAULT_EVIDENCE_DIR report = verify(evidence_dir) print(f"evidence_dir: {report['evidence_dir']}") print("files:") for f in report["files"]: mark = "OK " if f["exists"] else ( "MISSING(FAIL)" if f["criticality"] == "critical" else "MISSING(WARN)") print(f" [{mark}] {f['path']}") if report["observed"]: print("observed status:") for k, v in report["observed"].items(): print(f" {k}: {v}") if report["reasons"]: print("notes:") for r in report["reasons"]: print(f" - {r}") print(f"evidence_check: {report['result']}") return 1 if report["result"] == "FAIL" else 0 if __name__ == "__main__": sys.exit(main())