| """Evidence completeness/safety checker for the single-device smoke folder.
|
|
|
| Read-only and offline. Does NOT call Gemini/ComfyUI/S3, does not read customer
|
| images, and never modifies evidence. Tri+ state result:
|
|
|
| evidence_check: OK | WARN | WARN_WITH_FAILURE_EVIDENCE | FAIL
|
|
|
| Meaning:
|
| OK - all required files present and summary status lines are safe.
|
| WARN - a non-critical evidence file is missing, or an expected status line
|
| could not be located.
|
| WARN_WITH_FAILURE_EVIDENCE - a real Gemini call was attempted and recorded as
|
| RUN_FAILED (honest failure evidence). This is NOT a safety violation.
|
| FAIL - a critical file is missing, an inconsistent/unsafe claim is present, or
|
| an API key value appears to be exposed.
|
|
|
| Gemini real-call lifecycle (all of these keep Pilot Ready NOT CONFIRMED):
|
| NOT RUN -> safe pre-call state (human QA PENDING)
|
| RUN_FAILED -> call attempted but failed; output may be MISSING -> WARN_WITH_FAILURE_EVIDENCE
|
| RUN -> call succeeded; output image MUST exist; human QA PENDING/PARTIAL
|
|
|
| FAIL conditions (unsafe / inconsistent):
|
| - Pilot Ready: CONFIRMED
|
| - single-device smoke: PASS
|
| - gemini_human_visual_qa PASS while real call is NOT RUN or RUN_FAILED
|
| - gemini_real_call RUN but output image missing
|
| - gemini_output_image PRESENT but the actual file is missing
|
| - "actual Gemini result verified" claimed but no output image
|
| - an API-key-like string exposed in the summary or in the Gemini log
|
| """
|
| from __future__ import annotations
|
|
|
| import re
|
| import sys
|
| from pathlib import Path
|
|
|
| DEFAULT_EVIDENCE_DIR = Path(
|
| r"C:\Users\Admin\Documents\헤어\field-test-evidence"
|
| r"\v0.1.0-staging\2026-05-31-single-device-smoke-01"
|
| )
|
|
|
|
|
| REQUIRED_FILES = [
|
| ("summary_report.md", "critical"),
|
| ("api-evidence/pytest-backend-final-with-gemini.txt", "warn"),
|
| ("api-evidence/pytest-gemini-only-final.txt", "warn"),
|
| ("api-evidence/gemini-provider-commit-readiness.md", "warn"),
|
| ("localStorage-evidence/localStorage-browser-screenshot.png", "warn"),
|
| ("face-only-qa/human-visual-qa-checklist.md", "warn"),
|
| ]
|
|
|
| _SEVERITY = {"OK": 0, "WARN": 1, "WARN_WITH_FAILURE_EVIDENCE": 2, "FAIL": 3}
|
|
|
|
|
|
|
| _API_KEY_RE = re.compile(r"AIza[0-9A-Za-z_\-]{20,}")
|
|
|
|
|
| def _worst(a: str, b: str) -> str:
|
| return a if _SEVERITY[a] >= _SEVERITY[b] else b
|
|
|
|
|
| def _values(text: str, key: str) -> list[str]:
|
| prefix = key.lower() + ":"
|
| out = []
|
| for line in text.splitlines():
|
| stripped = line.strip().lstrip("-* ").strip()
|
| if stripped.lower().startswith(prefix):
|
| out.append(stripped.split(":", 1)[1].strip())
|
| return out
|
|
|
|
|
| def _first(text: str, key: str) -> str | None:
|
| vals = _values(text, key)
|
| return vals[0] if vals else None
|
|
|
|
|
| def _find_api_key_like(text: str) -> str | None:
|
| """Return a REDACTED indicator if the text looks like it contains an API key.
|
|
|
| The actual matched value is never returned, to avoid leaking it further.
|
| """
|
| return "AIza...(redacted)" if _API_KEY_RE.search(text) else None
|
|
|
|
|
| def _output_png_exists(evidence_dir: Path) -> bool:
|
| out = evidence_dir / "gemini-smoke" / "output"
|
| return out.exists() and any(out.glob("gemini-output-*.png"))
|
|
|
|
|
| def evaluate_summary(text: str, *, output_png_exists: bool) -> tuple[str, list[str], dict]:
|
| """Evaluate gemini lifecycle + safety claims in a summary_report body."""
|
| result = "OK"
|
| reasons: list[str] = []
|
|
|
| real_call = _first(text, "gemini_real_call")
|
| human_qa = _first(text, "gemini_human_visual_qa")
|
| output_line = _first(text, "gemini_output_image")
|
| field_test = _first(text, "field_test")
|
|
|
| rc = (real_call or "").strip().upper()
|
| hq = (human_qa or "").strip().upper()
|
| oi = (output_line or "").strip().upper()
|
|
|
| is_not_run = rc.startswith("NOT RUN")
|
| is_run_failed = rc.startswith("RUN_FAILED")
|
| is_run = rc.startswith("RUN") and not is_run_failed
|
| hq_pass = hq.startswith("PASS") or "CONFIRMED" in hq
|
| output_present_claim = oi.startswith("PRESENT")
|
| output_missing_claim = oi.startswith("MISSING")
|
|
|
|
|
| pilot = _values(text, "Pilot Ready")
|
| if not pilot:
|
| result = _worst(result, "WARN")
|
| reasons.append("WARN: Pilot Ready line not found")
|
| for val in pilot:
|
| u = val.upper()
|
| if "CONFIRMED" in u and "NOT CONFIRMED" not in u:
|
| result = _worst(result, "FAIL")
|
| reasons.append(f"FAIL: Pilot Ready is '{val}' (must be NOT CONFIRMED)")
|
|
|
| for val in _values(text, "single-device smoke"):
|
| if val.upper().startswith("PASS"):
|
| result = _worst(result, "FAIL")
|
| reasons.append("FAIL: single-device smoke is marked PASS")
|
|
|
| if field_test and field_test.upper().startswith("PASS"):
|
| result = _worst(result, "FAIL")
|
| reasons.append("FAIL: field_test is marked PASS")
|
|
|
| leak = _find_api_key_like(text)
|
| if leak:
|
| result = _worst(result, "FAIL")
|
| reasons.append(f"FAIL: possible API key value exposed in summary: {leak}")
|
|
|
|
|
| if output_present_claim and not output_png_exists:
|
| result = _worst(result, "FAIL")
|
| reasons.append("FAIL: gemini_output_image PRESENT but no output file found on disk")
|
|
|
| if "actual gemini result verified" in text.lower() and not output_png_exists:
|
| result = _worst(result, "FAIL")
|
| reasons.append("FAIL: 'actual Gemini result verified' claimed but no output image")
|
|
|
|
|
| if real_call is None:
|
| result = _worst(result, "WARN")
|
| reasons.append("WARN: gemini_real_call line not found")
|
| elif is_not_run:
|
| if hq_pass:
|
| result = _worst(result, "FAIL")
|
| reasons.append("FAIL: human QA PASS while gemini_real_call is NOT RUN")
|
| elif is_run_failed:
|
| if hq_pass:
|
| result = _worst(result, "FAIL")
|
| reasons.append("FAIL: human QA PASS while gemini_real_call is RUN_FAILED")
|
| else:
|
| result = _worst(result, "WARN_WITH_FAILURE_EVIDENCE")
|
| reasons.append("INFO: gemini_real_call RUN_FAILED - failure evidence recorded (not a safety violation)")
|
| elif is_run:
|
| if output_missing_claim or not output_png_exists:
|
| result = _worst(result, "FAIL")
|
| reasons.append("FAIL: gemini_real_call RUN but output image is missing")
|
| else:
|
| result = _worst(result, "WARN")
|
| reasons.append(f"WARN: unrecognized gemini_real_call value '{real_call}'")
|
|
|
| if human_qa is None:
|
| result = _worst(result, "WARN")
|
| reasons.append("WARN: gemini_human_visual_qa line not found")
|
|
|
| observed = {
|
| "gemini_real_call": real_call,
|
| "gemini_human_visual_qa": human_qa,
|
| "gemini_output_image": output_line,
|
| "output_png_on_disk": output_png_exists,
|
| "field_test": field_test,
|
| "pilot_ready": pilot[0] if pilot else None,
|
| }
|
| return result, reasons, observed
|
|
|
|
|
| def _scan_logs_for_key(evidence_dir: Path) -> list[str]:
|
| reasons: list[str] = []
|
| log_dir = evidence_dir / "gemini-smoke" / "logs"
|
| if not log_dir.exists():
|
| return reasons
|
| for log in log_dir.glob("gemini-real-call-*.txt"):
|
| try:
|
| if _find_api_key_like(log.read_text(encoding="utf-8", errors="replace")):
|
| reasons.append(f"FAIL: possible API key value exposed in log: {log.name}")
|
| except OSError:
|
| continue
|
| return reasons
|
|
|
|
|
| def verify(evidence_dir: Path | str = DEFAULT_EVIDENCE_DIR) -> dict:
|
| evidence_dir = Path(evidence_dir)
|
| result = "OK"
|
| reasons: list[str] = []
|
| files = []
|
|
|
| for rel, crit in REQUIRED_FILES:
|
| path = evidence_dir / rel
|
| exists = path.exists()
|
| files.append({"path": rel, "exists": exists, "criticality": crit})
|
| if not exists:
|
| if crit == "critical":
|
| result = _worst(result, "FAIL")
|
| reasons.append(f"FAIL: missing critical file: {rel}")
|
| else:
|
| result = _worst(result, "WARN")
|
| reasons.append(f"WARN: missing evidence file: {rel}")
|
|
|
|
|
| for r in _scan_logs_for_key(evidence_dir):
|
| result = _worst(result, "FAIL")
|
| reasons.append(r)
|
|
|
| observed: dict = {}
|
| summary = evidence_dir / "summary_report.md"
|
| if summary.exists():
|
| text = summary.read_text(encoding="utf-8", errors="replace")
|
| s_result, s_reasons, observed = evaluate_summary(
|
| text, output_png_exists=_output_png_exists(evidence_dir)
|
| )
|
| result = _worst(result, s_result)
|
| reasons.extend(s_reasons)
|
|
|
| return {
|
| "evidence_dir": str(evidence_dir),
|
| "files": files,
|
| "observed": observed,
|
| "reasons": reasons,
|
| "result": result,
|
| }
|
|
|
|
|
| def main(argv: list[str] | None = None) -> int:
|
| argv = list(sys.argv[1:] if argv is None else argv)
|
| evidence_dir = argv[0] if argv else DEFAULT_EVIDENCE_DIR
|
| report = verify(evidence_dir)
|
|
|
| print(f"evidence_dir: {report['evidence_dir']}")
|
| print("files:")
|
| for f in report["files"]:
|
| mark = "OK " if f["exists"] else (
|
| "MISSING(FAIL)" if f["criticality"] == "critical" else "MISSING(WARN)")
|
| print(f" [{mark}] {f['path']}")
|
| if report["observed"]:
|
| print("observed status:")
|
| for k, v in report["observed"].items():
|
| print(f" {k}: {v}")
|
| if report["reasons"]:
|
| print("notes:")
|
| for r in report["reasons"]:
|
| print(f" - {r}")
|
| print(f"evidence_check: {report['result']}")
|
| return 1 if report["result"] == "FAIL" else 0
|
|
|
|
|
| if __name__ == "__main__":
|
| sys.exit(main())
|
|
|
