import json import os import re from dataclasses import asdict, dataclass from typing import Dict, List @dataclass class FailureRecord: card_no: str ability_idx: str segment: str error_type: str effect_text: str expected: str actual: str raw_message: str @dataclass class ErrorPattern: pattern_id: str error_type: str effect: str count: int examples: List[str] value_mismatches: Dict[str, int] def parse_audit_report(filepath: str) -> List[FailureRecord]: failures = [] if not os.path.exists(filepath): print(f"File not found: {filepath}") return [] with open(filepath, "r", encoding="utf-8") as f: lines = f.readlines() # Regex for ability-level reporting rows # | Card No | Ability | Status | Details | # | PL!-PR-001-PR | Ab0 | ❌ FAIL | Stuck at segment 0: Mismatch OPPONENT_HAND_DELTA for 'EFFECT: ACTIVATE_MEMBER(1)': Exp 1, Got 0 | row_re = re.compile(r"\| (.*?) \| (.*?) \| ❌ FAIL \| (.*) \|") # Details parser: Stuck at segment (\d+): Mismatch (.*?) for '(.*?)': Exp (.*?), Got (.*) details_re = re.compile(r"Stuck at segment (\d+): Mismatch (.*?) for '(.*?)': Exp (.*?), Got (.*)") for line in lines: match = row_re.search(line) if match: card_no = match.group(1).strip() ability_idx = match.group(2).strip() details = match.group(3).strip() detail_match = details_re.search(details) if detail_match: segment = detail_match.group(1) error_type = detail_match.group(2) effect_text = detail_match.group(3) expected = detail_match.group(4) actual = detail_match.group(5) failures.append( FailureRecord( card_no=card_no, ability_idx=ability_idx, segment=segment, error_type=error_type, effect_text=effect_text, expected=expected, actual=actual, raw_message=details, ) ) else: # Handle other failure formats (e.g., panics or different mismatch phrasing) failures.append( FailureRecord( card_no=card_no, ability_idx=ability_idx, segment="?", error_type="OTHER", effect_text="?", expected="?", actual="?", raw_message=details, ) ) return failures def get_effect_category(effect_text: str) -> str: # Normalize effect text for grouping # e.g., "EFFECT: ACTIVATE_MEMBER(1)" -> "ACTIVATE_MEMBER" if "EFFECT:" in effect_text: match = re.search(r"EFFECT: ([A-Z_]+)", effect_text) if match: return match.group(1) return effect_text def analyze_failures(failures: List[FailureRecord]) -> Dict[str, ErrorPattern]: patterns = {} for f in failures: effect = get_effect_category(f.effect_text) pattern_id = f"{f.error_type}_{effect}" if pattern_id not in patterns: patterns[pattern_id] = ErrorPattern( pattern_id=pattern_id, error_type=f.error_type, effect=effect, count=0, examples=[], value_mismatches={} ) p = patterns[pattern_id] p.count += 1 if len(p.examples) < 5: p.examples.append(f"{f.card_no} ({f.ability_idx})") mismatch_key = f"Exp {f.expected}, Got {f.actual}" p.value_mismatches[mismatch_key] = p.value_mismatches.get(mismatch_key, 0) + 1 return patterns def generate_markdown(patterns: Dict[str, ErrorPattern], total_fails: int) -> str: sorted_patterns = sorted(patterns.values(), key=lambda x: x.count, reverse=True) md = "# Error Pattern Analysis Report\n\n" md += f"- Total Failures: {total_fails}\n" md += f"- Unique Patterns: {len(patterns)}\n\n" md += "## Summary Table\n\n" md += "| Pattern | Count | Error Type | Effect | Examples |\n" md += "| :--- | ---: | :--- | :--- | :--- |\n" for p in sorted_patterns: examples = ", ".join(p.examples[:3]) md += f"| **{p.pattern_id}** | {p.count} | {p.error_type} | {p.effect} | {examples} |\n" md += "\n## Detailed Pattern Breakdown\n\n" for p in sorted_patterns: md += f"### {p.pattern_id}\n\n" md += f"- **Count**: {p.count}\n" md += f"- **Error Type**: {p.error_type}\n" md += f"- **Effect**: {p.effect}\n\n" md += "#### Value Mismatches\n" sorted_mismatches = sorted(p.value_mismatches.items(), key=lambda x: x[1], reverse=True) for val, count in sorted_mismatches: md += f"- `{val}`: {count} occurrences\n" md += "\n#### Examples\n" for ex in p.examples: md += f"- {ex}\n" md += "\n---\n" return md def main(): report_path = "reports/COMPREHENSIVE_SEMANTIC_AUDIT.md" output_path = "reports/ERROR_PATTERN_ANALYSIS.md" json_path = "reports/ERROR_PATTERN_ANALYSIS.json" print(f"Reading audit report from {report_path}...") failures = parse_audit_report(report_path) print(f"Found {len(failures)} failures.") patterns = analyze_failures(failures) print(f"Identified {len(patterns)} unique error patterns.") md_content = generate_markdown(patterns, len(failures)) with open(output_path, "w", encoding="utf-8") as f: f.write(md_content) print(f"Markdown report written to {output_path}") # Save JSON for programmatic access json_data = {pid: asdict(p) for pid, p in patterns.items()} with open(json_path, "w", encoding="utf-8") as f: json.dump(json_data, f, indent=2) print(f"JSON data written to {json_path}") if __name__ == "__main__": main()