Spaces:
Sleeping
Sleeping
File size: 6,093 Bytes
9bd4ce5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 | import json
import os
import re
from dataclasses import asdict, dataclass
from typing import Dict, List
@dataclass
class FailureRecord:
card_no: str
ability_idx: str
segment: str
error_type: str
effect_text: str
expected: str
actual: str
raw_message: str
@dataclass
class ErrorPattern:
pattern_id: str
error_type: str
effect: str
count: int
examples: List[str]
value_mismatches: Dict[str, int]
def parse_audit_report(filepath: str) -> List[FailureRecord]:
failures = []
if not os.path.exists(filepath):
print(f"File not found: {filepath}")
return []
with open(filepath, "r", encoding="utf-8") as f:
lines = f.readlines()
# Regex for ability-level reporting rows
# | Card No | Ability | Status | Details |
# | PL!-PR-001-PR | Ab0 | ❌ FAIL | Stuck at segment 0: Mismatch OPPONENT_HAND_DELTA for 'EFFECT: ACTIVATE_MEMBER(1)': Exp 1, Got 0 |
row_re = re.compile(r"\| (.*?) \| (.*?) \| ❌ FAIL \| (.*) \|")
# Details parser: Stuck at segment (\d+): Mismatch (.*?) for '(.*?)': Exp (.*?), Got (.*)
details_re = re.compile(r"Stuck at segment (\d+): Mismatch (.*?) for '(.*?)': Exp (.*?), Got (.*)")
for line in lines:
match = row_re.search(line)
if match:
card_no = match.group(1).strip()
ability_idx = match.group(2).strip()
details = match.group(3).strip()
detail_match = details_re.search(details)
if detail_match:
segment = detail_match.group(1)
error_type = detail_match.group(2)
effect_text = detail_match.group(3)
expected = detail_match.group(4)
actual = detail_match.group(5)
failures.append(
FailureRecord(
card_no=card_no,
ability_idx=ability_idx,
segment=segment,
error_type=error_type,
effect_text=effect_text,
expected=expected,
actual=actual,
raw_message=details,
)
)
else:
# Handle other failure formats (e.g., panics or different mismatch phrasing)
failures.append(
FailureRecord(
card_no=card_no,
ability_idx=ability_idx,
segment="?",
error_type="OTHER",
effect_text="?",
expected="?",
actual="?",
raw_message=details,
)
)
return failures
def get_effect_category(effect_text: str) -> str:
# Normalize effect text for grouping
# e.g., "EFFECT: ACTIVATE_MEMBER(1)" -> "ACTIVATE_MEMBER"
if "EFFECT:" in effect_text:
match = re.search(r"EFFECT: ([A-Z_]+)", effect_text)
if match:
return match.group(1)
return effect_text
def analyze_failures(failures: List[FailureRecord]) -> Dict[str, ErrorPattern]:
patterns = {}
for f in failures:
effect = get_effect_category(f.effect_text)
pattern_id = f"{f.error_type}_{effect}"
if pattern_id not in patterns:
patterns[pattern_id] = ErrorPattern(
pattern_id=pattern_id, error_type=f.error_type, effect=effect, count=0, examples=[], value_mismatches={}
)
p = patterns[pattern_id]
p.count += 1
if len(p.examples) < 5:
p.examples.append(f"{f.card_no} ({f.ability_idx})")
mismatch_key = f"Exp {f.expected}, Got {f.actual}"
p.value_mismatches[mismatch_key] = p.value_mismatches.get(mismatch_key, 0) + 1
return patterns
def generate_markdown(patterns: Dict[str, ErrorPattern], total_fails: int) -> str:
sorted_patterns = sorted(patterns.values(), key=lambda x: x.count, reverse=True)
md = "# Error Pattern Analysis Report\n\n"
md += f"- Total Failures: {total_fails}\n"
md += f"- Unique Patterns: {len(patterns)}\n\n"
md += "## Summary Table\n\n"
md += "| Pattern | Count | Error Type | Effect | Examples |\n"
md += "| :--- | ---: | :--- | :--- | :--- |\n"
for p in sorted_patterns:
examples = ", ".join(p.examples[:3])
md += f"| **{p.pattern_id}** | {p.count} | {p.error_type} | {p.effect} | {examples} |\n"
md += "\n## Detailed Pattern Breakdown\n\n"
for p in sorted_patterns:
md += f"### {p.pattern_id}\n\n"
md += f"- **Count**: {p.count}\n"
md += f"- **Error Type**: {p.error_type}\n"
md += f"- **Effect**: {p.effect}\n\n"
md += "#### Value Mismatches\n"
sorted_mismatches = sorted(p.value_mismatches.items(), key=lambda x: x[1], reverse=True)
for val, count in sorted_mismatches:
md += f"- `{val}`: {count} occurrences\n"
md += "\n#### Examples\n"
for ex in p.examples:
md += f"- {ex}\n"
md += "\n---\n"
return md
def main():
report_path = "reports/COMPREHENSIVE_SEMANTIC_AUDIT.md"
output_path = "reports/ERROR_PATTERN_ANALYSIS.md"
json_path = "reports/ERROR_PATTERN_ANALYSIS.json"
print(f"Reading audit report from {report_path}...")
failures = parse_audit_report(report_path)
print(f"Found {len(failures)} failures.")
patterns = analyze_failures(failures)
print(f"Identified {len(patterns)} unique error patterns.")
md_content = generate_markdown(patterns, len(failures))
with open(output_path, "w", encoding="utf-8") as f:
f.write(md_content)
print(f"Markdown report written to {output_path}")
# Save JSON for programmatic access
json_data = {pid: asdict(p) for pid, p in patterns.items()}
with open(json_path, "w", encoding="utf-8") as f:
json.dump(json_data, f, indent=2)
print(f"JSON data written to {json_path}")
if __name__ == "__main__":
main()
|