Buckets:
| #!/usr/bin/env python3 | |
| """Preview code-output SFT examples before training.""" | |
| from __future__ import annotations | |
| import json | |
| import re | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| from dataset_quality_utils import ( | |
| has_code, | |
| has_doctype_or_html, | |
| has_external_assets, | |
| has_inline_style_script, | |
| ) | |
| ROOT = Path(__file__).resolve().parent.parent | |
| AUDIT_DIR = ROOT / "data" / "audit" | |
| REPORT_PATH = AUDIT_DIR / "code_output_preview.md" | |
| KEYWORDS = ( | |
| "index.html", "complete file", "single file", "code", "navbar", "hamburger", | |
| ".nav-links", "feature cards", "canvas", "game", "animation", | |
| ) | |
| def pick_sft_path() -> Path: | |
| for name in ("mythos_sft_messages_clean.jsonl", "mythos_sft_messages.jsonl"): | |
| p = ROOT / "data" / "train" / name | |
| if p.exists(): | |
| return p | |
| return ROOT / "data" / "train" / "mythos_sft_messages.jsonl" | |
| def passes_checks(user: str, assistant: str) -> tuple[bool, list[str]]: | |
| fails = [] | |
| a = assistant | |
| u = user.lower() | |
| if "index.html" in u or "single file" in u or "inline" in u: | |
| if has_external_assets(a): | |
| fails.append("external_css_js") | |
| if "inline" in u or "inside the file" in u or "single file" in u: | |
| if not has_inline_style_script(a): | |
| fails.append("missing_inline_style_script") | |
| if not has_code(a) and any(k in u for k in ("code", "index.html", "complete file")): | |
| fails.append("no_code") | |
| if ("nav-links" in u or "hamburger" in u) and ("broken" in a.lower() or "wrong target" in a.lower()): | |
| if re.search(r"nav-links[^;\n]*classList\.toggle\(['\"]open", a, re.I): | |
| pass # before/after teaching row | |
| elif not re.search(r'classList\.toggle\(["\']open["\']', a, re.I): | |
| fails.append("missing_nav_toggle") | |
| elif ("nav-links" in u or "toggle" in u) and not re.search(r'classList\.toggle\(["\']open["\']', a, re.I): | |
| if "broken" not in a.lower(): | |
| fails.append("missing_nav_toggle") | |
| if a.startswith("Diagnosis:") and ("complete file" in u or "index.html" in u): | |
| fails.append("starts_with_diagnosis") | |
| return len(fails) == 0, fails | |
| def main() -> int: | |
| AUDIT_DIR.mkdir(parents=True, exist_ok=True) | |
| sft_path = pick_sft_path() | |
| if not sft_path.exists(): | |
| print(f"Missing {sft_path}", file=sys.stderr) | |
| return 1 | |
| matches = [] | |
| with sft_path.open("r", encoding="utf-8") as f: | |
| for line in f: | |
| if not line.strip(): | |
| continue | |
| row = json.loads(line) | |
| user = row["messages"][1]["content"] | |
| if any(k in user.lower() for k in KEYWORDS): | |
| assistant = row["messages"][2]["content"] | |
| ok, fails = passes_checks(user, assistant) | |
| matches.append((user, assistant, ok, fails)) | |
| lines = [ | |
| "# Code Output Preview", | |
| "", | |
| f"Source: `{sft_path}`", | |
| f"Matched examples: {len(matches)} (showing up to 30)", | |
| "", | |
| ] | |
| pass_count = 0 | |
| toggle_pat = re.compile(r"""classList\.toggle\(['"]open['"]""", re.I) | |
| for i, (user, assistant, ok, fails) in enumerate(matches[:30], 1): | |
| if ok: | |
| pass_count += 1 | |
| has_toggle = bool(toggle_pat.search(assistant)) | |
| lines.extend([ | |
| f"## Example {i} — {'PASS' if ok else 'FAIL'}", | |
| "", | |
| f"**User prompt:** {user[:400]}", | |
| "", | |
| f"- `<!DOCTYPE html>`: {has_doctype_or_html(assistant)}", | |
| f"- `<style>`: {'<style' in assistant.lower()}", | |
| f"- `<script>`: {'<script' in assistant.lower()}", | |
| f"- classList.toggle open: {has_toggle}", | |
| f"- external styles.css/app.js: {has_external_assets(assistant)}", | |
| f"- fails: {', '.join(fails) if fails else 'none'}", | |
| "", | |
| "**Assistant (first 1600 chars):**", | |
| "", | |
| "```", | |
| assistant[:1600], | |
| "```", | |
| "", | |
| ]) | |
| lines.extend([ | |
| "## Summary", | |
| "", | |
| f"- Previewed: {min(len(matches), 30)}", | |
| f"- Pass: {pass_count}", | |
| f"- Fail: {min(len(matches), 30) - pass_count}", | |
| ]) | |
| REPORT_PATH.write_text("\n".join(lines), encoding="utf-8") | |
| print(f"Wrote {REPORT_PATH} ({pass_count}/{min(len(matches), 30)} pass)") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |
Xet Storage Details
- Size:
- 4.47 kB
- Xet hash:
- 4e2afd212c4464bfa17b354f167a91aacfd21d08e92a474121cfd9b4b304154a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.