Buckets:
| #!/usr/bin/env python3 | |
| """Shared dataset quality scoring, detection, and validation helpers.""" | |
| from __future__ import annotations | |
| import re | |
| from difflib import SequenceMatcher | |
| REQUIRED_CANONICAL_FIELDS = ( | |
| "id", "task_type", "difficulty", "language", "framework", | |
| "user_prompt", "repo_context", "failure_log", "investigation_steps", | |
| "plan", "solution", "verification", "lesson", "quality_score", | |
| ) | |
| BANNED_PHRASES = ( | |
| "plus 1 additional targeted edits", | |
| "plus additional targeted edits", | |
| "make targeted edits", | |
| "read the html structure", | |
| "fix the issue", | |
| "verify it works", | |
| "additional targeted edits", | |
| "apply targeted edits", | |
| "read the files", | |
| "add code", | |
| ) | |
| GENERIC_INVESTIGATION = ( | |
| "inspect the relevant project files", | |
| "read the files", | |
| "review the user request", | |
| ) | |
| GENERIC_LESSON = ( | |
| "prefer small verified edits", | |
| "inspect all relevant files before making changes", | |
| ) | |
| CODE_DETECT_MARKERS = ( | |
| "<!doctype html>", "<html", "<style", "<script", "```html", "```js", | |
| "```javascript", "```css", "function ", "const ", "let ", "class=", | |
| "addeventlistener", "classlist.toggle", "requestanimationframe", | |
| "import ", "export ", "npm ", "pytest", "describe(", "test(", "expect(", | |
| ) | |
| EXTERNAL_ASSET_PATTERNS = ( | |
| re.compile(r'<link[^>]+href=["\'][^"\']*\.css', re.I), | |
| re.compile(r'<script[^>]+src=["\'][^"\']*\.js', re.I), | |
| re.compile(r'href=["\']styles\.css["\']', re.I), | |
| re.compile(r'src=["\']app\.js["\']', re.I), | |
| re.compile(r'src=["\']script\.js["\']', re.I), | |
| re.compile(r'src=["\']main\.js["\']', re.I), | |
| ) | |
| SCORE_EXCELLENT = 90 | |
| SCORE_KEEP = 80 | |
| SCORE_REPAIR = 65 | |
| SCORE_WEAK = 40 | |
| def normalize_prompt(text: str) -> str: | |
| return re.sub(r"\s+", " ", str(text or "").strip().lower()) | |
| def has_code(text: str) -> bool: | |
| t = str(text or "") | |
| lower = t.lower() | |
| if "```" in t: | |
| return True | |
| if "<!doctype" in lower or "<html" in lower: | |
| return True | |
| if re.search(r"\b(function|const|let|class|import|export)\s+[\w{$(\[]", t): | |
| return True | |
| if any(call in lower for call in ("addeventlistener(", "getelementbyid(", "queryselector(", "classlist.")): | |
| return True | |
| if "<style" in lower or "<script" in lower: | |
| return True | |
| return False | |
| def asks_for_code(prompt: str) -> bool: | |
| p = str(prompt or "").lower() | |
| keys = ( | |
| "index.html", "complete file", "full file", "full code", "single file", | |
| "one file", "output the full", "return all file", "provide code", | |
| "write code", "give me code", "html/css/js", "complete code", | |
| "output only the complete code", "inline css", "inline js", | |
| ) | |
| if any(k in p for k in keys): | |
| return True | |
| return bool(re.search( | |
| r"\b(create|build|generate|write|provide|return|output)\b.*\b(" | |
| r"file|index\.html|html|code|script|component|app)\b", | |
| p, | |
| )) | |
| def asks_single_file_html(prompt: str) -> bool: | |
| p = str(prompt or "").lower() | |
| if "index.html" not in p and "single-file" not in p and "single file" not in p: | |
| if not ("one html file" in p or "one complete html" in p): | |
| return False | |
| inline_signals = ( | |
| "single file", "single index.html", "one index.html", "one file", | |
| "all inside one file", "inline css", "inline js", "css and js must be inline", | |
| "css and javascript inside", "inside the file", "in one file", | |
| "single-file", "one html file", | |
| ) | |
| return any(s in p for s in inline_signals) or ( | |
| "index.html" in p and any(s in p for s in ("single", "one ", "inline", "inside the file")) | |
| ) | |
| def asks_three_feature_cards(prompt: str) -> bool: | |
| p = str(prompt or "").lower() | |
| return "three feature card" in p or "exactly three feature" in p or "3 feature card" in p | |
| def asks_hero(prompt: str) -> bool: | |
| return "hero" in str(prompt or "").lower() | |
| def asks_logo(prompt: str) -> bool: | |
| return "logo" in str(prompt or "").lower() | |
| def asks_hamburger(prompt: str) -> bool: | |
| p = str(prompt or "").lower() | |
| return "hamburger" in p or ".hamburger" in p | |
| def asks_nav_links_open(prompt: str) -> bool: | |
| p = str(prompt or "").lower() | |
| return ".nav-links.open" in p or "nav-links.open" in p or "toggle .open on .nav-links" in p | |
| def prompt_overlap_ratio(a: str, b: str) -> float: | |
| a, b = normalize_prompt(a), normalize_prompt(b) | |
| if not a or not b: | |
| return 0.0 | |
| return SequenceMatcher(None, a, b).ratio() | |
| def text_repeats_prompt(source: str, prompt: str, threshold: float = 0.55) -> bool: | |
| if not source or not prompt: | |
| return False | |
| if normalize_prompt(prompt) in normalize_prompt(source): | |
| return True | |
| return prompt_overlap_ratio(source, prompt) >= threshold | |
| def has_external_assets(solution: str) -> bool: | |
| for pat in EXTERNAL_ASSET_PATTERNS: | |
| if pat.search(solution): | |
| return True | |
| return False | |
| def has_inline_style_script(solution: str) -> bool: | |
| lower = solution.lower() | |
| return "<style" in lower and "<script" in lower | |
| def has_doctype_or_html(solution: str) -> bool: | |
| lower = solution.lower() | |
| return "<!doctype" in lower or "<html" in lower | |
| def count_feature_cards(solution: str) -> int: | |
| lower = solution.lower() | |
| patterns = [ | |
| r'class=["\'][^"\']*card[^"\']*["\']', | |
| r'class=["\'][^"\']*feature[^"\']*["\']', | |
| r"<article", | |
| r"<div[^>]+feature", | |
| ] | |
| count = 0 | |
| for pat in patterns: | |
| count += len(re.findall(pat, lower, re.I)) | |
| return count | |
| def has_hero(solution: str) -> bool: | |
| lower = solution.lower() | |
| return "hero" in lower or 'id="hero"' in lower or "class=\"hero" in lower | |
| def has_logo(solution: str) -> bool: | |
| lower = solution.lower() | |
| return "logo" in lower | |
| def nav_selector_issues(prompt: str, solution: str) -> list[str]: | |
| issues: list[str] = [] | |
| sol = str(solution or "") | |
| lower = sol.lower() | |
| p = str(prompt or "").lower() | |
| if asks_nav_links_open(p) or ("nav-links" in p and "hamburger" in p): | |
| if re.search(r"\.hamburger[^;\n]*classlist\.toggle\(['\"]open", sol, re.I): | |
| issues.append("toggles_open_on_hamburger") | |
| if re.search(r"\.nav-item[^;\n]*classlist\.toggle\(['\"]open", sol, re.I): | |
| issues.append("toggles_open_on_nav_item") | |
| if ".nav-item-open" in lower or "nav-item-open" in lower: | |
| issues.append("invents_nav_item_open") | |
| if ".nav-links.open" in p or "nav-links.open" in p: | |
| if not re.search(r'classlist\.toggle\(["\']open["\']', sol, re.I): | |
| issues.append("missing_classlist_toggle_open") | |
| if "nav-links" not in lower and "navlinks" not in lower.replace("-", ""): | |
| issues.append("missing_nav_links_reference") | |
| return issues | |
| def has_hamburger_js(solution: str) -> bool: | |
| lower = solution.lower() | |
| return "hamburger" in lower and ("addeventlistener" in lower or "onclick" in lower) | |
| def contains_banned_phrase(text: str) -> list[str]: | |
| lower = str(text or "").lower() | |
| return [p for p in BANNED_PHRASES if p in lower] | |
| def verification_is_vague(verification: str) -> bool: | |
| v = str(verification or "").strip().lower() | |
| if not v: | |
| return True | |
| if "short test checklist" in v and v.count("-") < 1 and v.count("\n") < 2: | |
| return True | |
| vague = ("verify it works", "confirm the task requirements are met", "fix the issue") | |
| concrete = ("click", "open", "run ", "npm", "assert", "devtools", "check that", "toggle") | |
| if any(x in v for x in vague) and not any(c in v for c in concrete): | |
| return True | |
| return False | |
| def is_numbered_bullets_only(solution: str) -> bool: | |
| s = str(solution or "").strip() | |
| if not s: | |
| return True | |
| if has_code(s): | |
| return False | |
| if re.match(r"^1\)", s) or s.count("1)") >= 1: | |
| return True | |
| if s.lower().startswith("- ") and not has_code(s): | |
| return True | |
| return False | |
| def hallucination_theme(solution: str, prompt: str) -> bool: | |
| sol = str(solution or "").lower() | |
| prompt_l = str(prompt or "").lower() | |
| if "404 not found" in sol and "404" not in prompt_l: | |
| return True | |
| return False | |
| def grounded_tokens(row: dict) -> set[str]: | |
| parts = [ | |
| row.get("user_prompt", ""), | |
| row.get("repo_context", ""), | |
| row.get("failure_log", ""), | |
| row.get("plan", ""), | |
| row.get("solution", ""), | |
| ] | |
| text = " ".join(str(p) for p in parts).lower() | |
| tokens = set(re.findall(r"[a-z][a-z0-9_-]{2,}", text)) | |
| return tokens | |
| def ungrounded_class_names(row: dict) -> list[str]: | |
| solution = str(row.get("solution", "")) | |
| prompt = str(row.get("user_prompt", "")).lower() | |
| grounded = grounded_tokens(row) | |
| classes = re.findall(r'\.([a-z][a-z0-9_-]*)', solution.lower()) | |
| bad = [] | |
| for cls in classes: | |
| if cls in {"open", "active", "hidden", "show", "container", "wrapper"}: | |
| continue | |
| if cls == "nav-item-open" and "nav-item-open" not in prompt: | |
| bad.append(f".{cls}") | |
| elif cls not in grounded and cls.replace("-", "") not in "".join(grounded): | |
| if cls.endswith("-open") and cls.replace("-open", "") not in prompt: | |
| bad.append(f".{cls}") | |
| return bad | |
| def score_row(row: dict) -> tuple[int, list[str]]: | |
| """Return (score 0-100, issues list).""" | |
| issues: list[str] = [] | |
| score = 100 | |
| for field in REQUIRED_CANONICAL_FIELDS: | |
| if field not in row: | |
| issues.append(f"missing_field:{field}") | |
| score -= 15 | |
| user = str(row.get("user_prompt", "")).strip() | |
| solution = str(row.get("solution", "")).strip() | |
| failure_log = str(row.get("failure_log", "")).strip() | |
| verification = str(row.get("verification", "")).strip() | |
| lesson = str(row.get("lesson", "")).strip() | |
| steps = row.get("investigation_steps", []) | |
| has_good_code = has_code(solution) and len(solution) > 120 and ( | |
| "```" in solution or "<!doctype" in solution.lower() or "<html" in solution.lower() | |
| ) | |
| if len(solution) < 20: | |
| issues.append("tiny_solution") | |
| score -= 25 | |
| if is_numbered_bullets_only(solution) and not has_good_code: | |
| issues.append("generic_bullets_only") | |
| score -= 30 | |
| banned_hits = contains_banned_phrase(" ".join([solution, verification, lesson, failure_log])) | |
| for phrase in banned_hits: | |
| issues.append(f"banned_phrase:{phrase}") | |
| score -= 8 if not has_good_code else 3 | |
| if asks_for_code(user) and not has_code(solution): | |
| issues.append("code_request_no_code") | |
| score -= 35 | |
| if text_repeats_prompt(solution, user) and not has_good_code: | |
| issues.append("solution_repeats_prompt") | |
| score -= 20 | |
| if text_repeats_prompt(failure_log, user) and not has_good_code: | |
| issues.append("failure_log_repeats_prompt") | |
| score -= 15 | |
| if failure_log.startswith("Initial problem:") and user in failure_log and not has_good_code: | |
| issues.append("failure_log_copies_instruction") | |
| score -= 12 | |
| if verification_is_vague(verification) and not has_good_code: | |
| issues.append("vague_verification") | |
| score -= 10 | |
| if any(g in lesson.lower() for g in GENERIC_LESSON) and len(lesson) < 80 and not has_good_code: | |
| issues.append("generic_lesson") | |
| score -= 5 | |
| if isinstance(steps, list) and not has_good_code: | |
| generic_steps = sum(1 for s in steps if any(g in str(s).lower() for g in GENERIC_INVESTIGATION)) | |
| if generic_steps >= 2: | |
| issues.append("generic_investigation_steps") | |
| score -= 8 | |
| if asks_single_file_html(user): | |
| if has_external_assets(solution): | |
| issues.append("single_file_external_assets") | |
| score -= 40 | |
| if not has_inline_style_script(solution): | |
| issues.append("single_file_missing_inline_style_script") | |
| score -= 25 | |
| if not has_doctype_or_html(solution): | |
| issues.append("single_file_missing_html_doctype") | |
| score -= 25 | |
| if asks_three_feature_cards(user) and count_feature_cards(solution) < 3: | |
| issues.append("missing_three_feature_cards") | |
| score -= 30 | |
| if asks_hero(user) and not has_hero(solution): | |
| issues.append("missing_hero_section") | |
| score -= 15 | |
| if asks_logo(user) and not has_logo(solution): | |
| issues.append("missing_logo") | |
| score -= 10 | |
| if asks_hamburger(user) and not has_hamburger_js(solution): | |
| issues.append("missing_hamburger_js") | |
| score -= 20 | |
| for nav_issue in nav_selector_issues(user, solution): | |
| issues.append(nav_issue) | |
| score -= 25 | |
| if hallucination_theme(solution, user): | |
| issues.append("hallucinated_theme") | |
| score -= 20 | |
| ungrounded = ungrounded_class_names(row) | |
| if ungrounded and not has_good_code: | |
| issues.append(f"ungrounded_classes:{','.join(ungrounded[:3])}") | |
| score -= 10 | |
| if asks_for_code(user) and uses_diagnosis_format_for_row(row) and not has_good_code: | |
| issues.append("code_request_would_use_diagnosis_format") | |
| score -= 15 | |
| if has_good_code: | |
| score = max(score, 82) | |
| return max(0, min(100, score)), issues | |
| def score_band(score: int) -> str: | |
| if score >= SCORE_EXCELLENT: | |
| return "excellent" | |
| if score >= SCORE_KEEP: | |
| return "keep" | |
| if score >= SCORE_REPAIR: | |
| return "repair" | |
| if score >= SCORE_WEAK: | |
| return "weak" | |
| return "reject" | |
| def uses_diagnosis_format_for_row(row: dict) -> bool: | |
| task_type = str(row.get("task_type", "")) | |
| if task_type in {"bug_fix", "terminal_debug", "ui_repair", "migration", "refactor"}: | |
| return True | |
| if asks_for_code(row.get("user_prompt", "")): | |
| return False | |
| return task_type not in {"feature_build", "website_vibe", "test_writing"} | |
| def repair_recommendation(score: int, issues: list[str]) -> str: | |
| if score >= SCORE_KEEP: | |
| return "keep_as_is" | |
| if "failure_log_repeats_prompt" in issues or "failure_log_copies_instruction" in issues: | |
| return "trim_failure_log_prompt_echo" | |
| if "vague_verification" in issues: | |
| return "replace_with_concrete_verification" | |
| if "generic_lesson" in issues: | |
| return "sharpen_lesson" | |
| if "banned_phrase:" in " ".join(issues): | |
| return "remove_banned_phrases" | |
| if "code_request_no_code" in issues: | |
| return "reject_needs_code_output_rewrite" | |
| if "single_file_external_assets" in issues: | |
| return "reject_single_file_html_violation" | |
| if any(i.startswith("toggles_open") or i.startswith("invents_") for i in issues): | |
| return "reject_selector_mismatch" | |
| if score >= SCORE_REPAIR: | |
| return "mechanical_repair_candidate" | |
| return "reject_unsafe_to_auto_repair" | |
Xet Storage Details
- Size:
- 14.8 kB
- Xet hash:
- cc3a4b14d9b3e163986b63ebba16ff36eabac1e304abc84ed141b604129d822b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.