"""DummyScorer — a deterministic PLACEHOLDER scorer (Task 3). NOT machine learning. It derives the 5 axis scores from cheap surface heuristics over the user's turns so the result card reflects the real upload instead of hardcoded numbers. Every heuristic is logged in DECISIONS_LOG.md and is intentionally simple/transparent: Focus — fewer, longer conversations read as more focused (turns-per-conversation). Technique — rate of user turns using prompting moves ("act as", "step by step", examples…). Critical — rate of skeptical / verifying language ("why", "source", "cite", "wrong"…). Interaction — average user turns per conversation (back-and-forth depth). Input Quality— average user message length + presence of code fences / structure. Same input -> same output. The real ML scorer implements the identical `Scorer.score` seam. """ from __future__ import annotations import re from ..data import AXES from ..parsing import ParsedExport from .interface import AxisScore, ScoreResult _TECHNIQUE = re.compile( r"\b(act as|you are a|step by step|for example|e\.g\.|think|reason|options?|" r"role:|constraints?:|context:|few[- ]shot)\b", re.I) _CRITICAL = { "skepticism": re.compile(r"\b(actually|really|sure\?|doubt|skeptic|cherry|wrong|incorrect)\b", re.I), "source_req": re.compile(r"\b(source|cite|citation|reference|evidence|prove|link)\b", re.I), "rebuttal": re.compile(r"\b(but |however|disagree|that's not|not quite|i think you)\b", re.I), "verify": re.compile(r"\b(verify|double[- ]check|confirm|are you sure|is that right)\b", re.I), "re_ask": re.compile(r"\b(again|rephrase|try again|instead|redo|differently)\b", re.I), } _CODE_FENCE = re.compile(r"```") _TIPS = { "Focus": "Open a fresh chat per task to keep each thread on one goal.", "Technique": "Lean on roles, few-shot examples, and explicit step-by-step asks.", "Critical": "Keep pushing back; verify key claims against a second source.", "Interaction": "Build on the model's reasoning with follow-ups instead of restarting.", "Input Quality": "Lead with role + constraints + a concrete example for richer prompts.", } def _clamp(x: float) -> float: return max(0.0, min(10.0, x)) def _rate(matches: int, total: int) -> float: return matches / total if total else 0.0 class DummyScorer: """Heuristic placeholder. Implements the `Scorer` protocol.""" def score(self, parsed: ParsedExport, progress=None) -> ScoreResult: convs = [c for c in parsed.conversations if any(t.role == "user" for t in c.turns)] user_turns = parsed.user_turns n_users = len(user_turns) or 1 n_convs = len(convs) or 1 texts = [t.text for t in user_turns] turns_per_conv = n_users / n_convs avg_len = sum(len(t) for t in texts) / n_users technique_hits = sum(1 for t in texts if _TECHNIQUE.search(t)) code_turns = sum(1 for t in texts if _CODE_FENCE.search(t)) crit_counts = {k: sum(1 for t in texts if rx.search(t)) for k, rx in _CRITICAL.items()} crit_total = sum(crit_counts.values()) # --- map heuristics onto 0-10 (deterministic) --- focus = _clamp(3.0 + 1.6 * (turns_per_conv - 1)) # more turns/conv -> focused technique = _clamp(10 * _rate(technique_hits, n_users) * 2.5) # ~40% usage -> 10 critical = _clamp(10 * _rate(crit_total, n_users) * 1.2) # skeptical language rate interaction = _clamp(2.0 + 2.0 * (turns_per_conv - 1)) # back-and-forth depth input_quality = _clamp(avg_len / 80.0 + 2.0 * _rate(code_turns, n_users) + 2.0) scores = { "Focus": focus, "Technique": technique, "Critical": critical, "Interaction": interaction, "Input Quality": input_quality, } # confidence by sample volume; Focus pinned low (cluster signal is weakest — per spec). conf = "high" if n_users >= 40 else "medium" if n_users >= 10 else "low" confidence = {a: conf for a in AXES} confidence["Focus"] = "low" axes = [ AxisScore(a, scores[a], confidence[a], _evidence(a, texts, crit_counts), _TIPS[a]) for a in AXES ] improvement = _improvement(scores) return ScoreResult(axes=axes, critical_counts=crit_counts, improvement=improvement) def _evidence(axis: str, texts: list[str], crit_counts: dict) -> list[str]: """Pick 1-2 real user turns illustrating the axis; fall back to the first turns.""" def pick(pred): return [t for t in texts if pred(t)][:2] if axis == "Technique": got = pick(lambda t: _TECHNIQUE.search(t)) elif axis == "Critical": rx = re.compile("|".join(p.pattern for p in _CRITICAL.values()), re.I) got = pick(lambda t: rx.search(t)) elif axis == "Input Quality": got = sorted(texts, key=len, reverse=True)[:2] elif axis == "Interaction": got = pick(lambda t: t.strip().endswith("?")) else: # Focus got = texts[:2] got = got or texts[:1] or ["(no user turns found)"] return [_quote(t) for t in got] def _quote(t: str, limit: int = 160) -> str: t = " ".join(t.split()) if len(t) > limit: t = t[: limit - 1].rstrip() + "…" return f'"{t}"' def _improvement(scores: dict) -> str: weakest = min(scores, key=scores.get) return f"Biggest lever: {weakest.lower()} is your lowest axis — {_TIPS[weakest].lower()}"