Update script.py

script.py

@@ -5,6 +5,7 @@ SCRIPT_DIR = Path(__file__).resolve().parent if "__file__" in globals() else Pat
 WHEELHOUSE = SCRIPT_DIR / "wheelhouse"
 if not WHEELHOUSE.is_dir(): WHEELHOUSE = Path("wheelhouse")
 RUNTIME_DIR = Path("/tmp/qwen3deps")
+
 def emergency(reason):
     try:
         import pandas as pd, json as j
@@ -15,6 +16,7 @@ def emergency(reason):
     except Exception:
         try: open("submission.csv","w").write("id,pred,explanation\n")
         except Exception: pass
+
 try:
     wheels = [str(WHEELHOUSE / w) for w in os.listdir(WHEELHOUSE) if w.endswith(".whl")]
     if not wheels: raise FileNotFoundError(f"no wheels in {WHEELHOUSE}")
@@ -26,21 +28,28 @@ try:
     except Exception: pass
 except Exception as e:
     emergency(f"wheel install failed: {e}"); raise
-import re, json, time, ast as pyast
+
+import re, json, time
 import pandas as pd, torch
-from difflib import SequenceMatcher
-from collections import defaultdict
 from transformers import AutoTokenizer, AutoModelForCausalLM
-MODEL_ID="."; TIME_LIMIT=30*60; start=time.time()
-# Extra deterministic passes only run while this much time remains, measured
-# FRESHLY at each decision point (never a stale snapshot from earlier in the row).
-REPAIR_RESERVE=120
+
+MODEL_ID = "."
+TIME_LIMIT = 30 * 60
+SAFETY_S = 90
+start = time.time()
+
+LONG_THINK_TASKS   = {"translation", "match_letters", "fill_blanks"}
+LONG_TOTAL_TOKENS  = 1536
+THINK_CLOSE_TOKEN_ID = 151668   # </think> in Qwen3 tokenizer (documented)
+
 def write_csv(rows):
     import csv
     with open("submission.csv.tmp","w",newline="",encoding="utf-8") as f:
-        w=csv.DictWriter(f,fieldnames=["id","pred","explanation"]); w.writeheader()
+        w = csv.DictWriter(f, fieldnames=["id","pred","explanation"])
+        w.writeheader()
         for r in rows: w.writerow(r)
     os.replace("submission.csv.tmp","submission.csv")
+
 try:
     df = pd.read_csv("/tmp/data/test.csv", dtype=str).fillna("")
     write_csv([{"id":i,"pred":json.dumps([""]),"explanation":"placeholder"} for i in df["id"]])
@@ -50,274 +59,278 @@ try:
     print("loaded, quantized:", getattr(model.config,"quantization_config",None) is not None, flush=True)
 except Exception as e:
     emergency(f"load failed: {e}"); raise
-SYS=("You solve International Linguistics Olympiad problems about a language you have never seen. "
-     "Everything you need is in the examples. Answer every numbered item, in order. "
-     "Put each answer on its own line, with no numbering and no extra text.")
+
+SYS = ("You solve International Linguistics Olympiad problems about a language you have never seen. "
+       "Everything you need is in the examples. Answer every numbered item, in order. "
+       "Put each answer on its own line, with no numbering and no extra text.")
+
 def n_expected(query):
-    items=re.findall(r"(?m)^\s*(\d+)\s*[.\)]", query)
+    items = re.findall(r"(?m)^\s*(\d+)\s*[.\)]", query)
     if items: return len(items)
-    rng=re.search(r"\(\s*(\d+)\s*[-–—]\s*(\d+)\s*\)", query)
+    rng = re.search(r"\(\s*(\d+)\s*[-–—]\s*(\d+)\s*\)", query)
     if rng:
-        lo,hi=int(rng.group(1)),int(rng.group(2))
-        if 0<hi-lo<100: return hi-lo+1
+        lo, hi = int(rng.group(1)), int(rng.group(2))
+        if 0 < hi - lo < 100: return hi - lo + 1
     return None
-def norm_generic(s):
-    s = unicodedata.normalize("NFC", s).strip()
-    s = re.sub(r"^\s*\(?\d+\)?\s*[.):\-]\s+", "", s)
-    s = re.sub(r"^\s*[-*•·]\s+", "", s)
-    s = re.sub(r"(?i)^\s*(answer|translation|output)\s*\d*\s*[:.\-]\s+", "", s)
-    s = s.strip("* ")
-    s = re.sub(r"\s{2,}", " ", s)
-    return s.strip()
-def norm_number(s):
-    g = norm_generic(s)
-    m = re.search(r"-?\d[\d,\. ]*\d|\d", g)
-    if not m: return g
-    digits = re.sub(r"[^\d-]", "", m.group(0))
-    return digits if digits else g
-def normalize(task_type, s):
-    if task_type == "text_to_num":
-        return norm_number(s)
-    return norm_generic(s)
+
+# =============================================================================
+# PARSER -- designed as a compiler, not a regex patchwork.
+# Replaces the prior line-based parser. Every rule justified by a specific
+# failure mode derived from adversarial analysis of Qwen3 outputs.
+#
+# Failure modes handled that the old parser did NOT handle:
+#
+#  1. THINK BLOCK LEAKAGE (F1, F2): <think>...</think> and orphan </think>
+#     appear as text when thinking mode is active. Old parser: included the
+#     reasoning text as answers. New parser: strips all think content first.
+#
+#  2. PIPE-SEPARATED ANSWERS (A3): model emits "cat | dog | mouse" on one
+#     line (common because IOL contexts use | as field delimiter). Old parser:
+#     treats the whole line as one answer. New parser: tries pipe-split and
+#     selects it when count matches n.
+#
+#  3. COMMA-SEPARATED ANSWERS (A4): model emits "cat, dog, mouse". Old parser:
+#     one answer. New parser: tries comma-split for single-line short outputs.
+#
+#  4. SPACE-SEPARATED LETTERS FOR MATCHING (A6): model emits "A B C D" on one
+#     line. Old parser: one answer. New parser: splits when all tokens are
+#     single capital letters and task is match_letters.
+#
+#  5. PREAMBLE LINES (B5): "Here are the answers:\ncat\ndog". Old parser:
+#     "Here are the answers:" becomes the first answer. New parser: detects
+#     and discards preamble lines before applying count enforcement.
+#
+#  6. EXPLANATION LINES (B3): "cat\ndog\nThis is because...". Old parser with
+#     n=2: takes first 2 including explanation if it appears early. New parser:
+#     detects explanation lines and discards them.
+#
+#  7. TABLE FORMAT FOR MATCHING (B8): "| A | water |\n| B | child |". Old
+#     parser: pipe split disabled, each row becomes one answer. New parser:
+#     extracts the single-letter option from each table row.
+#
+#  8. ZERO-WIDTH SPACES AND NON-BREAKING SPACES (D2, D4): invisible Unicode
+#     characters that cause silent EM failure. Old parser: only NFC
+#     normalization, does not remove Cf category characters. New parser:
+#     explicitly strips zero-width space, ZWJ, ZWNJ, soft hyphen, BOM,
+#     and replaces non-breaking space with regular space.
+#
+#  9. ITEM N: PREFIX NOT STRIPPED (B10): "item 1: cat". Old parser: strips
+#     only answer/translation/output prefixes. New parser: also strips
+#     "item N:" prefix pattern.
+#
+# Failures NOT changed (old behavior preserved for these):
+#  - Numbered list "1. cat" → strips correctly (unchanged)
+#  - Bullet "- cat" → strips correctly (unchanged)
+#  - Answer: prefix → strips correctly (unchanged)
+#  - NFC normalization → preserved
+#  - text_to_num digit extraction → preserved
+# =============================================================================
+
+def _clean_unicode(s: str) -> str:
+    s = s.replace('\u00a0', ' ')   # non-breaking space → space
+    s = s.replace('\u200b', '')    # zero-width space
+    s = s.replace('\u200c', '')    # zero-width non-joiner
+    s = s.replace('\u200d', '')    # zero-width joiner
+    s = s.replace('\ufeff', '')    # BOM
+    s = s.replace('\u00ad', '')    # soft hyphen
+    return unicodedata.normalize("NFC", s)
+
+def _strip_think(s: str) -> str:
+    s = re.sub(r'<think>.*?</think>', '', s, flags=re.DOTALL | re.IGNORECASE)
+    s = re.sub(r'</think>', '', s, flags=re.IGNORECASE)
+    s = re.sub(r'<think>', '', s, flags=re.IGNORECASE)
+    return s
+
+_PREAMBLE = re.compile(
+    r"""(?ix)^(?:here\s+(?:are|is)\b|the\s+following\b|these\s+(?:are|follow)\b|
+    (?:answer|result|translation|output)s?\s*:|note\s*:|explanation\s*:)""")
+_EXPLANATION = re.compile(
+    r"""(?ix)^(?:(?:this|it)\s+is\s+because\b|(?:this|the)\s+(?:pattern|rule|form)\b|
+    (?:note|explanation|reason|because)\s*:|therefore\b|
+    the\s+answer\s+(?:is|follows)\b|(?:we\s+can\s+see|observe\s+that)\b|
+    (?:in\s+this\s+language)\b|(?:looking\s+at)\b|(?:from\s+the\s+examples)\b)""")
+
+def _is_preamble(line: str) -> bool:
+    s = line.strip()
+    return bool(s) and (s.endswith(':') and len(s) > 3 or bool(_PREAMBLE.match(s)))
+
+def _is_explanation(line: str) -> bool:
+    return bool(_EXPLANATION.match(line.strip()))
+
+def _norm_one(s: str, task_type: str) -> str:
+    s = _clean_unicode(s)
+    s = re.sub(r'^\s*\(?\d+\)?\s*[.):\-]\s+', '', s)          # "1. " / "1) " / "(1) "
+    s = re.sub(r'^\s*[-*•·]\s+', '', s)                         # bullets
+    s = re.sub(r'(?i)^\s*(?:the\s+)?(?:final\s+)?'
+               r'(?:answer|translation|output|item)\s*\d*\s*[:.\-]\s*', '', s)
+    s = re.sub(r'(?i)^\s*is\s*:\s*', '', s)
+    s = s.strip('* ')
+    s = re.sub(r'[ \t]+', ' ', s)
+    s = s.strip()
+    s = s.strip('"\'"\u201c\u201d\u2018\u2019')
+    if task_type == 'text_to_num':
+        m = re.search(r'-?\d[\d,\. ]*\d|\d', s)
+        if m:
+            d = re.sub(r'[^\d-]', '', m.group(0))
+            if d: s = d
+    return s
+
+def _table_match_answers(text: str, task_type: str) -> list | None:
+    if '|' not in text: return None
+    rows = [ln.strip() for ln in text.splitlines() if ln.strip() and '|' in ln]
+    if not rows: return None
+    result = []
+    for row in rows:
+        cells = [c.strip() for c in row.split('|') if c.strip()]
+        for cell in cells:
+            if re.match(r'^[A-Z]$', cell):
+                result.append(cell)
+                break
+        else:
+            if cells:
+                c = _norm_one(cells[0], task_type)
+                if c: result.append(c)
+    return result or None
+
+def _segmentations(text: str, task_type: str) -> list:
+    cands = []
+    lines = [_norm_one(ln, task_type) for ln in text.splitlines()
+             if ln.strip() and _norm_one(ln.strip(), task_type)]
+    if lines: cands.append(lines)
+
+    if '|' in text:
+        parts = [_norm_one(p, task_type) for p in re.split(r'\s*\|\s*', text.strip()) if p.strip()]
+        parts = [p for p in parts if p]
+        if parts and parts != lines: cands.append(parts)
+
+    if ',' in text and '\n' not in text.strip():
+        parts = [_norm_one(p, task_type) for p in text.split(',') if p.strip()]
+        parts = [p for p in parts if p and len(p) <= 60]
+        if parts and parts != lines: cands.append(parts)
+
+    if task_type == 'match_letters':
+        tokens = text.strip().split()
+        if tokens and all(re.match(r'^[A-Z]$', t) for t in tokens) and tokens != lines:
+            cands.append(tokens)
+
+    return cands
+
+def parse_answers(raw: str, task_type: str, n: int | None) -> list:
+    if not raw or not raw.strip():
+        return [""] * (n or 1)
+
+    text = _clean_unicode(raw)
+    text = _strip_think(text).strip()
+    if not text:
+        return [""] * (n or 1)
+
+    # Table extraction for match_letters
+    if task_type == 'match_letters' and '|' in text:
+        ta = _table_match_answers(text, task_type)
+        if ta and (n is None or len(ta) == n):
+            return ta
+
+    segs = _segmentations(text, task_type)
+    if not segs:
+        return [""] * (n or 1)
+
+    if n is not None:
+        # Pick first segmentation with exact count match
+        for seg in segs:
+            if len(seg) == n:
+                return seg
+        # Try filtering preamble/explanation lines
+        raw_lines = [ln for ln in text.splitlines() if ln.strip()]
+        filtered = [ln for ln in raw_lines if not _is_preamble(ln) and not _is_explanation(ln)]
+        filtered_norm = [_norm_one(ln, task_type) for ln in filtered if _norm_one(ln, task_type)]
+        if len(filtered_norm) == n:
+            return filtered_norm
+        # Apply count enforcement on best candidate
+        best = filtered_norm if filtered_norm else segs[0]
+        if len(best) > n: return best[:n]
+        if len(best) < n: return best + [best[-1] if best else ""] * (n - len(best))
+        return best
+
+    # n unknown: filter preamble/explanation, return remainder
+    raw_lines = [ln for ln in text.splitlines() if ln.strip()]
+    filtered = [ln for ln in raw_lines if not _is_preamble(ln) and not _is_explanation(ln)]
+    filtered_norm = [_norm_one(ln, task_type) for ln in filtered if _norm_one(ln, task_type)]
+    return filtered_norm if filtered_norm else (segs[0] if segs else [""])
+
 
 # ---------------------------------------------------------------------------
-# Generation. gen_msgs() is the general form (needed by the two repair
-# passes below); gen() is the exact proven baseline call, unchanged.
+# Generation -- documented single-call Qwen3 pattern.
 # ---------------------------------------------------------------------------
-def gen_msgs(msgs, max_new_tokens=512):
+def gen_baseline(context, query):
+    msgs = [{"role":"system","content":SYS},
+            {"role":"user","content":f"{context.strip()}\n\n{query.strip()}"}]
     try:
-        enc=tok.apply_chat_template(msgs,add_generation_prompt=True,enable_thinking=False,
-                                    return_tensors="pt",return_dict=True).to(model.device)
-        ilen=enc["input_ids"].shape[-1]
-        with torch.no_grad(): out=model.generate(**enc,max_new_tokens=max_new_tokens,do_sample=False)
+        enc = tok.apply_chat_template(msgs, add_generation_prompt=True, enable_thinking=False,
+                                      return_tensors="pt", return_dict=True).to(model.device)
+        ilen = enc["input_ids"].shape[-1]
+        with torch.no_grad(): out = model.generate(**enc, max_new_tokens=512, do_sample=False)
     except Exception:
-        ids=tok.apply_chat_template(msgs,add_generation_prompt=True,enable_thinking=False,
-                                    return_tensors="pt").to(model.device)
-        ilen=ids.shape[-1]
-        with torch.no_grad(): out=model.generate(ids,max_new_tokens=max_new_tokens,do_sample=False)
-    return tok.decode(out[0][ilen:],skip_special_tokens=True).strip()
-def gen(context, query):
-    return gen_msgs([{"role":"system","content":SYS},
-                     {"role":"user","content":f"{context.strip()}\n\n{query.strip()}"}])
-
-# ===========================================================================
-# ALGORITHM 1 -- match_letters: bijection conflict detection + ONE targeted
-# repair. A matching must be one-to-one, so any reused letter is a
-# guaranteed wrong answer for at least one item, independent of any
-# language knowledge. Never worse: the repair is accepted only if it
-# actually resolves the conflict and has the right shape.
-# ===========================================================================
-def find_bijection_conflicts(answers):
-    pos={}
-    for i,a in enumerate(answers):
-        if a: pos.setdefault(a,[]).append(i)
-    return {v:idxs for v,idxs in pos.items() if len(idxs)>1}
-def repair_bijection(answers, context, query):
-    conflicts=find_bijection_conflicts(answers)
-    if not conflicts: return answers
-    idxs=sorted({i for v in conflicts.values() for i in v})
-    used=sorted({answers[i] for i in range(len(answers)) if i not in idxs and answers[i]})
-    msgs=[{"role":"system","content":"You fix a one-to-one matching. Output only the requested letters, one per line."},
-          {"role":"user","content":(f"{context.strip()}\n\n{query.strip()}\n\n"
-            f"Items {', '.join(str(i+1) for i in idxs)} were all given the SAME answer, "
-            f"which is impossible in a one-to-one matching.\n"
-            f"Already used elsewhere (do not reuse): {', '.join(used) if used else '(none)'}\n"
-            f"Give a DIFFERENT answer for each of those {len(idxs)} items, in order, "
-            f"one per line, nothing else.")}]
-    try: rep=[norm_generic(l) for l in gen_msgs(msgs,96).splitlines() if l.strip()]
-    except Exception: return answers
-    if len(rep)!=len(idxs): return answers
-    cand=list(answers)
-    for i,v in zip(idxs,rep): cand[i]=v
-    return cand if not find_bijection_conflicts(cand) else answers
-
-# ===========================================================================
-# ALGORITHM 2 -- fill_blanks: extract the strongest transformation family
-# from the context's OWN given pairs, then apply it MECHANICALLY to each
-# query stem. An exact computation beats a free-text guess. Emits nothing
-# unless a family is supported by 2+ independent pairs AND shares a real
-# 2-char stem -- one occurrence is indistinguishable from coincidence.
-# ===========================================================================
-def extract_explicit_pairs(context):
-    pairs=[]
-    for line in context.splitlines():
-        line=line.strip()
-        if not (0<line.count("|")<=3): continue
-        fields=[re.sub(r"^\s*\d+\s*[.\)]\s*","",f.strip()).strip() for f in line.split("|")]
-        fields=[f for f in fields if f]
-        if len(fields)>=2: pairs.append((fields[0],fields[1]))
-    return pairs
-def edit_signature(a,b):
-    sm=SequenceMatcher(None,a,b,autojunk=False); allops=sm.get_opcodes()
-    ops=[o for o in allops if o[0]!="equal"]
-    if not ops or len(ops)>2: return None
-    if sum((i2-i1) for t,i1,i2,j1,j2 in allops if t=="equal")<2: return None
-    t,i1,i2,j1,j2=ops[0]
-    if i1==0: pos="prefix"
-    elif i2==len(a): pos="suffix"
-    else: pos="infix"
-    return (pos,a[i1:i2],b[j1:j2])
-def strongest_family(pairs):
-    groups=defaultdict(list)
-    for a,b in pairs:
-        if not a or not b or a==b: continue
-        sig=edit_signature(a,b)
-        if sig: groups[sig].append((a,b))
-    fams=[(len(list(dict.fromkeys(g))),s) for s,g in groups.items()
-          if len(list(dict.fromkeys(g)))>=2]
-    if not fams: return None
-    fams.sort(key=lambda x:-x[0])
-    return fams[0][1]
-def apply_transformation(sig,s):
-    pos,removed,inserted=sig
-    if pos=="prefix":
-        if removed=="": return inserted+s
-        return inserted+s[len(removed):] if s.startswith(removed) else None
-    if pos=="suffix":
-        if removed=="": return s+inserted
-        return s[:-len(removed)]+inserted if s.endswith(removed) else None
-    return None  # infix: no safe anchor, deliberately never applied
-def fill_blanks_override(answers, context, query):
-    pairs=extract_explicit_pairs(context)
-    if not pairs: return answers
-    sig=strongest_family(pairs)
-    if not sig: return answers
-    stems=[m.split("|")[0].strip()
-           for m in re.findall(r"(?m)^\s*\d+\s*[.\)]\s*(.*)$", query)]
-    if not stems: return answers
-    out=list(answers)
-    for i,stem in enumerate(stems):
-        if i>=len(out) or not stem: continue
-        mech=apply_transformation(sig,stem)
-        if mech: out[i]=norm_generic(mech)
-    return out
-
-# ===========================================================================
-# ALGORITHM 3 -- text_to_num: exact arithmetic verification. When the model
-# writes out its own arithmetic (e.g. "6*6+11"), evaluate it EXACTLY via an
-# AST walker (no exec/eval, +,-,* and parentheses only) and prefer the
-# computed value over the model's stated total. Catches the common case of
-# a correct rule with wrong mental arithmetic. Per-line, so index alignment
-# is guaranteed. No prompt change -- the winning prompt stays untouched.
-# ===========================================================================
-_ALLOWED_OPS=(pyast.Add,pyast.Sub,pyast.Mult)
-def safe_arithmetic(expr):
-    try: tree=pyast.parse(expr.strip(),mode="eval")
-    except Exception: return None
-    def _ev(n):
-        if isinstance(n,pyast.Expression): return _ev(n.body)
-        if isinstance(n,pyast.Constant) and isinstance(n.value,(int,float)): return n.value
-        if isinstance(n,pyast.BinOp) and isinstance(n.op,_ALLOWED_OPS):
-            l,r=_ev(n.left),_ev(n.right)
-            if l is None or r is None: return None
-            if isinstance(n.op,pyast.Add): return l+r
-            if isinstance(n.op,pyast.Sub): return l-r
-            if isinstance(n.op,pyast.Mult): return l*r
-        if isinstance(n,pyast.UnaryOp) and isinstance(n.op,pyast.USub):
-            v=_ev(n.operand)
-            return -v if v is not None else None
-        return None
-    return _ev(tree)
-def arithmetic_override(line):
-    for c in sorted(re.findall(r"[0-9]+(?:\s*[\+\-\*]\s*[0-9()]+)+", line), key=len, reverse=True):
-        v=safe_arithmetic(c)
-        if v is not None and float(v).is_integer() and v>=0: return str(int(v))
-    return None
+        ids = tok.apply_chat_template(msgs, add_generation_prompt=True, enable_thinking=False,
+                                      return_tensors="pt").to(model.device)
+        ilen = ids.shape[-1]
+        with torch.no_grad(): out = model.generate(ids, max_new_tokens=512, do_sample=False)
+    return tok.decode(out[0][ilen:], skip_special_tokens=True).strip()
+
+def gen_long_think(context, query):
+    msgs = [{"role":"system","content":SYS},
+            {"role":"user","content":f"{context.strip()}\n\n{query.strip()}"}]
+    try:
+        enc = tok.apply_chat_template(msgs, add_generation_prompt=True, enable_thinking=True,
+                                      return_tensors="pt", return_dict=True).to(model.device)
+        input_length = enc["input_ids"].shape[-1]
+    except Exception:
+        return gen_baseline(context, query)
+    try:
+        with torch.no_grad():
+            out = model.generate(**enc, max_new_tokens=LONG_TOTAL_TOKENS,
+                                 do_sample=True, temperature=0.6, top_p=0.95, top_k=20)
+    except Exception:
+        return gen_baseline(context, query)
+    output_ids = out[0][input_length:].tolist()
+    try:
+        idx = len(output_ids) - output_ids[::-1].index(THINK_CLOSE_TOKEN_ID)
+    except ValueError:
+        idx = 0
+    return tok.decode(output_ids[idx:], skip_special_tokens=True).strip("\n").strip()
 
-# ===========================================================================
-# ALGORITHM 4 -- translation INTO the unseen language: character-inventory
-# hallucination filter. A character that never appears anywhere in the
-# context cannot be part of a correct answer. Detection alone can't raise
-# EM, so a flagged row gets ONE targeted repair naming the constraint
-# (greedy decoding is deterministic -- an identical re-ask would be a
-# guaranteed no-op), and we keep whichever answer set has fewer
-# out-of-inventory characters. Strictly never worse.
-# Translating INTO English is deliberately excluded: English letters
-# legitimately won't appear in an unseen-language context, so the signal
-# would be meaningless there.
-# ===========================================================================
-_NEUTRAL=set(" \t.,;:!?'\"()[]-–—/0123456789\u2018\u2019\u201c\u201d")
-def out_of_inventory(answers, charset):
-    return sum(1 for a in answers for ch in unicodedata.normalize("NFC",a)
-               if ch not in _NEUTRAL and ch not in charset)
-def into_task_language(query, task_lang):
-    q=query.lower()
-    # Hard exclusion FIRST. Without this, a task_lang of "English" (or an
-    # into-English query) satisfies the generic check below and the filter
-    # would fire on exactly the case it must never touch -- flagging every
-    # legitimate English answer as hallucinated and burning a wasted repair
-    # call on each. Strictly worse than baseline; caught by an isolation test.
-    if re.search(r"into\s+english", q): return False
-    tl=(task_lang or "").strip().lower()
-    if tl and tl != "english" and tl in q and "into" in q: return True
-    return bool(re.search(r"translate .{0,40}into (?!english)", q))
-def inventory_repair(answers, context, query, n):
-    charset=set(unicodedata.normalize("NFC",context))
-    bad=out_of_inventory(answers,charset)
-    if bad==0: return answers
-    msgs=[{"role":"system","content":SYS},
-          {"role":"user","content":(f"{context.strip()}\n\n{query.strip()}\n\n"
-            f"Important: your answer must use ONLY characters that actually appear in the "
-            f"examples above. Do not invent letters or symbols that are not shown there. "
-            f"Answer again, one answer per line, nothing else.")}]
-    try: rep=[norm_generic(l) for l in gen_msgs(msgs,512).splitlines() if l.strip()]
-    except Exception: return answers
-    if not rep: return answers
-    if n:
-        if len(rep)<n: rep=rep+[rep[-1]]*(n-len(rep))
-        elif len(rep)>n: rep=rep[:n]
-    return rep if out_of_inventory(rep,charset)<bad else answers
-
-rows=[]; done=set()
+
+rows = []; done = set()
 try:
-    for _,r in df.iterrows():
+    for _, r in df.iterrows():
         try:
-            task_type=r.get("task_type",""); task_lang=r.get("task_lang","")
-            text=gen(r["context"],r["query"])
-            raw=[ln for ln in text.splitlines() if ln.strip()]
-            ans=[normalize(task_type,ln) for ln in raw]
-
-            # ALGO 3 -- exact arithmetic, per line so indices stay aligned.
-            if task_type=="text_to_num":
-                for i,ln in enumerate(raw):
-                    if i>=len(ans): break
-                    v=arithmetic_override(ln)
-                    if v is not None: ans[i]=v
-
-            n=n_expected(r["query"])
-            if n:
-                if len(ans)<n: ans=ans+[ans[-1] if ans else ""]*(n-len(ans))
-                elif len(ans)>n: ans=ans[:n]
-            if not ans: ans=[""]
-
-            # ALGO 2 -- mechanical transformation override (no extra call).
-            if task_type=="fill_blanks":
-                ans=fill_blanks_override(ans,r["context"],r["query"])
-
-            # ALGO 1 / ALGO 4 -- one targeted repair each, only if a real
-            # defect was detected AND real remaining time (freshly measured)
-            # allows it. Both are never-worse by construction.
-            if time.time()-start < TIME_LIMIT-REPAIR_RESERVE:
-                if task_type=="match_letters":
-                    ans=repair_bijection(ans,r["context"],r["query"])
-                elif task_type=="translation" and into_task_language(r["query"],task_lang):
-                    ans=inventory_repair(ans,r["context"],r["query"],n)
-
-            if not ans: ans=[""]
-            expl=re.sub(r"\s+"," ",text[:300]).strip() or "derived from the examples"
-            rows.append({"id":r["id"],"pred":json.dumps(ans,ensure_ascii=False),"explanation":expl})
+            task_type = r.get("task_type", "")
+            text = gen_long_think(r["context"], r["query"]) if task_type in LONG_THINK_TASKS \
+                   else gen_baseline(r["context"], r["query"])
+
+            n = n_expected(r["query"])
+            ans = parse_answers(text, task_type, n)
+            if not ans: ans = [""]
+
+            expl = re.sub(r"\s+", " ", text[:300]).strip() or "derived from the examples"
+            rows.append({"id":r["id"], "pred":json.dumps(ans, ensure_ascii=False),
+                         "explanation":expl})
         except Exception as e:
-            n=n_expected(r["query"]) or 1
-            rows.append({"id":r["id"],"pred":json.dumps([""]*n,ensure_ascii=False),"explanation":"fallback"})
-            print("row error",r["id"],e,flush=True)
+            n = n_expected(r["query"]) or 1
+            rows.append({"id":r["id"], "pred":json.dumps([""]*n, ensure_ascii=False),
+                         "explanation":"fallback"})
+            print("row error", r["id"], e, flush=True)
+
         done.add(r["id"]); write_csv(rows)
-        print(f"{len(rows)}/{len(df)} t={time.time()-start:.0f}s",flush=True)
-        if time.time()-start>TIME_LIMIT-60:
-            print("time up, stopping",flush=True); break
-    for _,r in df.iterrows():
+        elapsed = time.time() - start
+        print(f"{len(rows)}/{len(df)} t={elapsed:.0f}s task={task_type}", flush=True)
+        if elapsed > TIME_LIMIT - SAFETY_S:
+            print("time up, stopping", flush=True); break
+
+    for _, r in df.iterrows():
         if r["id"] in done: continue
-        n=n_expected(r["query"]) or 1
-        rows.append({"id":r["id"],"pred":json.dumps([""]*n,ensure_ascii=False),"explanation":"fallback"})
-    write_csv(rows); print("DONE",flush=True)
+        n = n_expected(r["query"]) or 1
+        rows.append({"id":r["id"], "pred":json.dumps([""]*n, ensure_ascii=False),
+                     "explanation":"fallback"})
+    write_csv(rows); print("DONE", flush=True)
 except Exception as e:
-    emergency(f"main loop: {e}"); print("FATAL",e,flush=True)
+    emergency(f"main loop: {e}"); print("FATAL", e, flush=True)