Buckets:

mahmad1882
/

blind-spots

Files

xet

mahmad1882/blind-spots / code /run_eval.py

mahmad1882

8 days ago

download

raw

18.2 kB

# Blind Spots: non-thinking runner (greedy decoding). Written to run as a notebook cell (Colab, T4).
# Produced the 1,848 thinking-off responses in data/results.csv (run_id 8d6cb8be, 2026-10-21).
# Change from the version that was run: the input path at the bottom points to data/run_input.xlsx
# instead of a Google Drive path. See EXPERIMENT_CONFIG.md.

import argparse
import json
import os
import platform
import time
import uuid

import pandas as pd

# ============================================================
# Configuration
# ============================================================

MODEL_ID = "Qwen/Qwen3.5-4B"

# One system prompt per language code used in the sheet's `language` column.
# Paste the same texts you use in qwen_chat.py (its keys are named differently:
# english -> en, urdu -> ur_script, roman_urdu -> roman_ur,
# colloquial -> colloquial_texting). A language left as None stops the run.
SYSTEM_PROMPTS = {
    "en": (
        "You are a helpful and knowledgeable assistant. "
        "Answer clearly, naturally, and accurately. "
        "Keep your responses concise and relevant to the question."
    ),
    "ur_script": (
        "آپ ایک مددگار اور باخبر معاون ہیں۔ "
        "واضح، فطری اور درست جواب دیں۔ "
        "جواب کو سوال سے متعلق اور مختصر رکھیں۔"
    ),
    "roman_ur": (
        "Aap aik madadgar aur ba-khabar assistant hain. "
        "Wazeh, fitri aur durust jawab dein. "
        "Jawab ko sawal se mutaliq aur mukhtasar rakhein."
    ),
    "colloquial_texting": (
        "Aap aik helpful aur knowledgeable assistant hain. "
        "Seedha, natural aur sahi jawab dein. "
        "Jawab ko relevant aur short rakhein."
    ),
}

EMPTY_THINK = "<think>\n\n</think>\n\n"
THINK_OPEN = "<think>\n"

# Columns copied from the sheet into every result record.
KEEP_COLUMNS = [
    "record_type", "sample_id", "item_id", "pool", "domain", "topic",
    "language", "language_label", "framing_id", "framing_name",
    "prompt", "turn1_prompt", "turn2_prompt", "canonical_gold_answer",
    "true_options", "false_options", "turn1_reference_sample_id",
    "branch_trigger", "turn2_branch", "expected_direction",
]

# ============================================================
# Model wrapper (the only part that needs a GPU)
# ============================================================

def pick_dtype(torch, choice):
    """
    auto: bf16 only on GPUs with native bf16 (compute capability 8.0+, e.g.
    A100, L4); fp16 on older GPUs such as the T4. torch.cuda.is_bf16_supported()
    is not used because it also returns True when bf16 is merely emulated,
    which on a T4 is correct but much slower.
    """
    if not torch.cuda.is_available():
        return torch.float32
    if choice == "bf16":
        return torch.bfloat16
    if choice == "fp16":
        return torch.float16
    major, _ = torch.cuda.get_device_capability(0)
    return torch.bfloat16 if major >= 8 else torch.float16


class QwenGreedy:
    """Batched greedy generation with thinking pinned off."""

    def __init__(self, model_id, max_new_tokens, dtype_choice="auto"):
        import torch
        from transformers import AutoProcessor, AutoModelForImageTextToText

        self.torch = torch
        self.max_new_tokens = max_new_tokens

        self.dtype_choice = dtype_choice
        dtype = pick_dtype(torch, dtype_choice)

        print(f"Loading {model_id} ({dtype})...")
        self.processor = AutoProcessor.from_pretrained(model_id)
        self.tokenizer = self.processor.tokenizer
        # Decoder-only batching needs left padding.
        self.tokenizer.padding_side = "left"
        try:
            self.model = AutoModelForImageTextToText.from_pretrained(
                model_id, dtype=dtype, device_map="auto", attn_implementation="sdpa",
            )
        except TypeError:  # older transformers
            self.model = AutoModelForImageTextToText.from_pretrained(
                model_id, torch_dtype=dtype, device_map="auto", attn_implementation="sdpa",
            )
        self.model.eval()

        eos = self.model.generation_config.eos_token_id
        if eos is None:
            eos = self.tokenizer.eos_token_id
        self.eos_ids = list(eos) if isinstance(eos, (list, tuple)) else [eos]
        self.pad_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.eos_ids[0]
        self.info = {
            "model": model_id,
            "model_revision": getattr(self.model.config, "_commit_hash", None),
            "dtype": str(dtype).replace("torch.", ""),
            "device": torch.cuda.get_device_name(0) if torch.cuda.is_available() else "cpu",
            "torch": torch.__version__,
            "transformers": __import__("transformers").__version__,
        }
        print("Model ready on", self.info["device"])

    def render(self, messages):
        """Chat template, with the prompt ending in an empty think block (thinking off)."""
        try:
            text = self.processor.apply_chat_template(
                messages, tokenize=False, add_generation_prompt=True, enable_thinking=False,
            )
        except TypeError:
            text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
        if text.endswith(EMPTY_THINK):
            text = text[: -len(EMPTY_THINK)]
        elif text.endswith(THINK_OPEN):
            text = text[: -len(THINK_OPEN)]
        return text + EMPTY_THINK

    def length(self, text):
        return len(self.tokenizer(text, add_special_tokens=False)["input_ids"])

    def generate(self, texts):
        """Greedy-decode a batch of rendered prompts. Returns one dict per prompt."""
        torch = self.torch
        enc = self.tokenizer(texts, return_tensors="pt", padding=True, add_special_tokens=False).to(self.model.device)
        with torch.inference_mode():
            out = self.model.generate(
                **enc,
                max_new_tokens=self.max_new_tokens,
                do_sample=False, temperature=None, top_p=None, top_k=None,
                eos_token_id=self.eos_ids, pad_token_id=self.pad_id, use_cache=True,
            )
        new = out[:, enc["input_ids"].shape[1]:].tolist()
        results = []
        for ids in new:
            stop = next((i for i, t in enumerate(ids) if t in self.eos_ids), None)
            used = ids if stop is None else ids[: stop + 1]
            text = self.tokenizer.decode(used, skip_special_tokens=True)
            if "</think>" in text:
                text = text.split("</think>", 1)[1]
            results.append({
                "response": text.strip(),
                "response_tokens": len(used),
                "hit_token_limit": stop is None,
            })
        return results

    def is_oom(self, exc):
        return isinstance(exc, self.torch.cuda.OutOfMemoryError)

    def free(self):
        if self.torch.cuda.is_available():
            self.torch.cuda.empty_cache()

# ============================================================
# Helpers
# ============================================================

def user_msg(text):
    return {"role": "user", "content": [{"type": "text", "text": text}]}


def build_messages(language, turns):
    """turns: list of (role, text). The system prompt depends on the language."""
    messages = [{"role": "system", "content": SYSTEM_PROMPTS[language]}]
    for role, text in turns:
        messages.append(user_msg(text) if role == "user" else {"role": "assistant", "content": text})
    return messages


def load_sheet(path):
    df = pd.read_excel(path, dtype=str).fillna("")
    df["ready_to_run"] = df["ready_to_run"].str.strip().str.lower() == "true"
    return df


def read_done(out_path):
    done = {}
    if os.path.exists(out_path):
        with open(out_path, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if line:
                    rec = json.loads(line)
                    done[rec["sample_id"]] = rec
    return done


def append(out_path, records):
    os.makedirs(os.path.dirname(os.path.abspath(out_path)), exist_ok=True)
    with open(out_path, "a", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec, ensure_ascii=False) + "\n")
        f.flush()
        os.fsync(f.fileno())


def run_batches(gen, jobs, batch_size, out_path, meta, label):
    """
    jobs: list of (row_dict, messages, extra_fields). Sorted longest first so
    an out-of-memory error shows up in the first batch, not the last.
    On OOM the batch size is halved and the batch retried.
    """
    if not jobs:
        print(f"[{label}] nothing to run")
        return
    rendered = [(row, gen.render(msgs), extra) for row, msgs, extra in jobs]
    rendered.sort(key=lambda x: -gen.length(x[1]))
    total, i, bs, started = len(rendered), 0, batch_size, time.time()
    print(f"[{label}] {total} prompts, batch size {bs}")
    while i < total:
        chunk = rendered[i: i + bs]
        try:
            outs = gen.generate([text for _, text, _ in chunk])
        except Exception as exc:
            if gen.is_oom(exc) and bs > 1:
                gen.free()
                bs = max(1, bs // 2)
                print(f"[{label}] out of memory; retrying with batch size {bs}")
                continue
            raise
        now = time.strftime("%Y-%m-%d %H:%M:%S")
        records = []
        for (row, _, extra), out in zip(chunk, outs):
            rec = {k: row.get(k, "") for k in KEEP_COLUMNS}
            rec.update(extra)
            rec.update(out)
            rec.update(meta)
            rec.update({"time": now, "batch_size_used": bs})
            records.append(rec)
        append(out_path, records)
        i += len(chunk)
        rate = (time.time() - started) / i
        print(f"[{label}] {i}/{total} done, ~{rate * (total - i) / 60:.1f} min left")

# ============================================================
# Phases
# ============================================================

def select(df, args):
    sel = df
    if getattr(args, "items", None):
        sel = sel[sel.item_id.isin(args.items.split(","))]
    if getattr(args, "languages", None):
        sel = sel[sel.language.isin(args.languages.split(","))]
    if getattr(args, "only_ready", False):
        sel = sel[sel.ready_to_run]
    return sel


def check_system_prompts(languages):
    missing = sorted({l for l in languages if not SYSTEM_PROMPTS.get(l)})
    if missing:
        raise SystemExit(f"Fill SYSTEM_PROMPTS for: {', '.join(missing)} (top of the script).")


def phase_single(gen, df, args, meta, done):
    rows = df[df.record_type == "single_turn"]
    empty = rows.prompt.str.strip() == ""
    if empty.any():
        print(f"[single] skipping {empty.sum()} rows with no prompt written yet")
        rows = rows[~empty]
    if getattr(args, "limit", None):
        rows = rows.head(args.limit)
    jobs = []
    for row in rows.to_dict("records"):
        if row["sample_id"] in done:
            continue
        msgs = build_messages(row["language"], [("user", row["prompt"])])
        jobs.append((row, msgs, {"system_prompt": SYSTEM_PROMPTS[row["language"]], "turn": 1}))
    print(f"[single] {len(rows)} selected, {len(rows) - len(jobs)} already done")
    run_batches(gen, jobs, args.batch_size, args.out, meta, "single")


def phase_multi(gen, df, args, meta, done):
    rows = df[df.record_type != "single_turn"]
    jobs, waiting = [], []
    for row in rows.to_dict("records"):
        if row["sample_id"] in done:
            continue
        ref = done.get(row["turn1_reference_sample_id"])
        if ref is None:
            waiting.append(row["sample_id"])
            continue
        if row["turn1_prompt"].strip() != ref["prompt"].strip():
            print(f"[multi] warning: {row['sample_id']} turn-1 prompt differs from {ref['sample_id']}")
        turn1 = ref["response"]
        msgs = build_messages(row["language"], [
            ("user", row["turn1_prompt"]), ("assistant", turn1), ("user", row["turn2_prompt"]),
        ])
        jobs.append((row, msgs, {
            "system_prompt": SYSTEM_PROMPTS[row["language"]],
            "turn": 2,
            "turn1_response": turn1,
            "turn1_source_sample_id": ref["sample_id"],
        }))
    if waiting:
        print(f"[multi] {len(waiting)} rows skipped: their F1 reference hasn't been run yet")
    print(f"[multi] {len(rows)} selected, {len(jobs)} to run")
    run_batches(gen, jobs, args.batch_size, args.out, meta, "multi")


def verify_batching(gen, df, args):
    rows = df[(df.record_type == "single_turn") & (df.prompt.str.strip() != "")]
    rows = rows.head(args.verify_batching).to_dict("records")
    check_system_prompts({r["language"] for r in rows})
    texts = [gen.render(build_messages(r["language"], [("user", r["prompt"])])) for r in rows]
    one = [gen.generate([t])[0]["response"] for t in texts]
    many = []
    for i in range(0, len(texts), args.batch_size):
        many += [o["response"] for o in gen.generate(texts[i: i + args.batch_size])]
    same = sum(a == b for a, b in zip(one, many))
    print(f"\nBatch size 1 vs {args.batch_size}: {same}/{len(rows)} identical responses")
    for r, a, b in zip(rows, one, many):
        if a != b:
            print(f"\n--- {r['sample_id']}\n[bs=1] {a[:300]}\n[batched] {b[:300]}")


def export_xlsx(sheet_path, out_path, xlsx_out):
    df = pd.read_excel(sheet_path, dtype=str).fillna("")
    done = read_done(out_path)
    resp = {k: v["response"] for k, v in done.items()}
    t1 = {k: v.get("turn1_response", "") for k, v in done.items()}
    df["model_response"] = df["sample_id"].map(resp).fillna("")
    pos = df.columns.get_loc("model_response")
    df.insert(pos, "turn1_response", df["sample_id"].map(t1).fillna(""))
    df.to_excel(xlsx_out, index=False)
    print(f"Wrote {xlsx_out} ({(df.model_response != '').sum()} responses)")


def push(bucket, paths, prefix="runs"):
    try:
        from huggingface_hub import batch_bucket_files, create_bucket
        create_bucket(bucket, private=True, exist_ok=True)
        batch_bucket_files(bucket, add=[(p, f"{prefix}/{os.path.basename(p)}") for p in paths if p and os.path.exists(p)])
        print(f"Pushed to hf://buckets/{bucket}/{prefix}/")
    except Exception as exc:
        print(f"Push failed ({type(exc).__name__}: {exc}); files are still saved locally.")

# ============================================================
# Entry point for direct notebook executions
# ============================================================

class ConfigArgs:
    def __init__(self, **kwargs):
        self.xlsx = kwargs.get("xlsx")
        self.out = kwargs.get("out", "results/results.jsonl")
        self.phase = kwargs.get("phase", "all")
        self.batch_size = kwargs.get("batch_size", 16)
        self.max_new_tokens = kwargs.get("max_new_tokens", 384)
        self.items = kwargs.get("items", None)
        self.languages = kwargs.get("languages", None)
        self.limit = kwargs.get("limit", None)
        self.only_ready = kwargs.get("only_ready", False)
        self.seed = kwargs.get("seed", 0)
        self.dtype = kwargs.get("dtype", "auto")  # "auto", "bf16" or "fp16"
        self.verify_batching = kwargs.get("verify_batching", None)
        self.export_xlsx = kwargs.get("export_xlsx", None)
        self.bucket = kwargs.get("bucket", None)

# The model stays loaded between calls, so a verify run followed by the full run
# (or a resume after an error) doesn't reload 9 GB of weights each time.
_GEN = None


def get_generator(args):
    global _GEN
    if _GEN is not None and _GEN.dtype_choice == args.dtype:
        _GEN.max_new_tokens = args.max_new_tokens
        print(f"Reusing loaded model ({_GEN.info['dtype']} on {_GEN.info['device']})")
        return _GEN
    if _GEN is not None:  # dtype changed: drop the old copy before loading a new one
        del _GEN.model
        _GEN.free()
        _GEN = None
    _GEN = QwenGreedy(MODEL_ID, args.max_new_tokens, args.dtype)
    return _GEN


def run_eval_notebook(config_dict):
    args = ConfigArgs(**config_dict)

    if not args.xlsx or not os.path.exists(args.xlsx):
        raise FileNotFoundError(f"Please upload your XLSX file or configure the correct path. Provided: {args.xlsx}")

    df = select(load_sheet(args.xlsx), args)
    not_ready = (~df.ready_to_run).sum()
    if not_ready and not args.only_ready:
        print(f"Note: {not_ready} selected rows are not marked ready_to_run; running them anyway.")

    from transformers import set_seed
    set_seed(args.seed)
    gen = get_generator(args)

    if args.verify_batching:
        verify_batching(gen, df, args)
        return

    check_system_prompts(set(df.language))
    meta = {
        "run_id": uuid.uuid4().hex[:8],
        "thinking": False,
        "decoding": "greedy",
        "max_new_tokens": args.max_new_tokens,
        "seed": args.seed,
        "python": platform.python_version(),
        **gen.info,
    }

    if args.phase in ("single", "all"):
        phase_single(gen, df, args, meta, read_done(args.out))
    if args.phase in ("multi", "all"):
        phase_multi(gen, df, args, meta, read_done(args.out))

    if args.export_xlsx:
        export_xlsx(args.xlsx, args.out, args.export_xlsx)
    if args.bucket:
        push(args.bucket, [args.out, args.export_xlsx])


config = {
    "xlsx": "data/run_input.xlsx",               # The run input: data/run_input.xlsx in this repo
    "out": "results.jsonl",                      # Local JSONL file where progress is saved
    "export_xlsx": "prompts_with_responses.xlsx", # Path to save the final populated Excel sheet
    "batch_size": 32,                            # Batch size (will auto-halve on CUDA OOM)
    "phase": "all",                              # Options: "single", "multi", or "all"
    # "items": "N01,B01,E01",                    # Uncomment to run a dry run on specific item IDs
    # "limit": 5,                                # Uncomment to limit single-turn rows during tests
}

# 4. Execute the evaluation
run_eval_notebook(config)

Xet Storage Details

Size:: 18.2 kB
Xet hash:: b005480a43fea643d9308f8a9ff9ba1a53e8be0536bf5faa2f9e51459078e6b5

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	# Blind Spots: non-thinking runner (greedy decoding). Written to run as a notebook cell (Colab, T4).
	# Produced the 1,848 thinking-off responses in data/results.csv (run_id 8d6cb8be, 2026-10-21).
	# Change from the version that was run: the input path at the bottom points to data/run_input.xlsx
	# instead of a Google Drive path. See EXPERIMENT_CONFIG.md.

	import argparse
	import json
	import os
	import platform
	import time
	import uuid

	import pandas as pd

	# ============================================================
	# Configuration
	# ============================================================

	MODEL_ID = "Qwen/Qwen3.5-4B"

	# One system prompt per language code used in the sheet's `language` column.
	# Paste the same texts you use in qwen_chat.py (its keys are named differently:
	# english -> en, urdu -> ur_script, roman_urdu -> roman_ur,
	# colloquial -> colloquial_texting). A language left as None stops the run.
	SYSTEM_PROMPTS = {
	"en": (
	"You are a helpful and knowledgeable assistant. "
	"Answer clearly, naturally, and accurately. "
	"Keep your responses concise and relevant to the question."
	),
	"ur_script": (
	"آپ ایک مددگار اور باخبر معاون ہیں۔ "
	"واضح، فطری اور درست جواب دیں۔ "
	"جواب کو سوال سے متعلق اور مختصر رکھیں۔"
	),
	"roman_ur": (
	"Aap aik madadgar aur ba-khabar assistant hain. "
	"Wazeh, fitri aur durust jawab dein. "
	"Jawab ko sawal se mutaliq aur mukhtasar rakhein."
	),
	"colloquial_texting": (
	"Aap aik helpful aur knowledgeable assistant hain. "
	"Seedha, natural aur sahi jawab dein. "
	"Jawab ko relevant aur short rakhein."
	),
	}

	EMPTY_THINK = "<think>\n\n</think>\n\n"
	THINK_OPEN = "<think>\n"

	# Columns copied from the sheet into every result record.
	KEEP_COLUMNS = [
	"record_type", "sample_id", "item_id", "pool", "domain", "topic",
	"language", "language_label", "framing_id", "framing_name",
	"prompt", "turn1_prompt", "turn2_prompt", "canonical_gold_answer",
	"true_options", "false_options", "turn1_reference_sample_id",
	"branch_trigger", "turn2_branch", "expected_direction",
	]

	# ============================================================
	# Model wrapper (the only part that needs a GPU)
	# ============================================================

	def pick_dtype(torch, choice):
	"""
	auto: bf16 only on GPUs with native bf16 (compute capability 8.0+, e.g.
	A100, L4); fp16 on older GPUs such as the T4. torch.cuda.is_bf16_supported()
	is not used because it also returns True when bf16 is merely emulated,
	which on a T4 is correct but much slower.
	"""
	if not torch.cuda.is_available():
	return torch.float32
	if choice == "bf16":
	return torch.bfloat16
	if choice == "fp16":
	return torch.float16
	major, _ = torch.cuda.get_device_capability(0)
	return torch.bfloat16 if major >= 8 else torch.float16


	class QwenGreedy:
	"""Batched greedy generation with thinking pinned off."""

	def __init__(self, model_id, max_new_tokens, dtype_choice="auto"):
	import torch
	from transformers import AutoProcessor, AutoModelForImageTextToText

	self.torch = torch
	self.max_new_tokens = max_new_tokens

	self.dtype_choice = dtype_choice
	dtype = pick_dtype(torch, dtype_choice)

	print(f"Loading {model_id} ({dtype})...")
	self.processor = AutoProcessor.from_pretrained(model_id)
	self.tokenizer = self.processor.tokenizer
	# Decoder-only batching needs left padding.
	self.tokenizer.padding_side = "left"
	try:
	self.model = AutoModelForImageTextToText.from_pretrained(
	model_id, dtype=dtype, device_map="auto", attn_implementation="sdpa",
	)
	except TypeError: # older transformers
	self.model = AutoModelForImageTextToText.from_pretrained(
	model_id, torch_dtype=dtype, device_map="auto", attn_implementation="sdpa",
	)
	self.model.eval()

	eos = self.model.generation_config.eos_token_id
	if eos is None:
	eos = self.tokenizer.eos_token_id
	self.eos_ids = list(eos) if isinstance(eos, (list, tuple)) else [eos]
	self.pad_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.eos_ids[0]
	self.info = {
	"model": model_id,
	"model_revision": getattr(self.model.config, "_commit_hash", None),
	"dtype": str(dtype).replace("torch.", ""),
	"device": torch.cuda.get_device_name(0) if torch.cuda.is_available() else "cpu",
	"torch": torch.__version__,
	"transformers": __import__("transformers").__version__,
	}
	print("Model ready on", self.info["device"])

	def render(self, messages):
	"""Chat template, with the prompt ending in an empty think block (thinking off)."""
	try:
	text = self.processor.apply_chat_template(
	messages, tokenize=False, add_generation_prompt=True, enable_thinking=False,
	)
	except TypeError:
	text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
	if text.endswith(EMPTY_THINK):
	text = text[: -len(EMPTY_THINK)]
	elif text.endswith(THINK_OPEN):
	text = text[: -len(THINK_OPEN)]
	return text + EMPTY_THINK

	def length(self, text):
	return len(self.tokenizer(text, add_special_tokens=False)["input_ids"])

	def generate(self, texts):
	"""Greedy-decode a batch of rendered prompts. Returns one dict per prompt."""
	torch = self.torch
	enc = self.tokenizer(texts, return_tensors="pt", padding=True, add_special_tokens=False).to(self.model.device)
	with torch.inference_mode():
	out = self.model.generate(
	**enc,
	max_new_tokens=self.max_new_tokens,
	do_sample=False, temperature=None, top_p=None, top_k=None,
	eos_token_id=self.eos_ids, pad_token_id=self.pad_id, use_cache=True,
	)
	new = out[:, enc["input_ids"].shape[1]:].tolist()
	results = []
	for ids in new:
	stop = next((i for i, t in enumerate(ids) if t in self.eos_ids), None)
	used = ids if stop is None else ids[: stop + 1]
	text = self.tokenizer.decode(used, skip_special_tokens=True)
	if "</think>" in text:
	text = text.split("</think>", 1)[1]
	results.append({
	"response": text.strip(),
	"response_tokens": len(used),
	"hit_token_limit": stop is None,
	})
	return results

	def is_oom(self, exc):
	return isinstance(exc, self.torch.cuda.OutOfMemoryError)

	def free(self):
	if self.torch.cuda.is_available():
	self.torch.cuda.empty_cache()

	# ============================================================
	# Helpers
	# ============================================================

	def user_msg(text):
	return {"role": "user", "content": [{"type": "text", "text": text}]}


	def build_messages(language, turns):
	"""turns: list of (role, text). The system prompt depends on the language."""
	messages = [{"role": "system", "content": SYSTEM_PROMPTS[language]}]
	for role, text in turns:
	messages.append(user_msg(text) if role == "user" else {"role": "assistant", "content": text})
	return messages


	def load_sheet(path):
	df = pd.read_excel(path, dtype=str).fillna("")
	df["ready_to_run"] = df["ready_to_run"].str.strip().str.lower() == "true"
	return df


	def read_done(out_path):
	done = {}
	if os.path.exists(out_path):
	with open(out_path, encoding="utf-8") as f:
	for line in f:
	line = line.strip()
	if line:
	rec = json.loads(line)
	done[rec["sample_id"]] = rec
	return done


	def append(out_path, records):
	os.makedirs(os.path.dirname(os.path.abspath(out_path)), exist_ok=True)
	with open(out_path, "a", encoding="utf-8") as f:
	for rec in records:
	f.write(json.dumps(rec, ensure_ascii=False) + "\n")
	f.flush()
	os.fsync(f.fileno())


	def run_batches(gen, jobs, batch_size, out_path, meta, label):
	"""
	jobs: list of (row_dict, messages, extra_fields). Sorted longest first so
	an out-of-memory error shows up in the first batch, not the last.
	On OOM the batch size is halved and the batch retried.
	"""
	if not jobs:
	print(f"[{label}] nothing to run")
	return
	rendered = [(row, gen.render(msgs), extra) for row, msgs, extra in jobs]
	rendered.sort(key=lambda x: -gen.length(x[1]))
	total, i, bs, started = len(rendered), 0, batch_size, time.time()
	print(f"[{label}] {total} prompts, batch size {bs}")
	while i < total:
	chunk = rendered[i: i + bs]
	try:
	outs = gen.generate([text for _, text, _ in chunk])
	except Exception as exc:
	if gen.is_oom(exc) and bs > 1:
	gen.free()
	bs = max(1, bs // 2)
	print(f"[{label}] out of memory; retrying with batch size {bs}")
	continue
	raise
	now = time.strftime("%Y-%m-%d %H:%M:%S")
	records = []
	for (row, _, extra), out in zip(chunk, outs):
	rec = {k: row.get(k, "") for k in KEEP_COLUMNS}
	rec.update(extra)
	rec.update(out)
	rec.update(meta)
	rec.update({"time": now, "batch_size_used": bs})
	records.append(rec)
	append(out_path, records)
	i += len(chunk)
	rate = (time.time() - started) / i
	print(f"[{label}] {i}/{total} done, ~{rate * (total - i) / 60:.1f} min left")

	# ============================================================
	# Phases
	# ============================================================

	def select(df, args):
	sel = df
	if getattr(args, "items", None):
	sel = sel[sel.item_id.isin(args.items.split(","))]
	if getattr(args, "languages", None):
	sel = sel[sel.language.isin(args.languages.split(","))]
	if getattr(args, "only_ready", False):
	sel = sel[sel.ready_to_run]
	return sel


	def check_system_prompts(languages):
	missing = sorted({l for l in languages if not SYSTEM_PROMPTS.get(l)})
	if missing:
	raise SystemExit(f"Fill SYSTEM_PROMPTS for: {', '.join(missing)} (top of the script).")


	def phase_single(gen, df, args, meta, done):
	rows = df[df.record_type == "single_turn"]
	empty = rows.prompt.str.strip() == ""
	if empty.any():
	print(f"[single] skipping {empty.sum()} rows with no prompt written yet")
	rows = rows[~empty]
	if getattr(args, "limit", None):
	rows = rows.head(args.limit)
	jobs = []
	for row in rows.to_dict("records"):
	if row["sample_id"] in done:
	continue
	msgs = build_messages(row["language"], [("user", row["prompt"])])
	jobs.append((row, msgs, {"system_prompt": SYSTEM_PROMPTS[row["language"]], "turn": 1}))
	print(f"[single] {len(rows)} selected, {len(rows) - len(jobs)} already done")
	run_batches(gen, jobs, args.batch_size, args.out, meta, "single")


	def phase_multi(gen, df, args, meta, done):
	rows = df[df.record_type != "single_turn"]
	jobs, waiting = [], []
	for row in rows.to_dict("records"):
	if row["sample_id"] in done:
	continue
	ref = done.get(row["turn1_reference_sample_id"])
	if ref is None:
	waiting.append(row["sample_id"])
	continue
	if row["turn1_prompt"].strip() != ref["prompt"].strip():
	print(f"[multi] warning: {row['sample_id']} turn-1 prompt differs from {ref['sample_id']}")
	turn1 = ref["response"]
	msgs = build_messages(row["language"], [
	("user", row["turn1_prompt"]), ("assistant", turn1), ("user", row["turn2_prompt"]),
	])
	jobs.append((row, msgs, {
	"system_prompt": SYSTEM_PROMPTS[row["language"]],
	"turn": 2,
	"turn1_response": turn1,
	"turn1_source_sample_id": ref["sample_id"],
	}))
	if waiting:
	print(f"[multi] {len(waiting)} rows skipped: their F1 reference hasn't been run yet")
	print(f"[multi] {len(rows)} selected, {len(jobs)} to run")
	run_batches(gen, jobs, args.batch_size, args.out, meta, "multi")


	def verify_batching(gen, df, args):
	rows = df[(df.record_type == "single_turn") & (df.prompt.str.strip() != "")]
	rows = rows.head(args.verify_batching).to_dict("records")
	check_system_prompts({r["language"] for r in rows})
	texts = [gen.render(build_messages(r["language"], [("user", r["prompt"])])) for r in rows]
	one = [gen.generate([t])[0]["response"] for t in texts]
	many = []
	for i in range(0, len(texts), args.batch_size):
	many += [o["response"] for o in gen.generate(texts[i: i + args.batch_size])]
	same = sum(a == b for a, b in zip(one, many))
	print(f"\nBatch size 1 vs {args.batch_size}: {same}/{len(rows)} identical responses")
	for r, a, b in zip(rows, one, many):
	if a != b:
	print(f"\n--- {r['sample_id']}\n[bs=1] {a[:300]}\n[batched] {b[:300]}")


	def export_xlsx(sheet_path, out_path, xlsx_out):
	df = pd.read_excel(sheet_path, dtype=str).fillna("")
	done = read_done(out_path)
	resp = {k: v["response"] for k, v in done.items()}
	t1 = {k: v.get("turn1_response", "") for k, v in done.items()}
	df["model_response"] = df["sample_id"].map(resp).fillna("")
	pos = df.columns.get_loc("model_response")
	df.insert(pos, "turn1_response", df["sample_id"].map(t1).fillna(""))
	df.to_excel(xlsx_out, index=False)
	print(f"Wrote {xlsx_out} ({(df.model_response != '').sum()} responses)")


	def push(bucket, paths, prefix="runs"):
	try:
	from huggingface_hub import batch_bucket_files, create_bucket
	create_bucket(bucket, private=True, exist_ok=True)
	batch_bucket_files(bucket, add=[(p, f"{prefix}/{os.path.basename(p)}") for p in paths if p and os.path.exists(p)])
	print(f"Pushed to hf://buckets/{bucket}/{prefix}/")
	except Exception as exc:
	print(f"Push failed ({type(exc).__name__}: {exc}); files are still saved locally.")

	# ============================================================
	# Entry point for direct notebook executions
	# ============================================================

	class ConfigArgs:
	def __init__(self, **kwargs):
	self.xlsx = kwargs.get("xlsx")
	self.out = kwargs.get("out", "results/results.jsonl")
	self.phase = kwargs.get("phase", "all")
	self.batch_size = kwargs.get("batch_size", 16)
	self.max_new_tokens = kwargs.get("max_new_tokens", 384)
	self.items = kwargs.get("items", None)
	self.languages = kwargs.get("languages", None)
	self.limit = kwargs.get("limit", None)
	self.only_ready = kwargs.get("only_ready", False)
	self.seed = kwargs.get("seed", 0)
	self.dtype = kwargs.get("dtype", "auto") # "auto", "bf16" or "fp16"
	self.verify_batching = kwargs.get("verify_batching", None)
	self.export_xlsx = kwargs.get("export_xlsx", None)
	self.bucket = kwargs.get("bucket", None)

	# The model stays loaded between calls, so a verify run followed by the full run
	# (or a resume after an error) doesn't reload 9 GB of weights each time.
	_GEN = None


	def get_generator(args):
	global _GEN
	if _GEN is not None and _GEN.dtype_choice == args.dtype:
	_GEN.max_new_tokens = args.max_new_tokens
	print(f"Reusing loaded model ({_GEN.info['dtype']} on {_GEN.info['device']})")
	return _GEN
	if _GEN is not None: # dtype changed: drop the old copy before loading a new one
	del _GEN.model
	_GEN.free()
	_GEN = None
	_GEN = QwenGreedy(MODEL_ID, args.max_new_tokens, args.dtype)
	return _GEN


	def run_eval_notebook(config_dict):
	args = ConfigArgs(**config_dict)

	if not args.xlsx or not os.path.exists(args.xlsx):
	raise FileNotFoundError(f"Please upload your XLSX file or configure the correct path. Provided: {args.xlsx}")

	df = select(load_sheet(args.xlsx), args)
	not_ready = (~df.ready_to_run).sum()
	if not_ready and not args.only_ready:
	print(f"Note: {not_ready} selected rows are not marked ready_to_run; running them anyway.")

	from transformers import set_seed
	set_seed(args.seed)
	gen = get_generator(args)

	if args.verify_batching:
	verify_batching(gen, df, args)
	return

	check_system_prompts(set(df.language))
	meta = {
	"run_id": uuid.uuid4().hex[:8],
	"thinking": False,
	"decoding": "greedy",
	"max_new_tokens": args.max_new_tokens,
	"seed": args.seed,
	"python": platform.python_version(),
	**gen.info,
	}

	if args.phase in ("single", "all"):
	phase_single(gen, df, args, meta, read_done(args.out))
	if args.phase in ("multi", "all"):
	phase_multi(gen, df, args, meta, read_done(args.out))

	if args.export_xlsx:
	export_xlsx(args.xlsx, args.out, args.export_xlsx)
	if args.bucket:
	push(args.bucket, [args.out, args.export_xlsx])


	config = {
	"xlsx": "data/run_input.xlsx", # The run input: data/run_input.xlsx in this repo
	"out": "results.jsonl", # Local JSONL file where progress is saved
	"export_xlsx": "prompts_with_responses.xlsx", # Path to save the final populated Excel sheet
	"batch_size": 32, # Batch size (will auto-halve on CUDA OOM)
	"phase": "all", # Options: "single", "multi", or "all"
	# "items": "N01,B01,E01", # Uncomment to run a dry run on specific item IDs
	# "limit": 5, # Uncomment to limit single-turn rows during tests
	}

	# 4. Execute the evaluation
	run_eval_notebook(config)