Buckets:
| # Blind Spots: thinking / non-thinking runner. Written to run as a notebook cell (Colab, T4). | |
| # Thinking on: Qwen's recommended sampling, a 2,048-token reasoning budget, then Qwen's early-exit phrase. | |
| # Produced the 428 thinking-on responses in data/results.csv and, with the fix below, the 40 traces in | |
| # data/thinking_traces.csv. See EXPERIMENT_CONFIG.md. | |
| # Changes from the version that produced the 428 rows: | |
| # 1. Bug fix: run metadata used the key "thinking", which overwrote each record's reasoning text with True. | |
| # The key is now "thinking_mode" (as in the traces run). The 428-row run's reasoning was lost to this bug. | |
| # 2. Input / output paths point to data/run_input.xlsx and local files instead of Google Drive. | |
| import argparse | |
| import json | |
| import os | |
| import platform | |
| import time | |
| import uuid | |
| import pandas as pd | |
| # ============================================================ | |
| # Configuration | |
| # ============================================================ | |
| MODEL_ID = "Qwen/Qwen3.5-4B" | |
| # One system prompt per language code used in the sheet's `language` column. | |
| # Paste the same texts you use in qwen_chat.py (its keys are named differently: | |
| # english -> en, urdu -> ur_script, roman_urdu -> roman_ur, | |
| # colloquial -> colloquial_texting). A language left as None stops the run. | |
| SYSTEM_PROMPTS = { | |
| "en": ( | |
| "You are a helpful and knowledgeable assistant. " | |
| "Answer clearly, naturally, and accurately. " | |
| "Keep your responses concise and relevant to the question." | |
| ), | |
| "ur_script": ( | |
| "آپ ایک مددگار اور باخبر معاون ہیں۔ " | |
| "واضح، فطری اور درست جواب دیں۔ " | |
| "جواب کو سوال سے متعلق اور مختصر رکھیں۔" | |
| ), | |
| "roman_ur": ( | |
| "Aap aik madadgar aur ba-khabar assistant hain. " | |
| "Wazeh, fitri aur durust jawab dein. " | |
| "Jawab ko sawal se mutaliq aur mukhtasar rakhein." | |
| ), | |
| "colloquial_texting": ( | |
| "Aap aik helpful aur knowledgeable assistant hain. " | |
| "Seedha, natural aur sahi jawab dein. " | |
| "Jawab ko relevant aur short rakhein." | |
| ), | |
| } | |
| EMPTY_THINK = "<think>\n\n</think>\n\n" | |
| THINK_OPEN = "<think>\n" | |
| THINK_CLOSE = "</think>\n\n" | |
| # Thinking mode. When the reasoning hasn't closed by the budget, this exact text | |
| # is appended before the answer: it is the early-exit phrase from Qwen's own | |
| # quickstart docs (qwen.readthedocs.io), so it is a standard, citable choice. | |
| QWEN_EARLY_EXIT = ( | |
| "\n\nConsidering the limited time by the user, I have to give the solution " | |
| "based on the thinking directly now.\n</think>\n\n" | |
| ) | |
| # Qwen's recommended sampling for thinking mode, general tasks. | |
| THINKING_SAMPLING = {"temperature": 1.0, "top_p": 0.95, "top_k": 20, "presence_penalty": 1.5} | |
| class PresencePenalty: | |
| """Subtract a flat penalty from every token generated so far (the prompt is not penalised).""" | |
| def __init__(self, penalty, prompt_length): | |
| self.penalty, self.prompt_length = penalty, prompt_length | |
| def __call__(self, input_ids, scores): | |
| import torch | |
| generated = input_ids[:, self.prompt_length:] | |
| if generated.shape[1] == 0: | |
| return scores | |
| seen = torch.zeros_like(scores, dtype=torch.bool) | |
| seen.scatter_(1, generated.clamp(max=scores.shape[1] - 1), True) | |
| return scores - self.penalty * seen.to(scores.dtype) | |
| # Columns copied from the sheet into every result record. | |
| KEEP_COLUMNS = [ | |
| "record_type", "sample_id", "item_id", "pool", "domain", "topic", | |
| "language", "language_label", "framing_id", "framing_name", | |
| "prompt", "turn1_prompt", "turn2_prompt", "canonical_gold_answer", | |
| "true_options", "false_options", "turn1_reference_sample_id", | |
| "branch_trigger", "turn2_branch", "expected_direction", | |
| ] | |
| # ============================================================ | |
| # Model wrapper (the only part that needs a GPU) | |
| # ============================================================ | |
| def pick_dtype(torch, choice): | |
| """ | |
| auto: bf16 only on GPUs with native bf16 (compute capability 8.0+, e.g. | |
| A100, L4); fp16 on older GPUs such as the T4. torch.cuda.is_bf16_supported() | |
| is not used because it also returns True when bf16 is merely emulated, | |
| which on a T4 is correct but much slower. | |
| """ | |
| if not torch.cuda.is_available(): | |
| return torch.float32 | |
| if choice == "bf16": | |
| return torch.bfloat16 | |
| if choice == "fp16": | |
| return torch.float16 | |
| major, _ = torch.cuda.get_device_capability(0) | |
| return torch.bfloat16 if major >= 8 else torch.float16 | |
| class QwenGreedy: | |
| """Batched greedy generation with thinking pinned off.""" | |
| def __init__(self, model_id, max_new_tokens, dtype_choice="auto"): | |
| import torch | |
| from transformers import AutoProcessor, AutoModelForImageTextToText | |
| self.torch = torch | |
| self.max_new_tokens = max_new_tokens | |
| self.dtype_choice = dtype_choice | |
| self.thinking = False # set per run by get_generator() | |
| self.think_budget = 2048 | |
| dtype = pick_dtype(torch, dtype_choice) | |
| print(f"Loading {model_id} ({dtype})...") | |
| self.processor = AutoProcessor.from_pretrained(model_id) | |
| self.tokenizer = self.processor.tokenizer | |
| # Decoder-only batching needs left padding. | |
| self.tokenizer.padding_side = "left" | |
| try: | |
| self.model = AutoModelForImageTextToText.from_pretrained( | |
| model_id, dtype=dtype, device_map="auto", attn_implementation="sdpa", | |
| ) | |
| except TypeError: # older transformers | |
| self.model = AutoModelForImageTextToText.from_pretrained( | |
| model_id, torch_dtype=dtype, device_map="auto", attn_implementation="sdpa", | |
| ) | |
| self.model.eval() | |
| eos = self.model.generation_config.eos_token_id | |
| if eos is None: | |
| eos = self.tokenizer.eos_token_id | |
| self.eos_ids = list(eos) if isinstance(eos, (list, tuple)) else [eos] | |
| self.pad_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.eos_ids[0] | |
| self.info = { | |
| "model": model_id, | |
| "model_revision": getattr(self.model.config, "_commit_hash", None), | |
| "dtype": str(dtype).replace("torch.", ""), | |
| "device": torch.cuda.get_device_name(0) if torch.cuda.is_available() else "cpu", | |
| "torch": torch.__version__, | |
| "transformers": __import__("transformers").__version__, | |
| } | |
| end = self.tokenizer.convert_tokens_to_ids("</think>") | |
| self.think_end_id = None if end is None or end == self.tokenizer.unk_token_id else end | |
| print("Model ready on", self.info["device"]) | |
| def render(self, messages): | |
| """ | |
| Chat template with the thinking state pinned: thinking off ends the prompt | |
| with an empty think block; thinking on ends it with an open <think>. | |
| """ | |
| try: | |
| text = self.processor.apply_chat_template( | |
| messages, tokenize=False, add_generation_prompt=True, enable_thinking=self.thinking, | |
| ) | |
| except TypeError: | |
| text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) | |
| if text.endswith(EMPTY_THINK): | |
| text = text[: -len(EMPTY_THINK)] | |
| elif text.endswith(THINK_OPEN): | |
| text = text[: -len(THINK_OPEN)] | |
| return text + (THINK_OPEN if self.thinking else EMPTY_THINK) | |
| def length(self, text): | |
| return len(self.tokenizer(text, add_special_tokens=False)["input_ids"]) | |
| def _generate_ids(self, texts, max_new_tokens, stop_ids): | |
| """One batched generate call. Returns the new token ids for each prompt.""" | |
| torch = self.torch | |
| enc = self.tokenizer(texts, return_tensors="pt", padding=True, add_special_tokens=False).to(self.model.device) | |
| prompt_length = enc["input_ids"].shape[1] | |
| kwargs = dict(max_new_tokens=max_new_tokens, eos_token_id=stop_ids, | |
| pad_token_id=self.pad_id, use_cache=True) | |
| if self.thinking: | |
| from transformers import LogitsProcessorList | |
| sp = THINKING_SAMPLING | |
| kwargs.update(do_sample=True, temperature=sp["temperature"], top_p=sp["top_p"], top_k=sp["top_k"], | |
| logits_processor=LogitsProcessorList([PresencePenalty(sp["presence_penalty"], prompt_length)])) | |
| else: | |
| kwargs.update(do_sample=False, temperature=None, top_p=None, top_k=None) | |
| with torch.inference_mode(): | |
| out = self.model.generate(**enc, **kwargs) | |
| return out[:, prompt_length:].tolist() | |
| def _answer(self, ids): | |
| """Cut an answer at its end token and decode it.""" | |
| stop = next((i for i, t in enumerate(ids) if t in self.eos_ids), None) | |
| used = ids if stop is None else ids[: stop + 1] | |
| text = self.tokenizer.decode(used, skip_special_tokens=True) | |
| if "</think>" in text: | |
| text = text.split("</think>", 1)[1] | |
| return {"response": text.strip(), "response_tokens": len(used), "hit_token_limit": stop is None} | |
| def generate(self, texts): | |
| """Generate for a batch of rendered prompts. Returns one dict per prompt.""" | |
| if not self.thinking: | |
| return [self._answer(ids) for ids in self._generate_ids(texts, self.max_new_tokens, self.eos_ids)] | |
| # Step 1: reasoning, stopped at </think> or at the budget. | |
| stop_ids = self.eos_ids + ([self.think_end_id] if self.think_end_id is not None else []) | |
| think_runs = self._generate_ids(texts, self.think_budget, stop_ids) | |
| results, follow_up = [], [] | |
| for i, (text, ids) in enumerate(zip(texts, think_runs)): | |
| stop = next((j for j, t in enumerate(ids) if t in stop_ids), None) | |
| body = ids if stop is None else ids[:stop] | |
| closed = stop is not None and ids[stop] == self.think_end_id | |
| ended_inside = stop is not None and not closed # end-of-text before closing its reasoning | |
| raw = self.tokenizer.decode(body, skip_special_tokens=False) | |
| rec = { | |
| "thinking": self.tokenizer.decode(body, skip_special_tokens=True).strip(), | |
| "thinking_tokens": len(body), | |
| "thinking_closed": closed, | |
| "budget_forced": stop is None, | |
| "ended_inside_thinking": ended_inside, | |
| "response": "", "response_tokens": 0, "hit_token_limit": False, | |
| } | |
| results.append(rec) | |
| if not ended_inside: | |
| # Step 2 prompt: the reasoning, closed normally or with Qwen's early-exit phrase. | |
| follow_up.append((i, text + raw + (THINK_CLOSE if closed else QWEN_EARLY_EXIT))) | |
| # Step 2: the answer, batched over the rows that need one. | |
| if follow_up: | |
| answers = self._generate_ids([t for _, t in follow_up], self.max_new_tokens, self.eos_ids) | |
| for (i, _), ids in zip(follow_up, answers): | |
| results[i].update(self._answer(ids)) | |
| return results | |
| def is_oom(self, exc): | |
| return isinstance(exc, self.torch.cuda.OutOfMemoryError) | |
| def free(self): | |
| if self.torch.cuda.is_available(): | |
| self.torch.cuda.empty_cache() | |
| # ============================================================ | |
| # Helpers | |
| # ============================================================ | |
| def user_msg(text): | |
| return {"role": "user", "content": [{"type": "text", "text": text}]} | |
| def build_messages(language, turns): | |
| """turns: list of (role, text). The system prompt depends on the language.""" | |
| messages = [{"role": "system", "content": SYSTEM_PROMPTS[language]}] | |
| for role, text in turns: | |
| messages.append(user_msg(text) if role == "user" else {"role": "assistant", "content": text}) | |
| return messages | |
| def load_sheet(path): | |
| df = pd.read_excel(path, dtype=str).fillna("") | |
| df["ready_to_run"] = df["ready_to_run"].str.strip().str.lower() == "true" | |
| return df | |
| def read_done(out_path): | |
| done = {} | |
| if os.path.exists(out_path): | |
| with open(out_path, encoding="utf-8") as f: | |
| for line in f: | |
| line = line.strip() | |
| if line: | |
| rec = json.loads(line) | |
| done[rec["sample_id"]] = rec | |
| return done | |
| def append(out_path, records): | |
| os.makedirs(os.path.dirname(os.path.abspath(out_path)), exist_ok=True) | |
| with open(out_path, "a", encoding="utf-8") as f: | |
| for rec in records: | |
| f.write(json.dumps(rec, ensure_ascii=False) + "\n") | |
| f.flush() | |
| os.fsync(f.fileno()) | |
| def run_batches(gen, jobs, batch_size, out_path, meta, label): | |
| """ | |
| jobs: list of (row_dict, messages, extra_fields). Sorted longest first so | |
| an out-of-memory error shows up in the first batch, not the last. | |
| On OOM the batch size is halved and the batch retried. | |
| """ | |
| if not jobs: | |
| print(f"[{label}] nothing to run") | |
| return | |
| rendered = [(row, gen.render(msgs), extra) for row, msgs, extra in jobs] | |
| rendered.sort(key=lambda x: -gen.length(x[1])) | |
| total, i, bs, started = len(rendered), 0, batch_size, time.time() | |
| print(f"[{label}] {total} prompts, batch size {bs}") | |
| while i < total: | |
| chunk = rendered[i: i + bs] | |
| try: | |
| outs = gen.generate([text for _, text, _ in chunk]) | |
| except Exception as exc: | |
| if gen.is_oom(exc) and bs > 1: | |
| gen.free() | |
| bs = max(1, bs // 2) | |
| print(f"[{label}] out of memory; retrying with batch size {bs}") | |
| continue | |
| raise | |
| now = time.strftime("%Y-%m-%d %H:%M:%S") | |
| records = [] | |
| for (row, _, extra), out in zip(chunk, outs): | |
| rec = {k: row.get(k, "") for k in KEEP_COLUMNS} | |
| rec.update(extra) | |
| rec.update(out) | |
| rec.update(meta) | |
| rec.update({"time": now, "batch_size_used": bs}) | |
| records.append(rec) | |
| append(out_path, records) | |
| i += len(chunk) | |
| rate = (time.time() - started) / i | |
| print(f"[{label}] {i}/{total} done, ~{rate * (total - i) / 60:.1f} min left") | |
| # ============================================================ | |
| # Phases | |
| # ============================================================ | |
| def select(df, args): | |
| sel = df | |
| if getattr(args, "items", None): | |
| sel = sel[sel.item_id.isin(args.items.split(","))] | |
| if getattr(args, "languages", None): | |
| sel = sel[sel.language.isin(args.languages.split(","))] | |
| if getattr(args, "only_ready", False): | |
| sel = sel[sel.ready_to_run] | |
| return sel | |
| def check_system_prompts(languages): | |
| missing = sorted({l for l in languages if not SYSTEM_PROMPTS.get(l)}) | |
| if missing: | |
| raise SystemExit(f"Fill SYSTEM_PROMPTS for: {', '.join(missing)} (top of the script).") | |
| def phase_single(gen, df, args, meta, done): | |
| rows = df[df.record_type == "single_turn"] | |
| empty = rows.prompt.str.strip() == "" | |
| if empty.any(): | |
| print(f"[single] skipping {empty.sum()} rows with no prompt written yet") | |
| rows = rows[~empty] | |
| if getattr(args, "limit", None): | |
| rows = rows.head(args.limit) | |
| jobs = [] | |
| for row in rows.to_dict("records"): | |
| if row["sample_id"] in done: | |
| continue | |
| msgs = build_messages(row["language"], [("user", row["prompt"])]) | |
| jobs.append((row, msgs, {"system_prompt": SYSTEM_PROMPTS[row["language"]], "turn": 1})) | |
| print(f"[single] {len(rows)} selected, {len(rows) - len(jobs)} already done") | |
| run_batches(gen, jobs, args.batch_size, args.out, meta, "single") | |
| def phase_multi(gen, df, args, meta, done): | |
| rows = df[df.record_type != "single_turn"] | |
| jobs, waiting = [], [] | |
| for row in rows.to_dict("records"): | |
| if row["sample_id"] in done: | |
| continue | |
| ref = done.get(row["turn1_reference_sample_id"]) | |
| if ref is None: | |
| waiting.append(row["sample_id"]) | |
| continue | |
| if row["turn1_prompt"].strip() != ref["prompt"].strip(): | |
| print(f"[multi] warning: {row['sample_id']} turn-1 prompt differs from {ref['sample_id']}") | |
| turn1 = ref["response"] | |
| msgs = build_messages(row["language"], [ | |
| ("user", row["turn1_prompt"]), ("assistant", turn1), ("user", row["turn2_prompt"]), | |
| ]) | |
| jobs.append((row, msgs, { | |
| "system_prompt": SYSTEM_PROMPTS[row["language"]], | |
| "turn": 2, | |
| "turn1_response": turn1, | |
| "turn1_source_sample_id": ref["sample_id"], | |
| })) | |
| if waiting: | |
| print(f"[multi] {len(waiting)} rows skipped: their F1 reference hasn't been run yet") | |
| print(f"[multi] {len(rows)} selected, {len(jobs)} to run") | |
| run_batches(gen, jobs, args.batch_size, args.out, meta, "multi") | |
| def verify_batching(gen, df, args): | |
| rows = df[(df.record_type == "single_turn") & (df.prompt.str.strip() != "")] | |
| rows = rows.head(args.verify_batching).to_dict("records") | |
| check_system_prompts({r["language"] for r in rows}) | |
| texts = [gen.render(build_messages(r["language"], [("user", r["prompt"])])) for r in rows] | |
| one = [gen.generate([t])[0]["response"] for t in texts] | |
| many = [] | |
| for i in range(0, len(texts), args.batch_size): | |
| many += [o["response"] for o in gen.generate(texts[i: i + args.batch_size])] | |
| same = sum(a == b for a, b in zip(one, many)) | |
| print(f"\nBatch size 1 vs {args.batch_size}: {same}/{len(rows)} identical responses") | |
| for r, a, b in zip(rows, one, many): | |
| if a != b: | |
| print(f"\n--- {r['sample_id']}\n[bs=1] {a[:300]}\n[batched] {b[:300]}") | |
| def export_xlsx(sheet_path, out_path, xlsx_out): | |
| df = pd.read_excel(sheet_path, dtype=str).fillna("") | |
| done = read_done(out_path) | |
| resp = {k: v["response"] for k, v in done.items()} | |
| t1 = {k: v.get("turn1_response", "") for k, v in done.items()} | |
| df["model_response"] = df["sample_id"].map(resp).fillna("") | |
| pos = df.columns.get_loc("model_response") | |
| df.insert(pos, "turn1_response", df["sample_id"].map(t1).fillna("")) | |
| if any(v.get("thinking") for v in done.values()): | |
| think = {k: v.get("thinking", "") for k, v in done.items()} | |
| forced = {k: str(v.get("budget_forced", "")) for k, v in done.items()} | |
| pos = df.columns.get_loc("model_response") + 1 | |
| df.insert(pos, "model_thinking", df["sample_id"].map(think).fillna("")) | |
| df.insert(pos + 1, "budget_forced", df["sample_id"].map(forced).fillna("")) | |
| df.to_excel(xlsx_out, index=False) | |
| print(f"Wrote {xlsx_out} ({(df.model_response != '').sum()} responses)") | |
| def push(bucket, paths, prefix="runs"): | |
| try: | |
| from huggingface_hub import batch_bucket_files, create_bucket | |
| create_bucket(bucket, private=True, exist_ok=True) | |
| batch_bucket_files(bucket, add=[(p, f"{prefix}/{os.path.basename(p)}") for p in paths if p and os.path.exists(p)]) | |
| print(f"Pushed to hf://buckets/{bucket}/{prefix}/") | |
| except Exception as exc: | |
| print(f"Push failed ({type(exc).__name__}: {exc}); files are still saved locally.") | |
| # ============================================================ | |
| # Entry point for direct notebook executions | |
| # ============================================================ | |
| class ConfigArgs: | |
| def __init__(self, **kwargs): | |
| self.xlsx = kwargs.get("xlsx") | |
| self.out = kwargs.get("out", "results/results.jsonl") | |
| self.phase = kwargs.get("phase", "all") | |
| self.batch_size = kwargs.get("batch_size", 16) | |
| self.max_new_tokens = kwargs.get("max_new_tokens", 384) | |
| self.items = kwargs.get("items", None) | |
| self.languages = kwargs.get("languages", None) | |
| self.limit = kwargs.get("limit", None) | |
| self.only_ready = kwargs.get("only_ready", False) | |
| self.seed = kwargs.get("seed", 0) | |
| self.dtype = kwargs.get("dtype", "auto") # "auto", "bf16" or "fp16" | |
| self.thinking = kwargs.get("thinking", False) | |
| self.think_budget = kwargs.get("think_budget", 2048) | |
| self.verify_batching = kwargs.get("verify_batching", None) | |
| self.export_xlsx = kwargs.get("export_xlsx", None) | |
| self.bucket = kwargs.get("bucket", None) | |
| # The model stays loaded between calls, so a verify run followed by the full run | |
| # (or a resume after an error) doesn't reload 9 GB of weights each time. | |
| _GEN = None | |
| def get_generator(args): | |
| global _GEN | |
| if _GEN is not None and _GEN.dtype_choice == args.dtype: | |
| print(f"Reusing loaded model ({_GEN.info['dtype']} on {_GEN.info['device']})") | |
| else: | |
| if _GEN is not None: # dtype changed: drop the old copy before loading a new one | |
| del _GEN.model | |
| _GEN.free() | |
| _GEN = None | |
| _GEN = QwenGreedy(MODEL_ID, args.max_new_tokens, args.dtype) | |
| _GEN.max_new_tokens = args.max_new_tokens | |
| _GEN.thinking = args.thinking | |
| _GEN.think_budget = args.think_budget | |
| print(f"Thinking: {'ON, budget ' + str(args.think_budget) + ' tokens' if args.thinking else 'OFF'}") | |
| return _GEN | |
| def run_eval_notebook(config_dict): | |
| args = ConfigArgs(**config_dict) | |
| if not args.xlsx or not os.path.exists(args.xlsx): | |
| raise FileNotFoundError(f"Please upload your XLSX file or configure the correct path. Provided: {args.xlsx}") | |
| df = select(load_sheet(args.xlsx), args) | |
| not_ready = (~df.ready_to_run).sum() | |
| if not_ready and not args.only_ready: | |
| print(f"Note: {not_ready} selected rows are not marked ready_to_run; running them anyway.") | |
| from transformers import set_seed | |
| set_seed(args.seed) | |
| gen = get_generator(args) | |
| if args.verify_batching: | |
| if args.thinking: | |
| print("Batching check skipped: thinking mode samples, so outputs differ run to run anyway.") | |
| return | |
| verify_batching(gen, df, args) | |
| return | |
| # Never mix thinking and non-thinking results in one file (resume would skip | |
| # every row already answered in the other mode). | |
| existing = read_done(args.out) | |
| modes = {rec.get("thinking_mode") for rec in existing.values()} | |
| if existing and modes != {args.thinking}: | |
| raise SystemExit(f"{args.out} already holds results with thinking={sorted(modes)}. " | |
| f"Use a different 'out' file for thinking={args.thinking}.") | |
| check_system_prompts(set(df.language)) | |
| meta = { | |
| "run_id": uuid.uuid4().hex[:8], | |
| "thinking_mode": args.thinking, # not "thinking": that key holds the reasoning text | |
| "decoding": "sampling" if args.thinking else "greedy", | |
| "sampling": THINKING_SAMPLING if args.thinking else None, | |
| "think_budget": args.think_budget if args.thinking else None, | |
| "early_exit_phrase": QWEN_EARLY_EXIT.strip() if args.thinking else None, | |
| "max_new_tokens": args.max_new_tokens, | |
| "seed": args.seed, | |
| "python": platform.python_version(), | |
| **gen.info, | |
| } | |
| if args.phase in ("single", "all"): | |
| phase_single(gen, df, args, meta, read_done(args.out)) | |
| if args.phase in ("multi", "all"): | |
| phase_multi(gen, df, args, meta, read_done(args.out)) | |
| if args.export_xlsx: | |
| export_xlsx(args.xlsx, args.out, args.export_xlsx) | |
| if args.bucket: | |
| push(args.bucket, [args.out, args.export_xlsx]) | |
| base = { | |
| "xlsx": "data/run_input.xlsx", | |
| "out": "results_thinking.jsonl", | |
| "dtype": "auto", # fp16 on a T4; "bf16" if fp16 output looks broken | |
| "batch_size": 16, | |
| } | |
| run_eval_notebook({**base, "thinking": True, "think_budget": 2048, # 4) thinking run: its OWN files | |
| "out": "results_thinking.jsonl", | |
| "export_xlsx": "prompts_with_responses_thinking.xlsx"}) |
Xet Storage Details
- Size:
- 23.8 kB
- Xet hash:
- cdd37daecbcb85a0bb4ea0a027f4fa5feb97f879589ade60b9278f6eeea3e639
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.