Buckets:
| """Shared paths, rendering and data loading for think-decide. | |
| Rendering uses jev-dllm's own `bench_diff_yesno.decision_input` unchanged (the renderer S1 was trained | |
| and evaluated with). Thought slots are spliced in afterwards: the ids for "\nNotes: " plus T mask ids are | |
| inserted right after the last answer mask. The mask is a special token, so the token boundaries are | |
| exactly those of rendering `user + "\nNotes: " + mask*T` (checked in tests). | |
| """ | |
| import json | |
| import os | |
| import sys | |
| from collections import Counter | |
| JEV = "/workspace/ext/jev-dllm" | |
| for p in (JEV + "/support/dllm_stub", JEV, JEV + "/research/scripts"): | |
| if p not in sys.path: | |
| sys.path.insert(0, p) | |
| os.environ.setdefault("USE_TF", "0") | |
| os.environ.setdefault("USE_TORCH", "1") | |
| os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") | |
| S1_PATH = "/workspace/models/shared-yesno-s1" | |
| DATA = "/workspace/data/training-legacy/s1/canonical" | |
| CORE = "/workspace/data/core-v1" | |
| PROC = "/workspace/data/procedural/canonical" | |
| NOTES_TEXT = "\nNotes: " | |
| # Pre-registered suite groups (plan section 5), fixed before any result. | |
| GROUPS = { | |
| "reasoning": ["jevbench_public_original", "jevbench_public_hard", "nimble_vitaminc-dev", | |
| "nimble_multinli", "contractnli", "nimble_pubmedqa"], | |
| "lookup": ["nimble_massive-en-US", "nimble_massive-de-DE", "nimble_civil_comments", | |
| "nimble_aegis2", "nimble_paws"], | |
| "other": ["jevbench_public_easy", "jabr_v2", "nimble_squad2", "nimble_helpsteer2", | |
| "nimble_summeval-relevance", "nimble_summeval-consistency"], | |
| } | |
| # jev-dllm's Core14 drops these (training-seen MultiNLI / Aegis2, dev-contact PAWS); sensitivity only. | |
| CORE14_EXCLUDE = ["nimble_multinli", "nimble_aegis2", "nimble_paws"] | |
| def load_env(): | |
| path = "/workspace/.env" | |
| if os.path.exists(path): | |
| for line in open(path): | |
| if "=" in line and not line.startswith("#"): | |
| k, v = line.strip().split("=", 1) | |
| os.environ.setdefault(k, v) | |
| def tokenizer_and_mapping(path=S1_PATH): | |
| from transformers import AutoTokenizer | |
| tok = AutoTokenizer.from_pretrained(path, trust_remote_code=True, local_files_only=True) | |
| mapping = {} | |
| for text in ("Yes", "No"): | |
| ids = tok.encode(text, add_special_tokens=False) | |
| assert len(ids) == 1 and tok.decode(ids) == text | |
| mapping[text] = ids[0] | |
| mask_id = tok.mask_token_id | |
| assert mask_id == 151669, mask_id | |
| mask_text = tok.convert_ids_to_tokens(mask_id) | |
| return tok, mapping, mask_id, mask_text | |
| class Renderer: | |
| """S1 renderer + optional T thought slots.""" | |
| def __init__(self, tok, mask_id, mask_text, n_think=0): | |
| from bench_diff_yesno import decision_input | |
| self._render = decision_input | |
| self.tok, self.mask_id, self.mask_text, self.n_think = tok, mask_id, mask_text, n_think | |
| self.notes_ids = tok.encode(NOTES_TEXT, add_special_tokens=False) | |
| def __call__(self, state, qdef, max_length=10**9): | |
| ids, positions, options, prompt = self._render(self.tok, state, qdef, self.mask_id, self.mask_text, 10**9) | |
| think = [] | |
| if self.n_think: | |
| cut = positions[-1] + 1 | |
| start = cut + len(self.notes_ids) | |
| think = list(range(start, start + self.n_think)) | |
| ids = ids[:cut] + self.notes_ids + [self.mask_id] * self.n_think + ids[cut:] | |
| return ids, positions, think, options | |
| def load_canonical(path, renderer, max_length, limit=0): | |
| """Train/dev canonical rows -> items. Overlong rows are dropped and counted (never truncated).""" | |
| from laya.common import QTYPES | |
| items, dropped = [], Counter() | |
| for line in open(path, encoding="utf-8"): | |
| row = json.loads(line) | |
| assert row["schema_version"] == "shared_yesno_data_v1" | |
| (qid, original), = row["questions"].items() | |
| prim = original["type"] | |
| assert prim in QTYPES | |
| q = {k: original[k] for k in ("type", "instructions", "criteria") if k in original} | |
| crit = q.get("criteria") | |
| keys = list(crit) if prim == "choice" else ([str(i) for i in range(len(crit))] if prim == "score" | |
| else ["false", "true"]) | |
| probs = row["gold"][qid]["probabilities"] | |
| target = [float(probs[k]) for k in keys] | |
| ids, pos, think, options = renderer(row["state"], q) | |
| if len(ids) > max_length: | |
| dropped[row["source"]["name"]] += 1 | |
| continue | |
| assert len(options) == len(target) and len(pos) == (1 if prim == "noul" else len(target)) | |
| items.append(dict(ids=ids, pos=pos, think=think, target=target, primitive=prim, | |
| source=row["source"]["name"], kind=row["gold"][qid]["kind"], | |
| case_id=row["case_id"], view_id=row["view_id"], group_id=row["group_id"], | |
| level=row.get("level"))) | |
| if limit and len(items) >= limit: | |
| break | |
| return items, dict(dropped) | |
| def load_core(renderer, max_length, suites=None, limit_per_suite=0): | |
| from bench_local import to_internal | |
| from laya.common import render_options, serialize_state | |
| targets = {} | |
| for line in open(os.path.join(CORE, "targets.jsonl")): | |
| t = json.loads(line) | |
| targets[t["id"]] = t | |
| items, unsupported, per = [], [], Counter() | |
| for line in open(os.path.join(CORE, "inputs.jsonl")): | |
| r = json.loads(line) | |
| if suites and r["suite"] not in suites: | |
| continue | |
| if limit_per_suite and per[r["suite"]] >= limit_per_suite: | |
| continue | |
| per[r["suite"]] += 1 | |
| t = targets[r["id"]] | |
| base = dict(id=r["id"], suite=r["suite"], primitive=r["qdef"]["type"], gold_idx=t["gold_idx"], | |
| soft=t["soft"], gold_score=t["gold_score"]) | |
| q = to_internal(r["qdef"]) | |
| texts = [serialize_state(r["state"]), q["ins"]] + render_options(q) | |
| if any(renderer.mask_text in x for x in texts): # same check as jev-dllm's evaluator | |
| unsupported.append(dict(base, status="unsupported_source_mask")) | |
| continue | |
| ids, pos, think, options = renderer(r["state"], r["qdef"]) | |
| if len(ids) > max_length: | |
| unsupported.append(dict(base, status="unsupported_length", length=len(ids))) | |
| continue | |
| items.append(dict(base, ids=ids, pos=pos, think=think, n_options=len(options))) | |
| return items, unsupported | |
| def write_jsonl(path, rows): | |
| with open(path, "w") as f: | |
| for r in rows: | |
| f.write(json.dumps(r) + "\n") | |
Xet Storage Details
- Size:
- 6.57 kB
- Xet hash:
- 07907d755993b082d269eba019b28112ee47fcfd7334dcb691ecb540f47c2146
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.