Buckets:
| """RL candidates the model has never trained on: tasksource train-split questions outside the V2 pool, dev, eval and | |
| calibration sets (by id and by content), weighted toward reasoning-heavy sources (logic, rules, spatial, agent safety). | |
| python rl_fresh.py --out data/rl_fresh_pool.jsonl # CPU only; rl_select.py --fresh then screens them | |
| OpenDecisions and LocalLLaMA have no training split (every question is in the eval), so nothing here comes from | |
| them: RL on these fresh questions is tested for transfer on those two untouched sets. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import collections | |
| import json | |
| import random | |
| import re | |
| import pandas as pd | |
| from transformers import AutoTokenizer | |
| from common import read_jsonl, render, write_jsonl | |
| from prep_data import TS, fetch, finish, key, noul_options | |
| # multi-step reasoning: logic and rule following, spatial and temporal relations, agent and contract judgements | |
| REASONING = re.compile(r"FOL-nli|LogicNLI|PARARULE|SpaRTUN|agent_action_safety|babi_nli/|causal-reasoning|clutrr|conceptrules|" | |
| r"contract-nli|folio|lifecycle-entailment|logical-entailment|logical-fallacy|logiqa|lsat|math_qa|" | |
| r"matres|monotonicity|naturallogic|probability_words_nli/reasoning|proofwriter|reclor|ruletaker|" | |
| r"spartqa|stepgame|strategy-qa|temporal-nli|winogrande") | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--raw", default="data/raw") | |
| ap.add_argument("--pool", default="data/train_pool.jsonl") | |
| ap.add_argument("--held", default="data/eval.jsonl,data/calib.jsonl,data/dev.jsonl") | |
| ap.add_argument("--out", default="data/rl_fresh_pool.jsonl") | |
| ap.add_argument("--n-reasoning", type=int, default=3000) | |
| ap.add_argument("--n-general", type=int, default=3000) | |
| ap.add_argument("--cap-reasoning", type=int, default=80, help="per source") | |
| ap.add_argument("--cap-general", type=int, default=5, help="per source, for breadth") | |
| ap.add_argument("--max-prompt-tokens", type=int, default=4096, help="keeps RL rounds short") | |
| ap.add_argument("--model", default="Qwen/Qwen3.5-4B") | |
| ap.add_argument("--seed", type=int, default=41) | |
| args = ap.parse_args() | |
| rng = random.Random(args.seed) | |
| counts = collections.Counter() | |
| pool = read_jsonl(args.pool) | |
| held_rows = [x for p in args.held.split(",") for x in read_jsonl(p)] | |
| seen_ids = {x["id"] for x in pool} | {x["id"] for x in held_rows} | |
| seen_keys = {key(x) for x in pool} | {key(x) for x in held_rows} | |
| tdf = pd.concat([pd.read_parquet(p, columns=["id", "state", "kind", "options", "target", "question", "source"]) | |
| for p in fetch(TS, "data/train", args.raw)], ignore_index=True) | |
| tdf = tdf[~tdf["source"].str.startswith("procedural-typed-decisions/")] | |
| tdf = tdf.sample(frac=1.0, random_state=rng.randrange(1 << 30)) | |
| tok = AutoTokenizer.from_pretrained(args.model) | |
| want = {True: args.n_reasoning, False: args.n_general} | |
| cap = {True: args.cap_reasoning, False: args.cap_general} | |
| per, out = collections.Counter(), {True: [], False: []} | |
| for row in tdf.itertuples(index=False): | |
| r = bool(REASONING.search(row.source)) | |
| if len(out[r]) >= want[r] or per[row.source] >= cap[r] or f"ts:{row.id}" in seen_ids: | |
| continue | |
| if row.kind == "noul": | |
| p = float(row.target[0]) | |
| opts, target = noul_options(None), [p, 1 - p] | |
| else: | |
| opts = ([{"id": str(i), "desc": o} for i, o in enumerate(row.options)] if row.kind == "score" | |
| else [{"id": None, "desc": o} for o in row.options]) | |
| target = list(row.target) | |
| item = finish({"id": f"ts:{row.id}", "set": "rl_fresh_ts", "family": row.source, "level": None, "tier": None, | |
| "qtype": row.kind, "evidence": row.state, "criterion": row.question, "options": opts, "target": target, | |
| "reasoning_source": r}, counts, "fresh") | |
| if item is None: | |
| continue | |
| if key(item) in seen_keys: | |
| counts["fresh:content_seen_in_pool_or_held"] += 1 | |
| continue | |
| n = len(tok(render(tok, item, think=True), add_special_tokens=False)["input_ids"]) | |
| if n > args.max_prompt_tokens: | |
| counts["fresh:prompt_too_long"] += 1 | |
| continue | |
| item["prompt_tokens"] = n | |
| seen_keys.add(key(item)) | |
| out[r].append(item) | |
| per[row.source] += 1 | |
| if all(len(out[k]) >= want[k] for k in out): | |
| break | |
| rows = out[True] + out[False] | |
| write_jsonl(args.out, rows) | |
| print(json.dumps({"reasoning": len(out[True]), "general": len(out[False]), | |
| "reasoning_sources": len({x["family"] for x in out[True]}), "general_sources": len({x["family"] for x in out[False]}), | |
| **counts}), flush=True) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 4.94 kB
- Xet hash:
- fb9aceeb87b35300868d21835fb01bc6751efb4d8684f183beb75c56a4fa7946
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.