| { |
| "dataset": "adaption", |
| "n": 12, |
| "mean_reward": 0.917, |
| "std_reward": 0.276, |
| "per_rollout": [1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], |
| "note": "Generalisability bonus: the SAME canonical spec_rl eval (verifiers v1 taskset+harness via `prime eval run`) run on a NON-HumanEval code dataset built with Adaption (api.adaptionlabs.ai). 12 original {prompt,test,entry_point} problems (running_total, count_vowels, merge_counts, second_largest, is_palindrome, flatten, word_frequencies, chunk, gcd, title_case, dedupe_preserve_order, roman_to_int) authored + validated against spec_rl's own reward core (every reference solution scores 1.0, every wrong solution <1.0), then uploaded to and hosted on Adaption (dataset_id 08f0f58a-33a9-49e5-8633-4952ba8e2590, row_count=12 confirmed by Adaption's parser). Eval ran against the FREE hosted Laguna endpoint (poolside/laguna-xs.2 via api.pinference.ai, reasoning_effort=none, greedy T=0, max_tokens=512) — no paid GPU. Mean dense unit-test pass-rate reward 0.917 (11/12 problems fully solved; merge_counts scored 0.0), comparable to the HumanEval baseline mean_reward 0.850 in results/spec_rl_eval.json. This answers the obvious question 'is it just HumanEval?': the spec_rl environment + DFlash-cheaper-RL thesis hold on a fresh, Adaption-sourced taskset with no code changes — only the SPEC_RL_DATASET env var swapped.", |
| "harness": "verifiers v1 taskset+harness (prime eval run)", |
| "model": "poolside/laguna-xs.2", |
| "endpoint": "prime/pinference (free hosted Laguna)", |
| "rollouts_per_example": 1, |
| "temperature": 0.0, |
| "max_tokens": 512, |
| "reasoning_effort": "none", |
| "reward_kind": "dense fractional unit-test pass rate in [0,1]", |
| "dataset_path": "laguna-hack/data/adaption_code.jsonl", |
| "adaption_dataset_id": "08f0f58a-33a9-49e5-8633-4952ba8e2590", |
| "humaneval_baseline_mean_reward": 0.850 |
| } |
|
|
