| { | |
| "version": "qev-0.1.1-final-comparison", | |
| "official_dataset_revision": "d51d993547ad8355b1c25157fbc1fea0649e8ffa", | |
| "official_dataset_sha256": "4881baae1cfd752311a58064cb2095c0457ee2c0b8914d1cfe2590e19933e7b3", | |
| "sources": { | |
| "ag_news": { | |
| "repo": "fancyzhx/ag_news", | |
| "revision": "eb185aade064a813bc0b7f42de02595523103ca4", | |
| "file": "data/test-00000-of-00001.parquet", | |
| "criteria": { | |
| "world": "world news and international politics", | |
| "sports": "sports", | |
| "business": "business and economy", | |
| "sci_tech": "science and technology" | |
| }, | |
| "field": "article", | |
| "question": "topic", | |
| "instructions": "What is the topic of `article`?", | |
| "laya_training_status": "In training according to upstream bench_apps.py", | |
| "download_sha256": "71de87ec66bc5737752a2502204dfa6d7fe9856ade3ea444dc6317789a4f13fb", | |
| "selected": "first 400 test rows", | |
| "class_counts": { | |
| "2": 72, | |
| "3": 102, | |
| "1": 123, | |
| "0": 103 | |
| } | |
| }, | |
| "emotion": { | |
| "repo": "dair-ai/emotion", | |
| "revision": "cab853a1dbdf4c42c2b3ef2173804746df8825fe", | |
| "file": "split/test-00000-of-00001.parquet", | |
| "criteria": { | |
| "sadness": "sadness", | |
| "joy": "joy", | |
| "love": "love", | |
| "anger": "anger", | |
| "fear": "fear", | |
| "surprise": "surprise" | |
| }, | |
| "field": "text", | |
| "question": "emotion", | |
| "instructions": "Which emotion is most strongly expressed in `text`?", | |
| "laya_training_status": "Held out according to upstream bench_apps.py", | |
| "download_sha256": "6f8407fa1ca9c310f55781f082ed73812f6551e8dda2c61973123a121869245b", | |
| "selected": "first 400 test rows", | |
| "class_counts": { | |
| "0": 119, | |
| "1": 118, | |
| "4": 52, | |
| "3": 64, | |
| "2": 37, | |
| "5": 10 | |
| } | |
| } | |
| }, | |
| "requests_sha256": "585850d575d9dbdf943683936dda7e07b9e5525b60875906926e9915c557d1d6", | |
| "questions": { | |
| "typed_decisions": 2000, | |
| "ag_news": 400, | |
| "emotion": 400 | |
| }, | |
| "script_sha256": "83bcc709680109b9c9f4c86a84e7c7659f34a30408de90af2695a6d1924edd0b", | |
| "adaptation_data_audit": { | |
| "corpora": 15, | |
| "rows_including_repeated_stages": 251457, | |
| "source_counts": { | |
| "veyra-original-generator": 5472, | |
| "AI-Lab-Makerere/beans": 8312, | |
| "veyra-policy-generator-v3": 8384, | |
| "veyra-evidence-interventions-v6": 2400, | |
| "veyra-policy-refresh-v8": 20294, | |
| "PolyAI/banking77": 24132, | |
| "veyra/conditional-probability": 480, | |
| "stanfordnlp/snli": 23220, | |
| "vminhkhoi/trashnet": 16683, | |
| "veyra-workflow-v12": 70560, | |
| "LocalLLaMA/typed-decisions": 49200, | |
| "veyra-workspace-calibration-v12": 4080, | |
| "workflow-depth-policy-fit-data-v12": 4080, | |
| "workflow-depth-policy-validation-data-v12": 4080, | |
| "workflow-cohort-policy-validation-1-v12": 2040, | |
| "workflow-cohort-policy-validation-2-v12": 2040, | |
| "workflow-recovery-policy-validation-1-v13": 2040, | |
| "workflow-recovery-policy-validation-2-v13": 2040, | |
| "workflow-recovery-fresh-final-v13": 1920 | |
| }, | |
| "normalized_substring_matches": [], | |
| "scope": "Released adaptation snapshots; unknown backbone pretraining overlap" | |
| }, | |
| "selection": "All 2000 typed decisions; first 400 AG News and Emotion test rows", | |
| "upstream_alignment": "Application sample count, row selection and instructions follow LAYA bench_apps.py. Emotion null descriptions become label names for both SDKs. State JSON is identical.", | |
| "evaluation": { | |
| "models": [ | |
| "qev", | |
| "laya-base", | |
| "laya-specialist" | |
| ], | |
| "one_question_per_call": true, | |
| "laya_max_len": 1024, | |
| "qev_max_tokens": 2048, | |
| "warmups_per_suite": 3, | |
| "torch_threads": 4, | |
| "device": "Single RTX 4060 Ti 8 GB, models run sequentially", | |
| "no_parameter_or_temperature_or_prompt_selection": true, | |
| "headline_accuracy": "All questions including abstained answers, explicit hard gold", | |
| "probability_metrics": "Same frozen decision_metrics.py for all models", | |
| "zero_shot": "No task-specific QEV adaptation or examples; not pretraining-clean", | |
| "typed_decisions": "Both QEV and specialist adapted; reused regression benchmark" | |
| } | |
| } | |