| { |
| "validation_date": "2026-08-12", |
| "method": "fresh CUDA forward on every local video with release QPreferScorer", |
| "environment": { |
| "gpu": "NVIDIA RTX 6000 Ada Generation", |
| "torch": "2.8.0+cu128", |
| "transformers": "5.3.0", |
| "peft": "0.18.1", |
| "qwen_vl_utils": "0.0.14", |
| "decord": "0.6.0" |
| }, |
| "fresh_forward_workload": { |
| "qprefer_bench_t2v_pairs": 275, |
| "qprefer_bench_i2v_pairs": 274, |
| "qprefer_bench_near_pairs": 294, |
| "human_tie_pairs": 210, |
| "human_forced_choice_pairs": 300, |
| "videogen_rewardbench_unique_videos": 4691, |
| "videoscore_bench_v2_videos": 500 |
| }, |
| "all_runs_zero_errors": true, |
| "all_pair_margins_and_ood_raw_scores_exact_against_legacy_d2": true, |
| "paper_number_checks": { |
| "main_macro_with_ties": { |
| "observed": 0.539625, |
| "paper": 0.54, |
| "passed": true |
| }, |
| "main_macro_without_ties": { |
| "observed": 0.73095, |
| "paper": 0.731, |
| "passed": true |
| }, |
| "near_vq_with_ties": { |
| "observed": 0.6397, |
| "paper": 0.64, |
| "passed": true |
| }, |
| "near_vq_without_ties": { |
| "observed": 0.66, |
| "paper": 0.66, |
| "passed": true |
| }, |
| "near_ta_with_ties": { |
| "observed": 0.6775, |
| "paper": 0.677, |
| "passed": true |
| }, |
| "near_ta_without_ties": { |
| "observed": 0.6196, |
| "paper": 0.62, |
| "passed": true |
| }, |
| "forced_human_macro": { |
| "observed": 0.615, |
| "paper": 0.615, |
| "passed": true |
| }, |
| "tie_human_macro_with_ties": { |
| "observed": 0.5833, |
| "paper": 0.583, |
| "passed": true |
| }, |
| "tie_human_macro_without_ties": { |
| "observed": 0.6327, |
| "paper": 0.633, |
| "passed": true |
| }, |
| "videogen_vq_with_ties_percent": { |
| "observed": 47.45581358484584, |
| "paper": 47.5, |
| "passed": true |
| }, |
| "videogen_vq_without_ties_percent": { |
| "observed": 63.66993217784477, |
| "paper": 63.7, |
| "passed": true |
| }, |
| "videogen_ta_with_ties_percent": { |
| "observed": 43.12039312039312, |
| "paper": 43.1, |
| "passed": true |
| }, |
| "videogen_ta_without_ties_percent": { |
| "observed": 58.307865529998395, |
| "paper": 58.3, |
| "passed": true |
| }, |
| "videogen_overall_with_ties_percent": { |
| "observed": 51.480204494114844, |
| "paper": 51.5, |
| "passed": true |
| }, |
| "videogen_overall_without_ties_percent": { |
| "observed": 61.86598085440777, |
| "paper": 61.9, |
| "passed": true |
| }, |
| "vs2_vq_plcc_percent": { |
| "observed": 49.093721381630886, |
| "paper": 49.1, |
| "passed": true |
| }, |
| "vs2_ta_plcc_percent": { |
| "observed": 24.09226069401914, |
| "paper": 24.1, |
| "passed": true |
| }, |
| "vs2_average_plcc_percent": { |
| "observed": 36.592991037825016, |
| "paper": 36.6, |
| "passed": true |
| }, |
| "main_t2v_vq_with": { |
| "observed": 0.5933, |
| "paper": 0.593, |
| "passed": true |
| }, |
| "main_t2v_vq_without": { |
| "observed": 0.7796, |
| "paper": 0.78, |
| "passed": true |
| }, |
| "main_t2v_ta_with": { |
| "observed": 0.4647, |
| "paper": 0.465, |
| "passed": true |
| }, |
| "main_t2v_ta_without": { |
| "observed": 0.625, |
| "paper": 0.625, |
| "passed": true |
| }, |
| "main_i2v_vq_with": { |
| "observed": 0.5758, |
| "paper": 0.576, |
| "passed": true |
| }, |
| "main_i2v_vq_without": { |
| "observed": 0.7791, |
| "paper": 0.779, |
| "passed": true |
| }, |
| "main_i2v_ta_with": { |
| "observed": 0.5247, |
| "paper": 0.525, |
| "passed": true |
| }, |
| "main_i2v_ta_without": { |
| "observed": 0.7401, |
| "paper": 0.74, |
| "passed": true |
| }, |
| "near_macro_with": { |
| "observed": 0.6586, |
| "paper": 0.659, |
| "passed": true |
| }, |
| "near_macro_without": { |
| "observed": 0.6398, |
| "paper": 0.64, |
| "passed": true |
| }, |
| "human_forced_t2v_vq": { |
| "observed": 0.67, |
| "paper": 0.67, |
| "passed": true |
| }, |
| "human_forced_t2v_ta": { |
| "observed": 0.63, |
| "paper": 0.63, |
| "passed": true |
| }, |
| "human_forced_i2v_vq": { |
| "observed": 0.74, |
| "paper": 0.74, |
| "passed": true |
| }, |
| "human_forced_i2v_ta": { |
| "observed": 0.68, |
| "paper": 0.68, |
| "passed": true |
| }, |
| "human_forced_near_vq": { |
| "observed": 0.44, |
| "paper": 0.44, |
| "passed": true |
| }, |
| "human_forced_near_ta": { |
| "observed": 0.53, |
| "paper": 0.53, |
| "passed": true |
| }, |
| "human_forced_all_vq": { |
| "observed": 0.6167, |
| "paper": 0.617, |
| "passed": true |
| }, |
| "human_forced_all_ta": { |
| "observed": 0.6133, |
| "paper": 0.613, |
| "passed": true |
| }, |
| "human_tie_t2v_vq": { |
| "observed": 0.6857, |
| "paper": 0.686, |
| "passed": true |
| }, |
| "human_tie_t2v_ta": { |
| "observed": 0.4857, |
| "paper": 0.486, |
| "passed": true |
| }, |
| "human_tie_i2v_vq": { |
| "observed": 0.7286, |
| "paper": 0.729, |
| "passed": true |
| }, |
| "human_tie_i2v_ta": { |
| "observed": 0.5571, |
| "paper": 0.557, |
| "passed": true |
| }, |
| "human_tie_near_vq": { |
| "observed": 0.4286, |
| "paper": 0.429, |
| "passed": true |
| }, |
| "human_tie_near_ta": { |
| "observed": 0.6143, |
| "paper": 0.614, |
| "passed": true |
| } |
| } |
| } |
|
|
