Buckets:
| {"grader_version":1,"suite_version":"0.1.0","release_id":"2026-09-18","as_of":"2026-09-18","license":"CC-BY-4.0","citation":"Jevals (jevals.com), release 2026-09-18, suite 0.1.0. CC-BY-4.0.","run_origin":"maintainer's laptop, UTC+3","gate_t":{"choice":0.96,"score":null,"noul":0.91},"benchmarks":[{"id":"banking77","version":"0.1.0","primitive":"choice","title":"Banking77","dataset":"mteb/banking77","config":"default","split":"test","hf_revision":"18072d2685ea682290f7b8924d94c62acc19c0b2","license":"CC-BY-4.0 (Banking77, PolyAI; mirror tagged MIT)","source_url":"https://huggingface.co/datasets/mteb/banking77","length_cap":6000,"n_items":300,"class_counts":{"activate_my_card":4,"age_limit":4,"apple_pay_or_google_pay":4,"atm_support":3,"automatic_top_up":4,"balance_not_updated_after_bank_transfer":4,"balance_not_updated_after_cheque_or_cash_deposit":4,"beneficiary_not_allowed":4,"cancel_transfer":4,"card_about_to_expire":4,"card_acceptance":4,"card_arrival":4,"card_delivery_estimate":4,"card_linking":4,"card_not_working":4,"card_payment_fee_charged":4,"card_payment_not_recognised":4,"card_payment_wrong_exchange_rate":4,"card_swallowed":4,"cash_withdrawal_charge":4,"cash_withdrawal_not_recognised":4,"change_pin":4,"compromised_card":4,"contactless_not_working":4,"country_support":4,"declined_card_payment":4,"declined_cash_withdrawal":4,"declined_transfer":4,"direct_debit_payment_not_recognised":4,"disposable_card_limits":4,"edit_personal_details":4,"exchange_charge":4,"exchange_rate":4,"exchange_via_app":3,"extra_charge_on_statement":4,"failed_transfer":4,"fiat_currency_support":4,"get_disposable_virtual_card":4,"get_physical_card":4,"getting_spare_card":4,"getting_virtual_card":4,"lost_or_stolen_card":4,"lost_or_stolen_phone":4,"order_physical_card":4,"passcode_forgotten":4,"pending_card_payment":4,"pending_cash_withdrawal":4,"pending_top_up":4,"pending_transfer":3,"pin_blocked":4,"receiving_money":4,"Refund_not_showing_up":4,"request_refund":4,"reverted_card_payment?":4,"supported_cards_and_currencies":4,"terminate_account":4,"top_up_by_bank_transfer_charge":4,"top_up_by_card_charge":4,"top_up_by_cash_or_cheque":4,"top_up_failed":4,"top_up_limits":4,"top_up_reverted":4,"topping_up_by_card":4,"transaction_charged_twice":4,"transfer_fee_charged":4,"transfer_into_account":4,"transfer_not_received_by_recipient":3,"transfer_timing":4,"unable_to_verify_identity":4,"verify_my_identity":4,"verify_source_of_funds":4,"verify_top_up":4,"virtual_card_not_working":4,"visa_or_mastercard":3,"why_verify_identity":3,"wrong_amount_of_cash_received":3,"wrong_exchange_rate_for_cash_withdrawal":3},"criteria_hash":"3d98e4b3e675d53f73b7b641d28cbc2552b47a574c102442bf683dad759e0174","n_options":77,"random_baseline":0.013,"label_source":"ground_truth"},{"id":"helpsteer2","version":"0.1.0","primitive":"score","title":"HelpSteer2 helpfulness","dataset":"nvidia/HelpSteer2","config":"default","split":"validation","hf_revision":"990b2711a36180dd19d9c94b8627844866f8982a","license":"CC-BY-4.0","source_url":"https://huggingface.co/datasets/nvidia/HelpSteer2","length_cap":6000,"n_items":300,"class_counts":{"0":23,"1":27,"2":35,"3":90,"4":125},"criteria_hash":"cd7113fe20542faa7e67f0ddf50076682e8101b9516e5fcc29fd2c1374426096","n_options":5,"random_baseline":0.2,"label_source":"ground_truth"},{"id":"pubmedqa","version":"0.1.0","primitive":"noul","title":"PubMedQA","dataset":"qiaojin/PubMedQA","config":"pqa_labeled","split":"train","hf_revision":"9001f2853fb87cab8d220904e0de81ac6973b318","license":"MIT","source_url":"https://huggingface.co/datasets/qiaojin/PubMedQA","length_cap":6000,"n_items":300,"class_counts":{"no":114,"yes":186},"criteria_hash":"f14633ff925f04eb85bfba47c030140831ce3e4901b1914e757d9cbaf980541a","n_options":2,"random_baseline":0.5,"label_source":"ground_truth"}],"rows":[{"slug":"gemini-3.8-flash","primitive":"choice","model_id":"google/gemini-3.8-flash","resolved_model":"google/gemini-3.8-flash","display_name":"Gemini 3.8 Flash","organization":"Google","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Google AI Studio","reasoning":"low (mandatory)","temperature":"provider default","verified":true,"decision_score":74.107,"ci_low":68.4397,"ci_high":79.7893,"rank_ub":1,"accuracy":0.846,"acc_ci_low":0.808,"acc_ci_high":0.882,"ece_10_bin":0.0319,"ece_ci":[0.0174,0.0679],"loss":0.2555,"cost_per_1k":1.367931,"price_asof":"2026-09-18","p50_ms":1636,"p95_ms":3938,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.03,"order_flip_rate":0.0867,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":0.1173,"accuracy_at_t":0.9943,"mass_at_conf_1":0,"per_task":[{"benchmark_id":"banking77","decision_score":74.107,"ci_low":68.4397,"ci_high":79.7893,"accuracy":0.846}],"run_ids":["gemini-3.8-flash__banking77__0.1.0"],"logs":["/data/runs/gemini-3.8-flash__banking77__0.1.0.jsonl"]},{"slug":"jev","primitive":"choice","model_id":"typesafe-ai/jev","resolved_model":"typesafe-ai/jev","display_name":"Jev","organization":"TypeSafe AI","interface":"native","adapter":"ai-sdk experimental_evaluate","adapter_version":"ai@7.0.106","probability_source":"native","probability_host":"Vercel AI Gateway","reasoning":"—","temperature":"—","verified":true,"decision_score":67.7806,"ci_low":61.3393,"ci_high":74.4909,"rank_ub":2,"accuracy":0.7967,"acc_ci_low":0.7547,"acc_ci_high":0.838,"ece_10_bin":0.0981,"ece_ci":[0.065,0.1355],"loss":0.318,"cost_per_1k":0.043276,"price_asof":"2026-09-18","p50_ms":467,"p95_ms":693,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["6d93f79a2310"],"repeat_flip_rate":0.0267,"order_flip_rate":0.1033,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":0.5947,"accuracy_at_t":0.9428,"mass_at_conf_1":0.4153,"per_task":[{"benchmark_id":"banking77","decision_score":67.7806,"ci_low":61.3393,"ci_high":74.4909,"accuracy":0.7967}],"run_ids":["jev__banking77__0.1.0"],"logs":["/data/runs/jev__banking77__0.1.0.jsonl"]},{"slug":"glm-5.3","primitive":"choice","model_id":"z-ai/glm-5.3","resolved_model":"z-ai/glm-5.3","display_name":"GLM-5.3","organization":"Z.ai","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Novita (fp8)","reasoning":"low (mandatory)","temperature":"provider default","verified":true,"decision_score":66.8264,"ci_low":61.9567,"ci_high":71.7196,"rank_ub":2,"accuracy":0.7887,"acc_ci_low":0.7493,"acc_ci_high":0.8267,"ece_10_bin":0.0616,"ece_ci":[0.0392,0.0964],"loss":0.3274,"cost_per_1k":0.718505,"price_asof":"2026-09-18","p50_ms":2110,"p95_ms":3186,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.0967,"order_flip_rate":0.2367,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":0.0033,"accuracy_at_t":1,"mass_at_conf_1":0,"per_task":[{"benchmark_id":"banking77","decision_score":66.8264,"ci_low":61.9567,"ci_high":71.7196,"accuracy":0.7887}],"run_ids":["glm-5.3__banking77__0.1.0"],"logs":["/data/runs/glm-5.3__banking77__0.1.0.jsonl"]},{"slug":"deepseek-v4.1-flash","primitive":"choice","model_id":"deepseek/deepseek-v4.1-flash","resolved_model":"deepseek/deepseek-v4.1-flash","display_name":"DeepSeek V4.1 Flash","organization":"DeepSeek","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · DeepSeek","reasoning":"none","temperature":"provider default","verified":true,"decision_score":63.9292,"ci_low":58.5065,"ci_high":69.6119,"rank_ub":4,"accuracy":0.7587,"acc_ci_low":0.7187,"acc_ci_high":0.8007,"ece_10_bin":0.0276,"ece_ci":[0.0204,0.0553],"loss":0.356,"cost_per_1k":0.138384,"price_asof":"2026-09-18","p50_ms":999,"p95_ms":1309,"n_items":300,"n_repeats":5,"n_answered":1498,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.1067,"order_flip_rate":0.302,"schema_valid_rate":0.9987,"refusal_rate":0,"coverage_at_t":0.148,"accuracy_at_t":0.991,"mass_at_conf_1":0.006,"per_task":[{"benchmark_id":"banking77","decision_score":63.9292,"ci_low":58.5065,"ci_high":69.6119,"accuracy":0.7587}],"run_ids":["deepseek-v4.1-flash__banking77__0.1.0"],"logs":["/data/runs/deepseek-v4.1-flash__banking77__0.1.0.jsonl"]},{"slug":"qwen3.8-flash","primitive":"choice","model_id":"qwen/qwen3.8-flash","resolved_model":"qwen/qwen3.8-flash","display_name":"Qwen3.8 Flash","organization":"Alibaba","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Alibaba","reasoning":"none","temperature":"provider default","verified":true,"decision_score":62.4389,"ci_low":55.9178,"ci_high":68.8961,"rank_ub":4,"accuracy":0.768,"acc_ci_low":0.726,"acc_ci_high":0.8087,"ece_10_bin":0.0951,"ece_ci":[0.0603,0.1322],"loss":0.3707,"cost_per_1k":0.122064,"price_asof":"2026-09-18","p50_ms":1713,"p95_ms":3105,"n_items":300,"n_repeats":5,"n_answered":1498,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["0401f0112fa8","4da1b6d6bc57"],"repeat_flip_rate":0.09,"order_flip_rate":0.2215,"schema_valid_rate":0.9987,"refusal_rate":0,"coverage_at_t":0.0227,"accuracy_at_t":1,"mass_at_conf_1":0,"per_task":[{"benchmark_id":"banking77","decision_score":62.4389,"ci_low":55.9178,"ci_high":68.8961,"accuracy":0.768}],"run_ids":["qwen3.8-flash__banking77__0.1.0"],"logs":["/data/runs/qwen3.8-flash__banking77__0.1.0.jsonl"]},{"slug":"mistral-medium-3.5","primitive":"choice","model_id":"mistralai/mistral-medium-3-5","resolved_model":"mistralai/mistral-medium-3-5","display_name":"Mistral Medium 3.5","organization":"Mistral AI","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Mistral","reasoning":"none","temperature":"provider default","verified":true,"decision_score":59.4547,"ci_low":54.2695,"ci_high":64.8301,"rank_ub":5,"accuracy":0.7447,"acc_ci_low":0.7013,"acc_ci_high":0.7887,"ece_10_bin":0.0807,"ece_ci":[0.059,0.1168],"loss":0.4002,"cost_per_1k":1.515244,"price_asof":"2026-09-18","p50_ms":936,"p95_ms":1444,"n_items":300,"n_repeats":5,"n_answered":1493,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.0234,"order_flip_rate":0.2218,"schema_valid_rate":0.9953,"refusal_rate":0,"coverage_at_t":0,"accuracy_at_t":null,"mass_at_conf_1":0,"per_task":[{"benchmark_id":"banking77","decision_score":59.4547,"ci_low":54.2695,"ci_high":64.8301,"accuracy":0.7447}],"run_ids":["mistral-medium-3.5__banking77__0.1.0"],"logs":["/data/runs/mistral-medium-3.5__banking77__0.1.0.jsonl"]},{"slug":"mercury-2.5","primitive":"choice","model_id":"inception/mercury-2.5","resolved_model":"inception/mercury-2.5","display_name":"Mercury 2.5","organization":"Inception","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Inception","reasoning":"none","temperature":"provider default","verified":true,"decision_score":54.0395,"ci_low":47.8178,"ci_high":60.6156,"rank_ub":7,"accuracy":0.7033,"acc_ci_low":0.6613,"acc_ci_high":0.7473,"ece_10_bin":0.1044,"ece_ci":[0.0658,0.1402],"loss":0.4536,"cost_per_1k":0.03606,"price_asof":"2026-09-18","p50_ms":638,"p95_ms":1478,"n_items":300,"n_repeats":5,"n_answered":1495,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.1376,"order_flip_rate":0.3615,"schema_valid_rate":0.9967,"refusal_rate":0,"coverage_at_t":0.0147,"accuracy_at_t":1,"mass_at_conf_1":0,"per_task":[{"benchmark_id":"banking77","decision_score":54.0395,"ci_low":47.8178,"ci_high":60.6156,"accuracy":0.7033}],"run_ids":["mercury-2.5__banking77__0.1.0"],"logs":["/data/runs/mercury-2.5__banking77__0.1.0.jsonl"]},{"slug":"label-prior","primitive":"choice","model_id":null,"resolved_model":null,"display_name":"Label prior","organization":"—","interface":"baseline","adapter":null,"adapter_version":null,"probability_source":"constant","probability_host":"—","reasoning":"—","temperature":"—","verified":true,"decision_score":0,"ci_low":0,"ci_high":0,"rank_ub":8,"accuracy":0.0133,"acc_ci_low":0.0033,"acc_ci_high":0.0267,"ece_10_bin":0,"ece_ci":[0,0.0133],"loss":0.9869,"cost_per_1k":0,"price_asof":null,"p50_ms":null,"p95_ms":null,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":null,"harness_commits":[],"repeat_flip_rate":0,"order_flip_rate":0,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":0,"accuracy_at_t":null,"mass_at_conf_1":0,"per_task":[{"benchmark_id":"banking77","decision_score":0,"ci_low":0,"ci_high":0,"accuracy":0.0133}],"run_ids":[],"logs":[]},{"slug":"jev","primitive":"score","model_id":"typesafe-ai/jev","resolved_model":"typesafe-ai/jev","display_name":"Jev","organization":"TypeSafe AI","interface":"native","adapter":"ai-sdk experimental_evaluate","adapter_version":"ai@7.0.106","probability_source":"native","probability_host":"Vercel AI Gateway","reasoning":"—","temperature":"—","verified":true,"decision_score":9.2034,"ci_low":-4.365,"ci_high":20.9655,"rank_ub":1,"accuracy":0.4127,"acc_ci_low":0.3593,"acc_ci_high":0.4687,"ece_10_bin":0.1966,"ece_ci":[0.1486,0.2502],"loss":0.1489,"cost_per_1k":0.036285,"price_asof":"2026-09-18","p50_ms":478,"p95_ms":670,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["6d93f79a2310"],"repeat_flip_rate":0.0233,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":null,"accuracy_at_t":null,"mass_at_conf_1":0.0033,"per_task":[{"benchmark_id":"helpsteer2","decision_score":9.2034,"ci_low":-4.365,"ci_high":20.9655,"accuracy":0.4127}],"run_ids":["jev__helpsteer2__0.1.0"],"logs":["/data/runs/jev__helpsteer2__0.1.0.jsonl"]},{"slug":"glm-5.3","primitive":"score","model_id":"z-ai/glm-5.3","resolved_model":"z-ai/glm-5.3","display_name":"GLM-5.3","organization":"Z.ai","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Novita (fp8)","reasoning":"low (mandatory)","temperature":"provider default","verified":true,"decision_score":7.7828,"ci_low":-4.5884,"ci_high":18.5647,"rank_ub":1,"accuracy":0.43,"acc_ci_low":0.386,"acc_ci_high":0.4787,"ece_10_bin":0.1287,"ece_ci":[0.0863,0.1711],"loss":0.1512,"cost_per_1k":0.784848,"price_asof":"2026-09-18","p50_ms":2211,"p95_ms":3631,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.3133,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":null,"accuracy_at_t":null,"mass_at_conf_1":0,"per_task":[{"benchmark_id":"helpsteer2","decision_score":7.7828,"ci_low":-4.5884,"ci_high":18.5647,"accuracy":0.43}],"run_ids":["glm-5.3__helpsteer2__0.1.0"],"logs":["/data/runs/glm-5.3__helpsteer2__0.1.0.jsonl"]},{"slug":"gemini-3.8-flash","primitive":"score","model_id":"google/gemini-3.8-flash","resolved_model":"google/gemini-3.8-flash","display_name":"Gemini 3.8 Flash","organization":"Google","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Google AI Studio","reasoning":"low (mandatory)","temperature":"provider default","verified":true,"decision_score":4.5909,"ci_low":-12.2314,"ci_high":18.9838,"rank_ub":1,"accuracy":0.424,"acc_ci_low":0.3693,"acc_ci_high":0.4753,"ece_10_bin":0.2259,"ece_ci":[0.1809,0.275],"loss":0.1564,"cost_per_1k":1.269273,"price_asof":"2026-09-18","p50_ms":1887,"p95_ms":3979,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.1167,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":null,"accuracy_at_t":null,"mass_at_conf_1":0.0007,"per_task":[{"benchmark_id":"helpsteer2","decision_score":4.5909,"ci_low":-12.2314,"ci_high":18.9838,"accuracy":0.424}],"run_ids":["gemini-3.8-flash__helpsteer2__0.1.0"],"logs":["/data/runs/gemini-3.8-flash__helpsteer2__0.1.0.jsonl"]},{"slug":"label-prior","primitive":"score","model_id":null,"resolved_model":null,"display_name":"Label prior","organization":"—","interface":"baseline","adapter":null,"adapter_version":null,"probability_source":"constant","probability_host":"—","reasoning":"—","temperature":"—","verified":true,"decision_score":0,"ci_low":0,"ci_high":0,"rank_ub":1,"accuracy":0.4167,"acc_ci_low":0.36,"acc_ci_high":0.47,"ece_10_bin":0,"ece_ci":[0,0.0633],"loss":0.1639,"cost_per_1k":0,"price_asof":null,"p50_ms":null,"p95_ms":null,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":null,"harness_commits":[],"repeat_flip_rate":0,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":null,"accuracy_at_t":null,"mass_at_conf_1":0,"per_task":[{"benchmark_id":"helpsteer2","decision_score":0,"ci_low":0,"ci_high":0,"accuracy":0.4167}],"run_ids":[],"logs":[]},{"slug":"qwen3.8-flash","primitive":"score","model_id":"qwen/qwen3.8-flash","resolved_model":"qwen/qwen3.8-flash","display_name":"Qwen3.8 Flash","organization":"Alibaba","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Alibaba","reasoning":"none","temperature":"provider default","verified":true,"decision_score":-1.3701,"ci_low":-15.4058,"ci_high":11.5108,"rank_ub":3,"accuracy":0.3607,"acc_ci_low":0.3147,"acc_ci_high":0.4073,"ece_10_bin":0.2577,"ece_ci":[0.2134,0.3019],"loss":0.1662,"cost_per_1k":0.125698,"price_asof":"2026-09-18","p50_ms":1422,"p95_ms":2638,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.26,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":null,"accuracy_at_t":null,"mass_at_conf_1":0,"per_task":[{"benchmark_id":"helpsteer2","decision_score":-1.3701,"ci_low":-15.4058,"ci_high":11.5108,"accuracy":0.3607}],"run_ids":["qwen3.8-flash__helpsteer2__0.1.0"],"logs":["/data/runs/qwen3.8-flash__helpsteer2__0.1.0.jsonl"]},{"slug":"mercury-2.5","primitive":"score","model_id":"inception/mercury-2.5","resolved_model":"inception/mercury-2.5","display_name":"Mercury 2.5","organization":"Inception","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Inception","reasoning":"none","temperature":"provider default","verified":true,"decision_score":-5.5457,"ci_low":-16.2223,"ci_high":3.9934,"rank_ub":3,"accuracy":0.4173,"acc_ci_low":0.3727,"acc_ci_high":0.462,"ece_10_bin":0.3148,"ece_ci":[0.2731,0.3571],"loss":0.173,"cost_per_1k":0.034331,"price_asof":"2026-09-18","p50_ms":618,"p95_ms":1165,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.3867,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":null,"accuracy_at_t":null,"mass_at_conf_1":0.0007,"per_task":[{"benchmark_id":"helpsteer2","decision_score":-5.5457,"ci_low":-16.2223,"ci_high":3.9934,"accuracy":0.4173}],"run_ids":["mercury-2.5__helpsteer2__0.1.0"],"logs":["/data/runs/mercury-2.5__helpsteer2__0.1.0.jsonl"]},{"slug":"mistral-medium-3.5","primitive":"score","model_id":"mistralai/mistral-medium-3-5","resolved_model":"mistralai/mistral-medium-3-5","display_name":"Mistral Medium 3.5","organization":"Mistral AI","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Mistral","reasoning":"none","temperature":"provider default","verified":true,"decision_score":-13.7288,"ci_low":-28.2641,"ci_high":-0.1455,"rank_ub":5,"accuracy":0.4387,"acc_ci_low":0.3847,"acc_ci_high":0.4927,"ece_10_bin":0.3361,"ece_ci":[0.2816,0.3889],"loss":0.1865,"cost_per_1k":1.325036,"price_asof":"2026-09-18","p50_ms":706,"p95_ms":1178,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.0367,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":null,"accuracy_at_t":null,"mass_at_conf_1":0.0213,"per_task":[{"benchmark_id":"helpsteer2","decision_score":-13.7288,"ci_low":-28.2641,"ci_high":-0.1455,"accuracy":0.4387}],"run_ids":["mistral-medium-3.5__helpsteer2__0.1.0"],"logs":["/data/runs/mistral-medium-3.5__helpsteer2__0.1.0.jsonl"]},{"slug":"deepseek-v4.1-flash","primitive":"score","model_id":"deepseek/deepseek-v4.1-flash","resolved_model":"deepseek/deepseek-v4.1-flash","display_name":"DeepSeek V4.1 Flash","organization":"DeepSeek","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · DeepSeek","reasoning":"none","temperature":"provider default","verified":true,"decision_score":-18.9946,"ci_low":-36.0158,"ci_high":-3.3185,"rank_ub":7,"accuracy":0.3467,"acc_ci_low":0.3047,"acc_ci_high":0.3867,"ece_10_bin":0.3348,"ece_ci":[0.2957,0.3735],"loss":0.1951,"cost_per_1k":0.125297,"price_asof":"2026-09-18","p50_ms":912,"p95_ms":1203,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.3433,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":null,"accuracy_at_t":null,"mass_at_conf_1":0.0153,"per_task":[{"benchmark_id":"helpsteer2","decision_score":-18.9946,"ci_low":-36.0158,"ci_high":-3.3185,"accuracy":0.3467}],"run_ids":["deepseek-v4.1-flash__helpsteer2__0.1.0"],"logs":["/data/runs/deepseek-v4.1-flash__helpsteer2__0.1.0.jsonl"]},{"slug":"gemini-3.8-flash","primitive":"noul","model_id":"google/gemini-3.8-flash","resolved_model":"google/gemini-3.8-flash","display_name":"Gemini 3.8 Flash","organization":"Google","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Google AI Studio","reasoning":"low (mandatory)","temperature":"provider default","verified":true,"decision_score":72.983,"ci_low":62.0659,"ci_high":82.2398,"rank_ub":1,"accuracy":0.9253,"acc_ci_low":0.8927,"acc_ci_high":0.9527,"ece_10_bin":0.0198,"ece_ci":[0.007,0.0469],"loss":0.1273,"cost_per_1k":0.803439,"price_asof":"2026-09-18","p50_ms":1854,"p95_ms":5538,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.0167,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":0.7307,"accuracy_at_t":0.9708,"mass_at_conf_1":0.008,"per_task":[{"benchmark_id":"pubmedqa","decision_score":72.983,"ci_low":62.0659,"ci_high":82.2398,"accuracy":0.9253}],"run_ids":["gemini-3.8-flash__pubmedqa__0.1.0"],"logs":["/data/runs/gemini-3.8-flash__pubmedqa__0.1.0.jsonl"]},{"slug":"jev","primitive":"noul","model_id":"typesafe-ai/jev","resolved_model":"typesafe-ai/jev","display_name":"Jev","organization":"TypeSafe AI","interface":"native","adapter":"ai-sdk experimental_evaluate","adapter_version":"ai@7.0.106","probability_source":"native","probability_host":"Vercel AI Gateway","reasoning":"—","temperature":"—","verified":true,"decision_score":69.0274,"ci_low":59.7741,"ci_high":76.5459,"rank_ub":1,"accuracy":0.9127,"acc_ci_low":0.8787,"acc_ci_high":0.9433,"ece_10_bin":0.0504,"ece_ci":[0.0321,0.0812],"loss":0.1459,"cost_per_1k":0.028535,"price_asof":"2026-09-18","p50_ms":438,"p95_ms":653,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["6d93f79a2310"],"repeat_flip_rate":0,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":0.49,"accuracy_at_t":0.9864,"mass_at_conf_1":0,"per_task":[{"benchmark_id":"pubmedqa","decision_score":69.0274,"ci_low":59.7741,"ci_high":76.5459,"accuracy":0.9127}],"run_ids":["jev__pubmedqa__0.1.0"],"logs":["/data/runs/jev__pubmedqa__0.1.0.jsonl"]},{"slug":"qwen3.8-flash","primitive":"noul","model_id":"qwen/qwen3.8-flash","resolved_model":"qwen/qwen3.8-flash","display_name":"Qwen3.8 Flash","organization":"Alibaba","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Alibaba","reasoning":"none","temperature":"provider default","verified":true,"decision_score":62.4422,"ci_low":50.905,"ci_high":72.5479,"rank_ub":2,"accuracy":0.8967,"acc_ci_low":0.8627,"acc_ci_high":0.9267,"ece_10_bin":0.0255,"ece_ci":[0.0081,0.0587],"loss":0.177,"cost_per_1k":0.08613,"price_asof":"2026-09-18","p50_ms":1088,"p95_ms":3515,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.0467,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":0.7227,"accuracy_at_t":0.9391,"mass_at_conf_1":0.0067,"per_task":[{"benchmark_id":"pubmedqa","decision_score":62.4422,"ci_low":50.905,"ci_high":72.5479,"accuracy":0.8967}],"run_ids":["qwen3.8-flash__pubmedqa__0.1.0"],"logs":["/data/runs/qwen3.8-flash__pubmedqa__0.1.0.jsonl"]},{"slug":"glm-5.3","primitive":"noul","model_id":"z-ai/glm-5.3","resolved_model":"z-ai/glm-5.3","display_name":"GLM-5.3","organization":"Z.ai","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Novita (fp8)","reasoning":"low (mandatory)","temperature":"provider default","verified":true,"decision_score":60.6328,"ci_low":49.9596,"ci_high":69.8189,"rank_ub":3,"accuracy":0.8867,"acc_ci_low":0.8533,"acc_ci_high":0.916,"ece_10_bin":0.0202,"ece_ci":[0.0096,0.0458],"loss":0.1855,"cost_per_1k":0.512414,"price_asof":"2026-09-18","p50_ms":1757,"p95_ms":2879,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.0533,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":0.45,"accuracy_at_t":0.9674,"mass_at_conf_1":0.0027,"per_task":[{"benchmark_id":"pubmedqa","decision_score":60.6328,"ci_low":49.9596,"ci_high":69.8189,"accuracy":0.8867}],"run_ids":["glm-5.3__pubmedqa__0.1.0"],"logs":["/data/runs/glm-5.3__pubmedqa__0.1.0.jsonl"]},{"slug":"mistral-medium-3.5","primitive":"noul","model_id":"mistralai/mistral-medium-3-5","resolved_model":"mistralai/mistral-medium-3-5","display_name":"Mistral Medium 3.5","organization":"Mistral AI","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Mistral","reasoning":"none","temperature":"provider default","verified":true,"decision_score":58.0327,"ci_low":45.3414,"ci_high":69.8002,"rank_ub":3,"accuracy":0.888,"acc_ci_low":0.8527,"acc_ci_high":0.922,"ece_10_bin":0.0523,"ece_ci":[0.0249,0.0895],"loss":0.1978,"cost_per_1k":0.874716,"price_asof":"2026-09-18","p50_ms":533,"p95_ms":917,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.01,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":0.6507,"accuracy_at_t":0.9549,"mass_at_conf_1":0.142,"per_task":[{"benchmark_id":"pubmedqa","decision_score":58.0327,"ci_low":45.3414,"ci_high":69.8002,"accuracy":0.888}],"run_ids":["mistral-medium-3.5__pubmedqa__0.1.0"],"logs":["/data/runs/mistral-medium-3.5__pubmedqa__0.1.0.jsonl"]},{"slug":"mercury-2.5","primitive":"noul","model_id":"inception/mercury-2.5","resolved_model":"inception/mercury-2.5","display_name":"Mercury 2.5","organization":"Inception","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · Inception","reasoning":"none","temperature":"provider default","verified":true,"decision_score":55.6583,"ci_low":44.8114,"ci_high":65.4167,"rank_ub":3,"accuracy":0.8713,"acc_ci_low":0.84,"acc_ci_high":0.9013,"ece_10_bin":0.0468,"ece_ci":[0.0249,0.0779],"loss":0.2089,"cost_per_1k":0.022595,"price_asof":"2026-09-18","p50_ms":584,"p95_ms":1036,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.1,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":0.7247,"accuracy_at_t":0.9246,"mass_at_conf_1":0.0107,"per_task":[{"benchmark_id":"pubmedqa","decision_score":55.6583,"ci_low":44.8114,"ci_high":65.4167,"accuracy":0.8713}],"run_ids":["mercury-2.5__pubmedqa__0.1.0"],"logs":["/data/runs/mercury-2.5__pubmedqa__0.1.0.jsonl"]},{"slug":"deepseek-v4.1-flash","primitive":"noul","model_id":"deepseek/deepseek-v4.1-flash","resolved_model":"deepseek/deepseek-v4.1-flash","display_name":"DeepSeek V4.1 Flash","organization":"DeepSeek","interface":"adapter","adapter":"jevals-llm","adapter_version":"0383a0e3e592","probability_source":"verbalized","probability_host":"OpenRouter · DeepSeek","reasoning":"none","temperature":"provider default","verified":true,"decision_score":47.5001,"ci_low":34.7497,"ci_high":59.0847,"rank_ub":7,"accuracy":0.8367,"acc_ci_low":0.7987,"acc_ci_high":0.8727,"ece_10_bin":0.0616,"ece_ci":[0.0318,0.0968],"loss":0.2474,"cost_per_1k":0.078411,"price_asof":"2026-09-18","p50_ms":838,"p95_ms":1123,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":["2026-09-18","2026-09-18"],"harness_commits":["4da1b6d6bc57"],"repeat_flip_rate":0.0667,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":0.5413,"accuracy_at_t":0.9298,"mass_at_conf_1":0.1947,"per_task":[{"benchmark_id":"pubmedqa","decision_score":47.5001,"ci_low":34.7497,"ci_high":59.0847,"accuracy":0.8367}],"run_ids":["deepseek-v4.1-flash__pubmedqa__0.1.0"],"logs":["/data/runs/deepseek-v4.1-flash__pubmedqa__0.1.0.jsonl"]},{"slug":"label-prior","primitive":"noul","model_id":null,"resolved_model":null,"display_name":"Label prior","organization":"—","interface":"baseline","adapter":null,"adapter_version":null,"probability_source":"constant","probability_host":"—","reasoning":"—","temperature":"—","verified":true,"decision_score":0,"ci_low":0,"ci_high":0,"rank_ub":8,"accuracy":0.62,"acc_ci_low":0.5667,"acc_ci_high":0.6733,"ece_10_bin":0,"ece_ci":[0,0.0633],"loss":0.4712,"cost_per_1k":0,"price_asof":null,"p50_ms":null,"p95_ms":null,"n_items":300,"n_repeats":5,"n_answered":1500,"run_window":null,"harness_commits":[],"repeat_flip_rate":0,"order_flip_rate":null,"schema_valid_rate":1,"refusal_rate":0,"coverage_at_t":0,"accuracy_at_t":null,"mass_at_conf_1":0,"per_task":[{"benchmark_id":"pubmedqa","decision_score":0,"ci_low":0,"ci_high":0,"accuracy":0.62}],"run_ids":[],"logs":[]}]} |
Xet Storage Details
- Size:
- 31 kB
- Xet hash:
- e6ba1979996858204787d8518d8b5b24e920178be93a4d88562e0add07907e2b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.