Spaces:
Running
Running
p-video-2-pro
#10
by minette-kaunismaki - opened
- README.md +10 -1
- app.py +109 -7
- data/i2v-pro.csv +21 -0
- data/p-video-2-leaderboard.csv +0 -17
- data/t2v-pro.csv +29 -0
- model_display.py +30 -0
- ui.py +25 -10
README.md
CHANGED
|
@@ -14,11 +14,20 @@ tags:
|
|
| 14 |
|
| 15 |
# P-Bench
|
| 16 |
|
| 17 |
-
Compare text-to-image
|
|
|
|
| 18 |
Gradio dashboard: leaderboards, Pareto plots, and side-by-side samples.
|
| 19 |
|
| 20 |
The live Space is [PrunaAI/P-Bench](https://huggingface.co/spaces/PrunaAI/P-Bench).
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
## Run locally
|
| 23 |
|
| 24 |
From the repo root:
|
|
|
|
| 14 |
|
| 15 |
# P-Bench
|
| 16 |
|
| 17 |
+
Compare text-to-image, text-to-video, image-to-video, and video-to-video
|
| 18 |
+
models on quality, speed, and price. This repo is the
|
| 19 |
Gradio dashboard: leaderboards, Pareto plots, and side-by-side samples.
|
| 20 |
|
| 21 |
The live Space is [PrunaAI/P-Bench](https://huggingface.co/spaces/PrunaAI/P-Bench).
|
| 22 |
|
| 23 |
+
The current P-Video-2 Pro evaluation snapshots are
|
| 24 |
+
`data/t2v-pro.csv` (text-to-video) and
|
| 25 |
+
`data/i2v-pro.csv` (image-to-video). Both feed the leaderboards and
|
| 26 |
+
price/time Pareto plots with Datapoint and Rapidata Elo. Variants without
|
| 27 |
+
either Elo stay in the raw CSVs but do not appear in ranked views. Image-to-video
|
| 28 |
+
sample media is not included in these exports.
|
| 29 |
+
Rapidata Elo comes directly from these exports.
|
| 30 |
+
|
| 31 |
## Run locally
|
| 32 |
|
| 33 |
From the repo root:
|
app.py
CHANGED
|
@@ -1252,6 +1252,23 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
|
|
| 1252 |
color: var(--pruna-text-muted) !important;
|
| 1253 |
font-size: 12px;
|
| 1254 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1255 |
.ranking-table .text-cell,
|
| 1256 |
.prose .ranking-table .text-cell {
|
| 1257 |
color: var(--pruna-text-muted) !important;
|
|
@@ -2351,7 +2368,7 @@ def _is_pruna_video_model(series):
|
|
| 2351 |
|
| 2352 |
|
| 2353 |
def _apply_video_timings(df, *, replace_displayed_time=False):
|
| 2354 |
-
"""Mix
|
| 2355 |
fal = df.get("Time / Output Video Second (s)")
|
| 2356 |
execution = df.get("Execution Time / Output Video Second (s)")
|
| 2357 |
if fal is None:
|
|
@@ -2411,8 +2428,8 @@ def load_video_editing_dataframe(path):
|
|
| 2411 |
return df.reset_index(drop=True)
|
| 2412 |
|
| 2413 |
|
| 2414 |
-
def
|
| 2415 |
-
"""Load
|
| 2416 |
df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
|
| 2417 |
model_column = "model_id" if "model_id" in df.columns else "Model"
|
| 2418 |
df = df[~df[model_column].astype(str).str.casefold().str.startswith("agnes")].copy()
|
|
@@ -2446,7 +2463,7 @@ def load_text_to_video_dataframe(path):
|
|
| 2446 |
"Price / Second of Video (USD)",
|
| 2447 |
],
|
| 2448 |
)
|
| 2449 |
-
# Pruna rows use model execution time;
|
| 2450 |
df = _apply_video_timings(df, replace_displayed_time=True)
|
| 2451 |
df = df.drop(
|
| 2452 |
columns=["Execution Time / Output Video Second (s)"],
|
|
@@ -2462,6 +2479,10 @@ def load_text_to_video_dataframe(path):
|
|
| 2462 |
return df.reset_index(drop=True)
|
| 2463 |
|
| 2464 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2465 |
df = load_oneig_dataframe(oneig_path)
|
| 2466 |
|
| 2467 |
oneig_metric_columns = [
|
|
@@ -2534,8 +2555,12 @@ video_path = _resolve_data_path(
|
|
| 2534 |
space_root.parent / "video-editing-leaderboard.csv",
|
| 2535 |
)
|
| 2536 |
text_to_video_path = _resolve_data_path(
|
| 2537 |
-
data_dir / "
|
| 2538 |
-
space_root.parent / "
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2539 |
)
|
| 2540 |
|
| 2541 |
qwen_df = load_qwen_combined_dataframe(qwen_path)
|
|
@@ -2543,6 +2568,7 @@ aa_df = load_artificial_analysis_dataframe(aa_path)
|
|
| 2543 |
arena_df = load_arena_ai_dataframe(arena_path)
|
| 2544 |
video_df = load_video_editing_dataframe(video_path)
|
| 2545 |
text_to_video_df = load_text_to_video_dataframe(text_to_video_path)
|
|
|
|
| 2546 |
qwen_display_columns = [
|
| 2547 |
col
|
| 2548 |
for col in [
|
|
@@ -2666,6 +2692,9 @@ video_metric_ids = _metric_ids_for(video_df, ["datapoint_elo"])
|
|
| 2666 |
text_to_video_metric_ids = _metric_ids_for(
|
| 2667 |
text_to_video_df, ["datapoint_elo", "rapidata_elo"]
|
| 2668 |
)
|
|
|
|
|
|
|
|
|
|
| 2669 |
|
| 2670 |
datasets = [
|
| 2671 |
{
|
|
@@ -2678,11 +2707,27 @@ datasets = [
|
|
| 2678 |
"note": (
|
| 2679 |
"Datapoint Elo and Rapidata Elo from pairwise text-to-video "
|
| 2680 |
"preference. Price is USD per second of output video. Time per "
|
| 2681 |
-
"second of video is
|
| 2682 |
"model execution time."
|
| 2683 |
),
|
| 2684 |
"samples": text_to_video_samples,
|
| 2685 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2686 |
{
|
| 2687 |
"id": "video_editing",
|
| 2688 |
"name": "Pruna Internal Video-Edit Benchmark",
|
|
@@ -2981,6 +3026,63 @@ custom_head = """
|
|
| 2981 |
})();
|
| 2982 |
</script>
|
| 2983 |
<script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2984 |
(() => {
|
| 2985 |
if (window.__inferbenchChipScrollBound) return;
|
| 2986 |
window.__inferbenchChipScrollBound = true;
|
|
|
|
| 1252 |
color: var(--pruna-text-muted) !important;
|
| 1253 |
font-size: 12px;
|
| 1254 |
}
|
| 1255 |
+
.model-name-tooltip {
|
| 1256 |
+
position: fixed;
|
| 1257 |
+
z-index: 10000;
|
| 1258 |
+
max-width: min(440px, calc(100vw - 24px));
|
| 1259 |
+
box-sizing: border-box;
|
| 1260 |
+
padding: 10px 14px;
|
| 1261 |
+
border: 1px solid var(--pruna-border);
|
| 1262 |
+
border-radius: 8px;
|
| 1263 |
+
background: var(--pruna-bg-elevated);
|
| 1264 |
+
color: var(--pruna-text-primary);
|
| 1265 |
+
box-shadow: 0 8px 24px rgba(0, 0, 0, 0.3);
|
| 1266 |
+
font: 13px/1.5 var(--pruna-font);
|
| 1267 |
+
white-space: normal;
|
| 1268 |
+
overflow-wrap: anywhere;
|
| 1269 |
+
pointer-events: none;
|
| 1270 |
+
}
|
| 1271 |
+
.model-name-tooltip[hidden] { display: none !important; }
|
| 1272 |
.ranking-table .text-cell,
|
| 1273 |
.prose .ranking-table .text-cell {
|
| 1274 |
color: var(--pruna-text-muted) !important;
|
|
|
|
| 2368 |
|
| 2369 |
|
| 2370 |
def _apply_video_timings(df, *, replace_displayed_time=False):
|
| 2371 |
+
"""Mix provider wall time with Pruna model execution time."""
|
| 2372 |
fal = df.get("Time / Output Video Second (s)")
|
| 2373 |
execution = df.get("Execution Time / Output Video Second (s)")
|
| 2374 |
if fal is None:
|
|
|
|
| 2428 |
return df.reset_index(drop=True)
|
| 2429 |
|
| 2430 |
|
| 2431 |
+
def load_video_generation_dataframe(path):
|
| 2432 |
+
"""Load a text-to-video or image-to-video evaluation snapshot."""
|
| 2433 |
df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
|
| 2434 |
model_column = "model_id" if "model_id" in df.columns else "Model"
|
| 2435 |
df = df[~df[model_column].astype(str).str.casefold().str.startswith("agnes")].copy()
|
|
|
|
| 2463 |
"Price / Second of Video (USD)",
|
| 2464 |
],
|
| 2465 |
)
|
| 2466 |
+
# Pruna rows use model execution time; other rows keep provider wall time.
|
| 2467 |
df = _apply_video_timings(df, replace_displayed_time=True)
|
| 2468 |
df = df.drop(
|
| 2469 |
columns=["Execution Time / Output Video Second (s)"],
|
|
|
|
| 2479 |
return df.reset_index(drop=True)
|
| 2480 |
|
| 2481 |
|
| 2482 |
+
def load_text_to_video_dataframe(path):
|
| 2483 |
+
return load_video_generation_dataframe(path)
|
| 2484 |
+
|
| 2485 |
+
|
| 2486 |
df = load_oneig_dataframe(oneig_path)
|
| 2487 |
|
| 2488 |
oneig_metric_columns = [
|
|
|
|
| 2555 |
space_root.parent / "video-editing-leaderboard.csv",
|
| 2556 |
)
|
| 2557 |
text_to_video_path = _resolve_data_path(
|
| 2558 |
+
data_dir / "t2v-pro.csv",
|
| 2559 |
+
space_root.parent / "t2v-pro.csv",
|
| 2560 |
+
)
|
| 2561 |
+
image_to_video_path = _resolve_data_path(
|
| 2562 |
+
data_dir / "i2v-pro.csv",
|
| 2563 |
+
space_root.parent / "i2v-pro.csv",
|
| 2564 |
)
|
| 2565 |
|
| 2566 |
qwen_df = load_qwen_combined_dataframe(qwen_path)
|
|
|
|
| 2568 |
arena_df = load_arena_ai_dataframe(arena_path)
|
| 2569 |
video_df = load_video_editing_dataframe(video_path)
|
| 2570 |
text_to_video_df = load_text_to_video_dataframe(text_to_video_path)
|
| 2571 |
+
image_to_video_df = load_video_generation_dataframe(image_to_video_path)
|
| 2572 |
qwen_display_columns = [
|
| 2573 |
col
|
| 2574 |
for col in [
|
|
|
|
| 2692 |
text_to_video_metric_ids = _metric_ids_for(
|
| 2693 |
text_to_video_df, ["datapoint_elo", "rapidata_elo"]
|
| 2694 |
)
|
| 2695 |
+
image_to_video_metric_ids = _metric_ids_for(
|
| 2696 |
+
image_to_video_df, ["datapoint_elo", "rapidata_elo"]
|
| 2697 |
+
)
|
| 2698 |
|
| 2699 |
datasets = [
|
| 2700 |
{
|
|
|
|
| 2707 |
"note": (
|
| 2708 |
"Datapoint Elo and Rapidata Elo from pairwise text-to-video "
|
| 2709 |
"preference. Price is USD per second of output video. Time per "
|
| 2710 |
+
"second of video is provider wall time, except Pruna models which use "
|
| 2711 |
"model execution time."
|
| 2712 |
),
|
| 2713 |
"samples": text_to_video_samples,
|
| 2714 |
},
|
| 2715 |
+
{
|
| 2716 |
+
"id": "image_to_video",
|
| 2717 |
+
"name": "Image-to-Video Benchmark",
|
| 2718 |
+
"modality": "image_to_video",
|
| 2719 |
+
"data": image_to_video_df,
|
| 2720 |
+
"columns": text_to_video_display_columns,
|
| 2721 |
+
"metric_ids": image_to_video_metric_ids,
|
| 2722 |
+
"note": (
|
| 2723 |
+
"Datapoint Elo and Rapidata Elo from pairwise image-to-video "
|
| 2724 |
+
"preference, comparing P-Video-2 Pro variants with other models. "
|
| 2725 |
+
"Price is USD per second of output video. Time per second of "
|
| 2726 |
+
"video is provider wall time, except Pruna models which use "
|
| 2727 |
+
"model execution time."
|
| 2728 |
+
),
|
| 2729 |
+
"samples": None,
|
| 2730 |
+
},
|
| 2731 |
{
|
| 2732 |
"id": "video_editing",
|
| 2733 |
"name": "Pruna Internal Video-Edit Benchmark",
|
|
|
|
| 3026 |
})();
|
| 3027 |
</script>
|
| 3028 |
<script>
|
| 3029 |
+
(() => {
|
| 3030 |
+
if (window.__inferbenchModelTooltipBound) return;
|
| 3031 |
+
window.__inferbenchModelTooltipBound = true;
|
| 3032 |
+
let tooltip;
|
| 3033 |
+
let active;
|
| 3034 |
+
let previousDescription;
|
| 3035 |
+
|
| 3036 |
+
const hide = () => {
|
| 3037 |
+
if (tooltip) tooltip.hidden = true;
|
| 3038 |
+
if (active) {
|
| 3039 |
+
if (previousDescription === null) active.removeAttribute("aria-describedby");
|
| 3040 |
+
else active.setAttribute("aria-describedby", previousDescription);
|
| 3041 |
+
}
|
| 3042 |
+
active = null;
|
| 3043 |
+
};
|
| 3044 |
+
const show = (event) => {
|
| 3045 |
+
const target = event.target.closest?.(
|
| 3046 |
+
".ranking-table tbody .model-cell, .view-filters .filter-chips .token span"
|
| 3047 |
+
);
|
| 3048 |
+
if (!target || target === active) return;
|
| 3049 |
+
hide();
|
| 3050 |
+
const text = target.querySelector("strong") || target;
|
| 3051 |
+
if (text.scrollWidth <= text.clientWidth) return;
|
| 3052 |
+
if (!tooltip) {
|
| 3053 |
+
tooltip = document.createElement("div");
|
| 3054 |
+
tooltip.id = "model-name-tooltip";
|
| 3055 |
+
tooltip.className = "model-name-tooltip";
|
| 3056 |
+
tooltip.setAttribute("role", "tooltip");
|
| 3057 |
+
document.body.appendChild(tooltip);
|
| 3058 |
+
}
|
| 3059 |
+
tooltip.textContent = target.dataset.modelName || text.textContent.trim();
|
| 3060 |
+
tooltip.hidden = false;
|
| 3061 |
+
active = target;
|
| 3062 |
+
previousDescription = target.getAttribute("aria-describedby");
|
| 3063 |
+
target.setAttribute("aria-describedby", [previousDescription, tooltip.id].filter(Boolean).join(" "));
|
| 3064 |
+
const rect = target.getBoundingClientRect();
|
| 3065 |
+
const width = tooltip.offsetWidth;
|
| 3066 |
+
const height = tooltip.offsetHeight;
|
| 3067 |
+
tooltip.style.left = Math.max(12, Math.min(rect.left, window.innerWidth - width - 12)) + "px";
|
| 3068 |
+
const top = rect.top >= height + 12 ? rect.top - height - 8 : rect.bottom + 8;
|
| 3069 |
+
tooltip.style.top = Math.max(12, Math.min(top, window.innerHeight - height - 12)) + "px";
|
| 3070 |
+
};
|
| 3071 |
+
document.addEventListener("pointerover", show);
|
| 3072 |
+
document.addEventListener("focusin", show);
|
| 3073 |
+
for (const eventName of ["pointerout", "focusout"]) {
|
| 3074 |
+
document.addEventListener(eventName, (event) => {
|
| 3075 |
+
if (active && active.contains(event.target) && !active.contains(event.relatedTarget)) hide();
|
| 3076 |
+
});
|
| 3077 |
+
}
|
| 3078 |
+
document.addEventListener("scroll", hide, true);
|
| 3079 |
+
window.addEventListener("resize", hide);
|
| 3080 |
+
document.addEventListener("keydown", (event) => {
|
| 3081 |
+
if (event.key === "Escape") hide();
|
| 3082 |
+
});
|
| 3083 |
+
})();
|
| 3084 |
+
</script>
|
| 3085 |
+
<script>
|
| 3086 |
(() => {
|
| 3087 |
if (window.__inferbenchChipScrollBound) return;
|
| 3088 |
window.__inferbenchChipScrollBound = true;
|
data/i2v-pro.csv
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
task,model_id,display_name,wandb_run_ids,wandb_project,provider,endpoint,resolution,output_duration_s,n_generations,min_generation_s,p20_generation_s,median_generation_s,mean_generation_s,generation_s_per_output_video_s,median_generation_s_per_output_video_s,model_execution_s_per_output_video_s,model_execution_source,predict_s_per_output_video_s,list_price_usd_per_second,yaml_price_usd_per_second,currently_billed_usd_per_second,flat_price_usd_per_request,price_usd_per_second,price_usd_per_clip,price_usd_per_5s,price_config,price_note,disabled_expansion_model_execution_s_per_output_video_s,disabled_expansion_timing_source,datapoint_elo,rapidata_elo
|
| 2 |
+
i2v,p_video_2_pro__mode_cost__prompt_upsampler_off__resolution_480p,"P-Video 2 Pro (cost mode, 480p)",lqcey8fm,p-video-i2v,replicate,prunaai/p-video-2-pro,480p,5.0,10,9.646036516000096,11.469126884000797,14.284900057999948,13.203384606500368,2.6406769213000736,2.8569800115999895,0.95356,replicate model_execution_time_s,2.8225635115,0.01,0.01,0.01,0.0,0.01,0.05,0.05,model/remote/text+image_video/p_video_2_pro/replicate_480p_cost_off.yaml,,,,961.0,984.0
|
| 3 |
+
i2v,p_video_2_pro__mode_speed__prompt_upsampler_turbo__resolution_480p,"P-Video 2 Pro (speed, upsampler turbo, 480p)",uflwzoyr,p-video-i2v,replicate,prunaai/p-video-2-pro,480p,5.0,100,7.607446426998649,10.114996741200594,12.339616352997837,12.90726687451992,2.5814533749039836,2.4679232705995675,0.307928,replicate model_execution_time_s,2.4315828523,0.02,0.02,0.02,0.0,0.02,0.1,0.1,model/remote/text+image_video/p_video_2_pro/replicate_480p_speed_turbo.yaml,,,,961.0,984.0
|
| 4 |
+
i2v,minimax_h3_max_turbo__prompt_expansion_mode_balanced__resolution_480p,"MiniMax H3 Max Turbo (expansion balanced, 480p)",rynohmot,p-video-i2v,fal,minimax/h3-max-turbo/image-to-video,480p,5.0,85,2.0626198340032715,2.676248207790195,3.3828403819934465,3.5529157923407824,0.7105831584681565,0.6765680763986893,0.6701127952941176,fal request duration,,0.025,0.0125,0.0125,0.0,0.025,0.125,0.125,model/remote/text+image_video/minimax_h3_max_turbo/fal_480p_balanced.yaml,fal list; fal still bills 50% promo $0.0125/s (page says promo ended 2026-09-30),0.53,estimated,978.0,938.0
|
| 5 |
+
i2v,p_video_2_pro__mode_cost__prompt_upsampler_off__resolution_768p,"P-Video 2 Pro (cost mode, 768p)",rn66usgo,p-video-i2v,replicate,prunaai/p-video-2-pro,768p,5.0,10,18.36896738299947,20.285163481000563,21.051096533999953,28.224230830200213,5.644846166040042,4.210219306799991,2.15268,replicate model_execution_time_s,4.0450694081,0.025,0.025,0.025,0.0,0.025,0.125,0.125,model/remote/text+image_video/p_video_2_pro/replicate_768p_cost_off.yaml,,,,994.0,995.0
|
| 6 |
+
i2v,agnes_video_2_5,Agnes Video 2.5 (720p),9mk5by2e,p-video-i2v,agnes,agnes-video-2.5,720p,5.0,85,48.16999681999732,72.84911508619844,73.43243030199665,86.38921110764632,17.277842221529266,14.686486060399329,,,,0.025,0.025,0.025,0.0,0.025,0.125,0.125,model/remote/text+image_video/agnes_video_2_5/agnes.yaml,,,,994.0,1009.0
|
| 7 |
+
i2v,p_video_2_pro__mode_speed__prompt_upsampler_turbo__resolution_768p,"P-Video 2 Pro (speed, upsampler turbo, 768p)",tpm41r5n,p-video-i2v,replicate,prunaai/p-video-2-pro,768p,5.0,100,9.887984333006898,12.503273962804816,13.168237036006758,16.083959509319683,3.2167919018639366,2.6336474072013516,0.600798,replicate model_execution_time_s,2.5795859222999997,0.035,0.035,0.035,0.0,0.035,0.17500000000000002,0.17500000000000002,model/remote/text+image_video/p_video_2_pro/replicate_768p_speed_turbo.yaml,,,,994.0,995.0
|
| 8 |
+
i2v,minimax_h3_max_turbo__prompt_expansion_mode_disabled__resolution_768p,"MiniMax H3 Max Turbo (expansion disabled, 768p)",pjwfymx5,p-video-i2v,fal,minimax/h3-max-turbo/image-to-video,768p,5.0,85,2.6675288240003283,3.140221786400434,3.638195310999436,3.8602934768940673,0.7720586953788134,0.7276390621998872,0.7294826282352943,fal request duration,,0.04,0.04,0.02,0.0,0.04,0.2,0.2,model/remote/text+image_video/minimax_h3_max_turbo/fal.yaml,fal list; fal still bills 50% promo $0.02/s (page says promo ended 2026-09-30),0.7294826282352943,"measured (fal duration, run pjwfymx5)",,
|
| 9 |
+
i2v,minimax_h3_max_turbo__prompt_expansion_mode_balanced__resolution_768p,"MiniMax H3 Max Turbo (expansion balanced, 768p)",8nqtrs6k,p-video-i2v,fal,minimax/h3-max-turbo/image-to-video,768p,5.0,85,3.360883756977273,3.894791807589354,4.465880261996063,4.664087176540702,0.9328174353081404,0.8931760523992127,0.8866546423529411,fal request duration,,0.04,0.04,0.02,0.0,0.04,0.2,0.2,model/remote/text+image_video/minimax_h3_max_turbo/fal_768p_balanced.yaml,fal list; fal still bills 50% promo $0.02/s (page says promo ended 2026-09-30),0.7294826282352943,"measured (fal duration, run pjwfymx5)",993.0,1028.0
|
| 10 |
+
i2v,minimax_h3_max__prompt_expansion_mode_balanced__resolution_480p,"MiniMax H3 Max (expansion balanced, 480p)",zd77gnta,p-video-i2v,fal,minimax/h3-max/image-to-video,480p,5.0,85,2.4688335089995235,2.904314552000142,3.387883318999229,3.737727538835273,0.7475455077670545,0.6775766637998457,0.7043849176470587,fal request duration,,0.05,0.025,0.025,0.0,0.05,0.25,0.25,model/remote/text+image_video/minimax_h3_max/fal_480p_balanced.yaml,fal list; fal still bills 50% promo $0.025/s (page says promo ended 2026-09-30),0.56,estimated,989.0,922.0
|
| 11 |
+
i2v,veo_3_1_lite,Veo 3.1 Lite (720p),py4m5xd9,p-video-i2v,wavespeed,google/veo3.1-lite/image-to-video,720p,6.0,98,41.81875224500254,49.431026957599535,56.17033411549892,58.89651842808168,9.816086404680279,9.361722352583154,,,,0.05,0.05,0.05,0.0,0.05000000000000001,0.30000000000000004,0.25,model/remote/text+image_video/veo_3_1_lite/wavespeed.yaml,,,,1016.0,1029.0
|
| 12 |
+
i2v,grok_imagine_video,Grok Imagine Video (720p),1pj3o83x,p-video-i2v,wavespeed,x-ai/grok-imagine-video/image-to-video,720p,6.0,100,62.43508535699948,69.995984365201,81.70177976000014,85.91732073955012,14.319553456591688,13.616963293333356,,,,0.05,0.05,0.05,0.0,0.05000000000000001,0.30000000000000004,0.25,model/remote/text+image_video/grok_imagine_video/wavespeed.yaml,,,,998.0,993.0
|
| 13 |
+
i2v,p_video_2_pro__mode_quality__prompt_upsampler_max__resolution_768p,"P-Video 2 Pro (quality, upsampler max, 768p)",92gnt2qk,p-video-i2v,replicate,prunaai/p-video-2-pro,768p,5.0,100,54.11051772299106,68.09604510500503,82.57195331449475,102.8043552905605,20.5608710581121,16.514390662898947,1.8536400000000002,replicate model_execution_time_s,16.2845354215,0.075,0.075,0.075,0.0,0.075,0.375,0.375,model/remote/text+image_video/p_video_2_pro/replicate_768p_quality_max.yaml,,,,1013.0,1049.0
|
| 14 |
+
i2v,minimax_h3_max__prompt_expansion_mode_disabled__resolution_768p,"MiniMax H3 Max (expansion disabled, 768p)",3m1dlqdk,p-video-i2v,fal,minimax/h3-max/image-to-video,768p,5.0,85,4.174281786999927,4.483852291799485,5.105214381999758,5.227787268011718,1.0455574536023435,1.0210428763999517,0.9978179623529411,fal request duration,,0.08,0.08,0.04,0.0,0.08,0.4,0.4,model/remote/text+image_video/minimax_h3_max/fal.yaml,fal list; fal still bills 50% promo $0.04/s (page says promo ended 2026-09-30),0.9978179623529411,"measured (fal duration, run 3m1dlqdk)",,
|
| 15 |
+
i2v,minimax_h3_max__prompt_expansion_mode_balanced__resolution_768p,"MiniMax H3 Max (expansion balanced, 768p)",6l5e62xs,p-video-i2v,fal,minimax/h3-max/image-to-video,768p,5.0,85,4.774919076007791,5.433260408189381,5.80711786300526,6.162871519518975,1.232574303903795,1.161423572601052,1.1812099717647058,fal request duration,,0.08,0.08,0.04,0.0,0.08,0.4,0.4,model/remote/text+image_video/minimax_h3_max/fal_768p_balanced.yaml,fal list; fal still bills 50% promo $0.04/s (page says promo ended 2026-09-30),0.9978179623529411,"measured (fal duration, run 3m1dlqdk)",999.0,993.0
|
| 16 |
+
i2v,ltx_2_5_fast,LTX 2.5 Fast (720p),mdwwaui9,p-video-i2v,fal,lightricks/ltx-2.5/image-to-video/fast,720p,6.0,100,19.59916955500012,25.333775025999785,30.807174913999916,30.09333773863007,5.015556289771679,5.13452915233332,,,,0.09,0.09,0.09,0.0,0.09000000000000001,0.54,0.44999999999999996,model/remote/text+image_video/ltx_2_5_fast/fal.yaml,,,,1006.0,943.0
|
| 17 |
+
i2v,gemini_omni_1_1_flash,Gemini Omni 1.1 Flash (720p),rgls30vm,p-video-i2v,wavespeed,google/gemini-omni-1.1-flash/image-to-video,720p,5.0,96,33.23962910800037,40.41112426700056,47.169307519498034,76.65512052288535,15.331024104577072,9.433861503899607,,,,0.1,0.1,0.1,0.0,0.1,0.5,0.5,model/remote/text+image_video/gemini_omni_1_1_flash/wavespeed.yaml,,,,1018.0,993.0
|
| 18 |
+
i2v,minimax_h3,MiniMax H3 (768p),4i8707tj,p-video-i2v,wavespeed,minimax/h3/image-to-video,768p,5.0,98,122.5933248429974,146.0719200146006,155.3556124200004,163.95953334504068,32.791906669008135,31.071122484000078,,,,0.1,0.1,0.1,0.0,0.1,0.5,0.5,model/remote/text+image_video/minimax_h3/wavespeed.yaml,,,,1019.0,1023.0
|
| 19 |
+
i2v,ltx_2_5_pro,LTX 2.5 Pro (720p),oq5adzi8,p-video-i2v,fal,lightricks/ltx-2.5/image-to-video/pro,720p,6.0,100,23.96283850599866,30.78141485860033,32.351289345499936,32.846369120970046,5.474394853495008,5.391881557583323,,,,0.12,0.12,0.12,0.0,0.12,0.72,0.6,model/remote/text+image_video/ltx_2_5_pro/fal.yaml,,,,1009.0,1016.0
|
| 20 |
+
i2v,grok_imagine_video_v1_5,Grok Imagine Video v1.5 (720p),c6iao0m8,p-video-i2v,wavespeed,x-ai/grok-imagine-video-v1.5/image-to-video,720p,5.0,100,37.19857703399976,44.88756618759872,55.24653504449998,59.73615804140987,11.947231608281975,11.049307008899996,,,,0.14,0.14,0.14,0.01,0.14200000000000002,0.7100000000000001,0.7100000000000001,model/remote/text+image_video/grok_imagine_video_v1_5/wavespeed.yaml,,,,1000.0,1023.0
|
| 21 |
+
i2v,seedance_2_5_turbo,Seedance 2.5 Turbo (720p),9pr8w9xy,p-video-i2v,wavespeed,bytedance/seedance-2.5/image-to-video-turbo,720p,5.0,100,151.128106406999,203.1471570823989,253.8349715054992,280.7854034020103,56.15708068040205,50.76699430109984,,,,0.2,0.2,0.2,0.0,0.2,1.0,1.0,model/remote/text+image_video/seedance_2_5_turbo/wavespeed.yaml,,,,975.0,935.0
|
data/p-video-2-leaderboard.csv
DELETED
|
@@ -1,17 +0,0 @@
|
|
| 1 |
-
model_id,wandb_run_ids,n_generations,min_generation_s,median_generation_s,p20_generation_s,generation_s_per_output_video_s,model_execution_s_per_output_video_s,datapoint_elo,rapidata_elo,price_usd_per_second
|
| 2 |
-
gemini_omni_1_1_flash,mfajagum,89,26.276599962002365,31.884095050001633,29.0771403791965,6.399738597206617,,1037,1197,0.100
|
| 3 |
-
grok_imagine_video,q39r8v9e,90,68.9776645039965,99.17614604350092,78.69151808420138,16.13514411181308,,958,994,0.050
|
| 4 |
-
grok_imagine_video_v1_5,z2fp7hd7,90,38.42492011199647,52.268091876499966,46.47850653079659,11.452002471104521,,959,1016,0.140
|
| 5 |
-
minimax_h3,xpcv2m58,89,97.61419671699696,117.51345164199665,106.57570995900024,24.968780374361877,,1026,1108,0.100
|
| 6 |
-
minimax_h3_max,0d2lkmp6,90,4.173863318999793,4.49869444649994,4.4671880990012145,1.3807388451000264,0.644,1031,1197,0.080
|
| 7 |
-
minimax_h3_max_turbo__prompt_expansion_mode_balanced,kxuwq31c,90,2.891819470001792,3.487746603501364,3.134695611400821,1.35231252479333,0.61,1040,1214,0.040
|
| 8 |
-
p_video_2__draft_false__prompt_upsampling_false__resolution_1080p,9hgkhujw,90,13.33159556199098,16.702878594005597,14.793676785795833,4.033411242884629,2.151682222222222,,,0.050
|
| 9 |
-
p_video_2__draft_false__prompt_upsampling_false__resolution_720p,m2jf4z7h,89,5.859760620005545,8.162274362999597,7.147269867203431,2.1525601170473116,0.9053123595505614,,,0.025
|
| 10 |
-
p_video_2__draft_false__prompt_upsampling_true__resolution_1080p,uoqpv10b,90,14.615217412007041,18.316011337505188,16.441080821005745,4.5638632221758275,2.152566666666667,979,991,0.050
|
| 11 |
-
p_video_2__draft_false__prompt_upsampling_true__resolution_720p,2q5jydr8,90,7.227749306999613,9.880705762494472,9.154919161600992,2.4789854674801206,0.9079755555555558,985,1039,0.025
|
| 12 |
-
p_video_2__draft_true__prompt_upsampling_false__resolution_1080p,qbt0atz4,89,6.250065040003392,9.528199804000906,7.504525098402519,2.431133616615736,0.7517640449438197,,,0.030
|
| 13 |
-
p_video_2__draft_true__prompt_upsampling_false__resolution_720p,9ur57h54,88,3.379928643000312,5.454214365498046,4.208131522199255,1.4960965825225272,0.40954545454545455,,,0.015
|
| 14 |
-
p_video_2__draft_true__prompt_upsampling_true__resolution_1080p,gcbyy4wn,90,8.047415203996934,11.128662197996164,9.22702455239487,2.8557857865532665,0.7514066666666668,,,0.030
|
| 15 |
-
p_video_2__draft_true__prompt_upsampling_true__resolution_720p,6hi994vk,90,4.985804016003385,7.4922584965024726,6.472871531004785,2.4024102996157146,0.40678444444444434,982,1048,0.015
|
| 16 |
-
seedance_2_5_turbo,l9zjnv7n,90,168.997338743,252.07832621250054,211.93920676639829,67.60899374696221,,1037,1080,0.200
|
| 17 |
-
veo_3_1_lite,jjbobes4,90,33.894167409991496,37.83729844300251,37.009501291197374,6.745116482181308,,968,986,0.050
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
data/t2v-pro.csv
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
task,model_id,display_name,wandb_run_ids,wandb_project,provider,endpoint,resolution,output_duration_s,n_generations,min_generation_s,p20_generation_s,median_generation_s,mean_generation_s,generation_s_per_output_video_s,median_generation_s_per_output_video_s,model_execution_s_per_output_video_s,model_execution_source,predict_s_per_output_video_s,list_price_usd_per_second,yaml_price_usd_per_second,currently_billed_usd_per_second,flat_price_usd_per_request,price_usd_per_second,price_usd_per_clip,price_usd_per_5s,price_config,price_note,disabled_expansion_model_execution_s_per_output_video_s,disabled_expansion_timing_source,datapoint_elo,rapidata_elo
|
| 2 |
+
t2v,p_video_2_pro__mode_cost__prompt_upsampler_off__resolution_480p,"P-Video 2 Pro (cost mode, 480p)",jycb3yyj,p-video-2-pro,replicate,prunaai/p-video-2-pro,480p,5.0,10,9.408479179999633,9.510774168599891,9.72883671299951,10.391401128699908,2.0782802257399817,1.9457673425999018,0.9005,replicate model_execution_time_s,1.8816932411,0.01,0.01,0.01,0.0,0.01,0.05,0.05,model/remote/text_video/p_video_2_pro/replicate_480p_cost_off.yaml,,,,961.0,905.0
|
| 3 |
+
t2v,p_video_2__draft_true__prompt_upsampling_false__resolution_720p,"P-Video 2 (draft, no upsampling, 720p)",9ur57h54,p-video-2-0,replicate,prunaai/p-video-2.0,720p,5.0,88,3.379928643000312,4.208131522199255,5.454214365498046,7.4804829126126355,1.4960965825225272,1.0908428730996094,0.40954545454545455,replicate model_execution_time_s,1.0549488494,0.015,0.015,0.015,0.0,0.015,0.075,0.075,model/remote/text_video/p_video_2/replicate_720p_draft.yaml,,,,,
|
| 4 |
+
t2v,p_video_2__draft_true__prompt_upsampling_true__resolution_720p,"P-Video 2 (draft, upsampling, 720p)",6hi994vk,p-video-2-0,replicate,prunaai/p-video-2.0,720p,5.0,90,4.985804016003385,6.472871531004785,7.4922584965024726,12.012051498078574,2.4024102996157146,1.4984516993004944,0.40678444444444445,replicate model_execution_time_s,1.4668891899999998,0.015,0.015,0.015,0.0,0.015,0.075,0.075,model/remote/text_video/p_video_2/replicate_720p_draft_upsample.yaml,,,,984.0,1044.0
|
| 5 |
+
t2v,p_video_2_pro__mode_cost__prompt_upsampler_off__resolution_768p,"P-Video 2 Pro (cost mode, 768p)",zeio3rpy,p-video-2-pro,replicate,prunaai/p-video-2-pro,768p,5.0,10,15.82823935200031,16.06025371800024,18.991634191500452,18.809975293400203,3.7619950586800406,3.7983268383000905,2.0337,replicate model_execution_time_s,3.7674125726,0.025,0.025,0.025,0.0,0.025,0.125,0.125,model/remote/text_video/p_video_2_pro/replicate_768p_cost_off.yaml,,,,1011.0,1129.0
|
| 6 |
+
t2v,agnes_video_2_5,Agnes Video 2.5 (720p),krnwtzyy,p-video-2-0,agnes,agnes-video-2.5,720p,5.0,90,39.94301277900013,48.21273472360044,56.4691387010007,62.43517924498891,12.487035848997783,11.29382774020014,,,,0.025,0.025,0.025,0.0,0.025,0.125,0.125,model/remote/text_video/agnes_video_2_5/agnes.yaml,,,,990.0,1009.0
|
| 7 |
+
t2v,p_video_2__draft_false__prompt_upsampling_false__resolution_720p,"P-Video 2 (full, no upsampling, 720p)",m2jf4z7h,p-video-2-0,replicate,prunaai/p-video-2.0,720p,5.0,89,5.859760620005545,7.147269867203431,8.162274362999597,10.762800585236556,2.1525601170473116,1.6324548725999193,0.9053123595505618,replicate model_execution_time_s,1.5813138966,0.03,0.025,0.03,0.0,0.03,0.15,0.15,model/remote/text_video/p_video_2/replicate_720p.yaml,Replicate prunaai/p-video-2 list; YAML had $0.025/s,,,,
|
| 8 |
+
t2v,p_video_2__draft_true__prompt_upsampling_false__resolution_1080p,"P-Video 2 (draft, no upsampling, 1080p)",qbt0atz4,p-video-2-0,replicate,prunaai/p-video-2.0,1080p,5.0,89,6.250065040003392,7.504525098402519,9.528199804000906,12.155668083078679,2.431133616615736,1.905639960800181,0.7517640449438201,replicate model_execution_time_s,1.7764584898,0.03,0.03,0.03,0.0,0.03,0.15,0.15,model/remote/text_video/p_video_2/replicate_1080p_draft.yaml,,,,,
|
| 9 |
+
t2v,p_video_2__draft_false__prompt_upsampling_true__resolution_720p,"P-Video 2 (full, upsampling, 720p)",2q5jydr8,p-video-2-0,replicate,prunaai/p-video-2.0,720p,5.0,90,7.227749306999613,9.154919161600994,9.880705762494472,12.394927337400603,2.4789854674801206,1.9761411524988943,0.9079755555555554,replicate model_execution_time_s,1.8864087514999999,0.03,0.025,0.03,0.0,0.03,0.15,0.15,model/remote/text_video/p_video_2/replicate_720p_upsample.yaml,Replicate prunaai/p-video-2 list; YAML had $0.025/s,,,987.0,1032.0
|
| 10 |
+
t2v,p_video_2__draft_true__prompt_upsampling_true__resolution_1080p,"P-Video 2 (draft, upsampling, 1080p)",gcbyy4wn,p-video-2-0,replicate,prunaai/p-video-2.0,1080p,5.0,90,8.047415203996934,9.22702455239487,11.128662197996164,14.278928932766332,2.8557857865532665,2.225732439599233,0.7514066666666667,replicate model_execution_time_s,2.178066129,0.03,0.03,0.03,0.0,0.03,0.15,0.15,model/remote/text_video/p_video_2/replicate_1080p_draft_upsample.yaml,,,,,
|
| 11 |
+
t2v,p_video_2_pro__mode_speed__prompt_upsampler_turbo__resolution_768p,"P-Video 2 Pro (speed, upsampler turbo, 768p)",syngi7ri,p-video-2-pro,replicate,prunaai/p-video-2-pro,768p,5.0,100,7.353956573002506,9.24571067720244,11.115983718002099,11.52855639862959,2.305711279725918,2.2231967436004196,0.807034,replicate model_execution_time_s,2.1527908192,0.035,0.035,0.035,0.0,0.035,0.17500000000000002,0.17500000000000002,model/remote/text_video/p_video_2_pro/replicate_768p_speed_turbo.yaml,,,,1011.0,1129.0
|
| 12 |
+
t2v,p_video_2_pro__mode_speed__prompt_upsampler_off__resolution_768p,"P-Video 2 Pro (speed, upsampler off, 768p)",jgyitbf0,p-video-2-pro,replicate,prunaai/p-video-2-pro,768p,5.0,99,4.8043301480065566,7.719320420798613,10.179680373999872,11.869338264453667,2.373867652890733,2.0359360747999746,0.8455151515151514,replicate model_execution_time_s,1.7538052489999998,0.035,0.035,0.035,0.0,0.035,0.17500000000000002,0.17500000000000002,model/remote/text_video/p_video_2_pro/replicate_768p_speed_off.yaml,,,,,
|
| 13 |
+
t2v,p_video_2_pro__mode_speed__prompt_upsampler_max__resolution_768p,"P-Video 2 Pro (speed, upsampler max, 768p)",94ledtdd,p-video-2-pro,replicate,prunaai/p-video-2-pro,768p,5.0,100,11.908190361995366,16.36631814179418,21.841233791004925,32.62473624780017,6.524947249560035,4.368246758200986,0.7760619999999999,replicate model_execution_time_s,4.344442295,0.035,0.035,0.035,0.0,0.035,0.17500000000000002,0.17500000000000002,model/remote/text_video/p_video_2_pro/replicate_768p_speed_max.yaml,,,,,
|
| 14 |
+
t2v,minimax_h3_max_turbo__prompt_expansion_mode_disabled,"MiniMax H3 Max Turbo (expansion disabled, 768p)",3ddm84m0,p-video-2-0,fal,minimax/h3-max-turbo/text-to-video,768p,5.0,90,2.062721883998165,2.2604693188011877,2.277240929495747,3.0311573386223043,0.6062314677244609,0.45544818589914937,0.5372808644444445,fal request duration,,0.04,0.04,0.02,0.0,0.04,0.2,0.2,model/remote/text_video/minimax_h3_max_turbo/fal.yaml,fal list; fal still bills 50% promo $0.02/s (page says promo ended 2026-09-30),0.5372808644444445,"measured (fal duration, run 3ddm84m0)",,
|
| 15 |
+
t2v,minimax_h3_max_turbo__prompt_expansion_mode_balanced,"MiniMax H3 Max Turbo (expansion balanced, 768p)",kxuwq31c,p-video-2-0,fal,minimax/h3-max-turbo/text-to-video,768p,5.0,90,2.891819470001792,3.134695611400821,3.487746603501364,6.76156262396665,1.35231252479333,0.6975493207002728,1.3032290133333335,fal request duration,,0.04,0.04,0.02,0.0,0.04,0.2,0.2,model/remote/text_video/minimax_h3_max_turbo/fal.yaml,fal list; fal still bills 50% promo $0.02/s (page says promo ended 2026-09-30),0.5372808644444445,"measured (fal duration, run 3ddm84m0)",1039.0,1208.0
|
| 16 |
+
t2v,veo_3_1_lite,Veo 3.1 Lite (720p),jjbobes4,p-video-2-0,wavespeed,google/veo3.1-lite/text-to-video,720p,6.0,90,33.894167409991496,37.009501291197374,37.83729844300251,40.47069889308784,6.745116482181308,6.306216407167086,,,,0.05,0.05,0.05,0.0,0.05000000000000001,0.30000000000000004,0.25,model/remote/text_video/veo_3_1_lite/wavespeed.yaml,,,,969.0,976.0
|
| 17 |
+
t2v,grok_imagine_video,Grok Imagine Video (720p),q39r8v9e,p-video-2-0,wavespeed,x-ai/grok-imagine-video/text-to-video,720p,6.0,90,68.9776645039965,78.69151808420138,99.17614604350092,96.81086467087847,16.13514411181308,16.52935767391682,,,,0.05,0.05,0.05,0.0,0.05000000000000001,0.30000000000000004,0.25,model/remote/text_video/grok_imagine_video/wavespeed.yaml,,,,957.0,987.0
|
| 18 |
+
t2v,p_video_2__draft_false__prompt_upsampling_false__resolution_1080p,"P-Video 2 (full, no upsampling, 1080p)",9hgkhujw,p-video-2-0,replicate,prunaai/p-video-2.0,1080p,5.0,90,13.33159556199098,14.793676785795833,16.702878594005597,20.167056214423145,4.033411242884629,3.3405757188011194,2.151682222222222,replicate model_execution_time_s,3.2819483332,0.06,0.05,0.06,0.0,0.06,0.3,0.3,model/remote/text_video/p_video_2/replicate_1080p.yaml,Replicate prunaai/p-video-2 list; YAML had $0.05/s,,,,
|
| 19 |
+
t2v,p_video_2__draft_false__prompt_upsampling_true__resolution_1080p,"P-Video 2 (full, upsampling, 1080p)",uoqpv10b,p-video-2-0,replicate,prunaai/p-video-2.0,1080p,5.0,90,14.61521741200704,16.441080821005745,18.316011337505188,22.819316110879136,4.5638632221758275,3.6632022675010374,2.1525666666666665,replicate model_execution_time_s,3.6489784669,0.06,0.05,0.06,0.0,0.06,0.3,0.3,model/remote/text_video/p_video_2/replicate_1080p_upsample.yaml,Replicate prunaai/p-video-2 list; YAML had $0.05/s,,,981.0,985.0
|
| 20 |
+
t2v,p_video_2_pro__mode_quality__prompt_upsampler_off__resolution_768p,"P-Video 2 Pro (quality, upsampler off, 768p)",xhk419ab,p-video-2-pro,replicate,prunaai/p-video-2-pro,768p,5.0,100,9.819089652010009,11.889321832402493,14.033189198496984,14.014546667209652,2.80290933344193,2.8066378396993965,1.6132959999999998,replicate model_execution_time_s,2.799488201,0.075,0.075,0.075,0.0,0.075,0.375,0.375,model/remote/text_video/p_video_2_pro/replicate_768p_quality_off.yaml,,,,,
|
| 21 |
+
t2v,p_video_2_pro__mode_quality__prompt_upsampler_turbo__resolution_768p,"P-Video 2 Pro (quality, upsampler turbo, 768p)",12cddspr,p-video-2-pro,replicate,prunaai/p-video-2-pro,768p,5.0,99,11.383456109993858,12.709244234193466,14.387750708992826,17.637302765524254,3.5274605531048504,2.877550141798565,1.6795858585858587,replicate model_execution_time_s,2.8335850258,0.075,0.075,0.075,0.0,0.075,0.375,0.375,model/remote/text_video/p_video_2_pro/replicate_768p_quality_turbo.yaml,,,,,
|
| 22 |
+
t2v,p_video_2_pro__mode_quality__prompt_upsampler_max__resolution_768p,"P-Video 2 Pro (quality, upsampler max, 768p)",w7xnkv2n,p-video-2-pro,replicate,prunaai/p-video-2-pro,768p,5.0,100,16.86041410799953,19.295531428194955,27.29956308000692,30.495400724648935,6.099080144929787,5.459912616001384,1.6686840000000005,replicate model_execution_time_s,5.1314450994000005,0.075,0.075,0.075,0.0,0.075,0.375,0.375,model/remote/text_video/p_video_2_pro/replicate_768p_quality_max.yaml,,,,1022.0,1087.0
|
| 23 |
+
t2v,minimax_h3_max,"MiniMax H3 Max (expansion balanced, 768p)",0d2lkmp6,p-video-2-0,fal,minimax/h3-max/text-to-video,768p,5.0,90,4.173863318999793,4.4671880990012145,4.49869444649994,6.903694225500132,1.3807388451000264,0.899738889299988,1.3319782333333334,fal request duration,,0.08,0.08,0.04,0.0,0.08,0.4,0.4,model/remote/text_video/minimax_h3_max/fal.yaml,fal list; fal still bills 50% promo $0.04/s (page says promo ended 2026-09-30),0.74,estimated,1032.0,1191.0
|
| 24 |
+
t2v,ltx_2_5_fast,LTX 2.5 Fast (720p),h6se1hfi,p-video-2-0,fal,lightricks/ltx-2.5/text-to-video/fast,720p,6.0,90,21.695961473000352,24.118977216200438,26.684429597007693,31.620479494244176,5.270079915707363,4.4474049328346155,,,,0.09,0.09,0.09,0.0,0.09000000000000001,0.54,0.44999999999999996,model/remote/text_video/ltx_2_5_fast/fal.yaml,,,,1003.0,1029.0
|
| 25 |
+
t2v,gemini_omni_1_1_flash,Gemini Omni 1.1 Flash (720p),mfajagum,p-video-2-0,wavespeed,google/gemini-omni-1.1-flash/text-to-video,720p,5.0,89,26.276599962002365,29.0771403791965,31.884095050001637,31.99869298603308,6.399738597206617,6.376819010000327,,,,0.1,0.1,0.1,0.0,0.1,0.5,0.5,model/remote/text_video/gemini_omni_1_1_flash/wavespeed.yaml,,,,1039.0,1189.0
|
| 26 |
+
t2v,minimax_h3,MiniMax H3 (768p),xpcv2m58,p-video-2-0,wavespeed,minimax/h3/text-to-video,768p,5.0,89,97.61419671699696,106.57570995900022,117.51345164199664,124.8439018718094,24.968780374361877,23.50269032839933,,,,0.1,0.1,0.1,0.0,0.1,0.5,0.5,model/remote/text_video/minimax_h3/wavespeed.yaml,,,,1025.0,1103.0
|
| 27 |
+
t2v,ltx_2_5_pro,LTX 2.5 Pro (720p),kjbgxbab,p-video-2-0,fal,lightricks/ltx-2.5/text-to-video/pro,720p,6.0,90,23.997383733993047,25.263995286799037,27.897931821993552,29.617955809731697,4.936325968288616,4.649655303665591,,,,0.12,0.12,0.12,0.0,0.12,0.72,0.6,model/remote/text_video/ltx_2_5_pro/fal.yaml,,,,1007.0,975.0
|
| 28 |
+
t2v,grok_imagine_video_v1_5,Grok Imagine Video v1.5 (720p),z2fp7hd7,p-video-2-0,wavespeed,x-ai/grok-imagine-video-v1.5/text-to-video,720p,5.0,90,38.42492011199647,46.47850653079659,52.268091876499966,57.260012355522605,11.452002471104521,10.453618375299992,,,,0.14,0.14,0.14,0.0,0.14,0.7000000000000001,0.7000000000000001,model/remote/text_video/grok_imagine_video_v1_5/wavespeed.yaml,,,,961.0,1009.0
|
| 29 |
+
t2v,seedance_2_5_turbo,Seedance 2.5 Turbo (720p),l9zjnv7n,p-video-2-0,wavespeed,bytedance/seedance-2.5/text-to-video-turbo,720p,5.0,90,168.997338743,211.93920676639829,252.07832621250054,338.04496873481105,67.60899374696221,50.41566524250011,,,,0.2,0.2,0.2,0.0,0.2,1.0,1.0,model/remote/text_video/seedance_2_5_turbo/wavespeed.yaml,,,,1038.0,1071.0
|
model_display.py
CHANGED
|
@@ -96,6 +96,8 @@ MODEL_DISPLAY_NAMES = {
|
|
| 96 |
"p_video_edit_preview__replicate_final": "P-Video-Edit",
|
| 97 |
"p_video_edit_preview__replicate_final__draft": "P-Video-Edit Draft",
|
| 98 |
# Text-to-video leaderboard
|
|
|
|
|
|
|
| 99 |
"gemini_omni_1_1_flash": "Gemini Omni 1.1 Flash",
|
| 100 |
"grok_imagine_video": "Grok Imagine Video",
|
| 101 |
"grok_imagine_video_v1_5": "Grok Imagine Video 1.5",
|
|
@@ -239,6 +241,31 @@ def _p_video_2_display_name(model_id: str):
|
|
| 239 |
return label
|
| 240 |
|
| 241 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
def _prettify_snake_case(model_id: str) -> str:
|
| 243 |
parts = [part for part in str(model_id).split("_") if part]
|
| 244 |
pretty = []
|
|
@@ -263,6 +290,9 @@ def display_model_name(model_id) -> str:
|
|
| 263 |
return ""
|
| 264 |
if raw in MODEL_DISPLAY_NAMES:
|
| 265 |
return MODEL_DISPLAY_NAMES[raw]
|
|
|
|
|
|
|
|
|
|
| 266 |
p_video_2 = _p_video_2_display_name(raw)
|
| 267 |
if p_video_2:
|
| 268 |
return p_video_2
|
|
|
|
| 96 |
"p_video_edit_preview__replicate_final": "P-Video-Edit",
|
| 97 |
"p_video_edit_preview__replicate_final__draft": "P-Video-Edit Draft",
|
| 98 |
# Text-to-video leaderboard
|
| 99 |
+
"ltx_2_5_fast": "LTX 2.5 Fast",
|
| 100 |
+
"ltx_2_5_pro": "LTX 2.5 Pro",
|
| 101 |
"gemini_omni_1_1_flash": "Gemini Omni 1.1 Flash",
|
| 102 |
"grok_imagine_video": "Grok Imagine Video",
|
| 103 |
"grok_imagine_video_v1_5": "Grok Imagine Video 1.5",
|
|
|
|
| 241 |
return label
|
| 242 |
|
| 243 |
|
| 244 |
+
def _video_variant_display_name(model_id: str):
|
| 245 |
+
"""Keep Pro settings and resolution-specific comparison models distinct."""
|
| 246 |
+
parts = model_id.split("__")
|
| 247 |
+
base = parts[0]
|
| 248 |
+
if base in {"p_video_2_pro", "p-video-2-pro"}:
|
| 249 |
+
label = "P-Video-2 Pro"
|
| 250 |
+
elif base.startswith("minimax_") and any(
|
| 251 |
+
part.startswith("resolution_") for part in parts[1:]
|
| 252 |
+
):
|
| 253 |
+
label = display_model_name(base)
|
| 254 |
+
else:
|
| 255 |
+
return None
|
| 256 |
+
|
| 257 |
+
for part in parts[1:]:
|
| 258 |
+
if part.startswith("mode_"):
|
| 259 |
+
label += " " + part[len("mode_") :].capitalize()
|
| 260 |
+
elif part.startswith("prompt_upsampler_"):
|
| 261 |
+
label += " (upsampler " + part[len("prompt_upsampler_") :] + ")"
|
| 262 |
+
elif part.startswith("prompt_expansion_mode_"):
|
| 263 |
+
label += " (expansion " + part[len("prompt_expansion_mode_") :] + ")"
|
| 264 |
+
elif part.startswith("resolution_"):
|
| 265 |
+
label += " " + part[len("resolution_") :]
|
| 266 |
+
return label
|
| 267 |
+
|
| 268 |
+
|
| 269 |
def _prettify_snake_case(model_id: str) -> str:
|
| 270 |
parts = [part for part in str(model_id).split("_") if part]
|
| 271 |
pretty = []
|
|
|
|
| 290 |
return ""
|
| 291 |
if raw in MODEL_DISPLAY_NAMES:
|
| 292 |
return MODEL_DISPLAY_NAMES[raw]
|
| 293 |
+
video_variant = _video_variant_display_name(raw)
|
| 294 |
+
if video_variant:
|
| 295 |
+
return video_variant
|
| 296 |
p_video_2 = _p_video_2_display_name(raw)
|
| 297 |
if p_video_2:
|
| 298 |
return p_video_2
|
ui.py
CHANGED
|
@@ -54,10 +54,12 @@ TAB_SAMPLES = "samples"
|
|
| 54 |
TAB_ABOUT = "about"
|
| 55 |
|
| 56 |
MODALITY_TEXT_TO_VIDEO = "text_to_video"
|
|
|
|
| 57 |
MODALITY_VIDEO_TO_VIDEO = "video_to_video"
|
| 58 |
MODALITY_TEXT_TO_IMAGE = "text_to_image"
|
| 59 |
MODALITY_CHOICES = [
|
| 60 |
("Text to Video", MODALITY_TEXT_TO_VIDEO),
|
|
|
|
| 61 |
("Video to Video", MODALITY_VIDEO_TO_VIDEO),
|
| 62 |
("Text to Image", MODALITY_TEXT_TO_IMAGE),
|
| 63 |
]
|
|
@@ -73,14 +75,14 @@ _VIEW_EVENTS = {
|
|
| 73 |
ABOUT_OVERVIEW_CONTENT = """
|
| 74 |
# About P-Bench
|
| 75 |
|
| 76 |
-
P-Bench compares **text-to-video**, **video-to-video**, and **text-to-image**
|
| 77 |
models, including optimized or accelerated endpoints, on **quality, speed,
|
| 78 |
and price**. Each view is a **dataset** scored with a **metric**, written as
|
| 79 |
`Dataset | Metric`. There is no single score across P-Bench.
|
| 80 |
|
| 81 |
## How to read it
|
| 82 |
|
| 83 |
-
1. Pick a **model type** (Text to Video, Video to Video, or Text to Image),
|
| 84 |
then a **dataset** and a **metric**.
|
| 85 |
2. **Leaderboards**: ranked by that metric. Price and generation time sit in
|
| 86 |
the same table when the source publishes them.
|
|
@@ -110,12 +112,19 @@ prompt suites, so samples are not shown.
|
|
| 110 |
## Current datasets
|
| 111 |
|
| 112 |
### VBench-2.0 Dataset
|
| 113 |
-
VBench-2.0 prompts, comparing P-Video-2 variants with
|
| 114 |
Quality is Datapoint Elo and Rapidata Elo from pairwise preference. Price
|
| 115 |
-
is USD per second of output video. Time per second of video is
|
| 116 |
time, except Pruna models which use model execution time. Samples are
|
| 117 |
available.
|
| 118 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
### Pruna Internal Video-Edit Benchmark
|
| 120 |
Pruna's internal video-to-video editing benchmark, collected by our
|
| 121 |
research engineers. It combines prompts from public video-editing
|
|
@@ -172,12 +181,12 @@ ABOUT_DETAILS_CONTENT = """
|
|
| 172 |
- **Generation time**: median and minimum generation time in seconds for
|
| 173 |
images, as reported in the evaluation table. For video, generation time
|
| 174 |
per second of output video is the more informative figure. On the
|
| 175 |
-
text-to-video
|
| 176 |
use model execution time. On video-edit it is end-to-end wall time. This
|
| 177 |
is not a p95, and we do not state warm vs cold or concurrent load. Not
|
| 178 |
available for Arena AI.
|
| 179 |
- **Price**: USD per image for text-to-image, or USD per second of output
|
| 180 |
-
video for text-to-video and video-to-video. We do not state list price vs
|
| 181 |
amount paid, or whether failed generations are included. Not available
|
| 182 |
for Arena AI.
|
| 183 |
|
|
@@ -187,7 +196,8 @@ models *within* a Dataset | Metric view.
|
|
| 187 |
|
| 188 |
## Setup
|
| 189 |
|
| 190 |
-
- **Evaluation
|
|
|
|
| 191 |
- **Update policy:** numbers come from evaluation snapshots in the tables,
|
| 192 |
not a live API poll.
|
| 193 |
- **Prompt counts:** OneIG Alignment uses 100 anime, 100 human, and 99 object
|
|
@@ -205,7 +215,10 @@ models *within* a Dataset | Metric view.
|
|
| 205 |
same prompt. Steps, CFG, prompt rewrite, and safety filters follow each
|
| 206 |
endpoint's default. This does not describe Artificial Analysis or Arena AI.
|
| 207 |
- **Generation (Text-to-Video):** one clip per prompt per endpoint when the
|
| 208 |
-
run exists.
|
|
|
|
|
|
|
|
|
|
| 209 |
- **Generation (Video-Edit):** one edited clip per prompt per endpoint when
|
| 210 |
the run exists. Every model sees the same source video for a prompt.
|
| 211 |
- **Datapoint (Qwen and OneIG):** every model pair is compared on every
|
|
@@ -665,9 +678,11 @@ def _leaderboard_html(data, columns):
|
|
| 665 |
css = _leaderboard_cell_class(column)
|
| 666 |
sort_value = escape(_leaderboard_sort_value(column, row[column]), quote=True)
|
| 667 |
if column == "Model":
|
|
|
|
| 668 |
cells.append(
|
| 669 |
-
f'<td class="{css}" data-sort-value="{sort_value}"
|
| 670 |
-
f"
|
|
|
|
| 671 |
)
|
| 672 |
else:
|
| 673 |
cells.append(
|
|
|
|
| 54 |
TAB_ABOUT = "about"
|
| 55 |
|
| 56 |
MODALITY_TEXT_TO_VIDEO = "text_to_video"
|
| 57 |
+
MODALITY_IMAGE_TO_VIDEO = "image_to_video"
|
| 58 |
MODALITY_VIDEO_TO_VIDEO = "video_to_video"
|
| 59 |
MODALITY_TEXT_TO_IMAGE = "text_to_image"
|
| 60 |
MODALITY_CHOICES = [
|
| 61 |
("Text to Video", MODALITY_TEXT_TO_VIDEO),
|
| 62 |
+
("Image to Video", MODALITY_IMAGE_TO_VIDEO),
|
| 63 |
("Video to Video", MODALITY_VIDEO_TO_VIDEO),
|
| 64 |
("Text to Image", MODALITY_TEXT_TO_IMAGE),
|
| 65 |
]
|
|
|
|
| 75 |
ABOUT_OVERVIEW_CONTENT = """
|
| 76 |
# About P-Bench
|
| 77 |
|
| 78 |
+
P-Bench compares **text-to-video**, **image-to-video**, **video-to-video**, and **text-to-image**
|
| 79 |
models, including optimized or accelerated endpoints, on **quality, speed,
|
| 80 |
and price**. Each view is a **dataset** scored with a **metric**, written as
|
| 81 |
`Dataset | Metric`. There is no single score across P-Bench.
|
| 82 |
|
| 83 |
## How to read it
|
| 84 |
|
| 85 |
+
1. Pick a **model type** (Text to Video, Image to Video, Video to Video, or Text to Image),
|
| 86 |
then a **dataset** and a **metric**.
|
| 87 |
2. **Leaderboards**: ranked by that metric. Price and generation time sit in
|
| 88 |
the same table when the source publishes them.
|
|
|
|
| 112 |
## Current datasets
|
| 113 |
|
| 114 |
### VBench-2.0 Dataset
|
| 115 |
+
VBench-2.0 prompts, comparing P-Video-2 and P-Video-2 Pro variants with other models.
|
| 116 |
Quality is Datapoint Elo and Rapidata Elo from pairwise preference. Price
|
| 117 |
+
is USD per second of output video. Time per second of video is provider wall
|
| 118 |
time, except Pruna models which use model execution time. Samples are
|
| 119 |
available.
|
| 120 |
|
| 121 |
+
### Image-to-Video Benchmark
|
| 122 |
+
Image-to-video evaluations comparing P-Video-2 Pro cost, speed, and quality modes
|
| 123 |
+
with other models, including 480p and 768p variants. Quality is Datapoint
|
| 124 |
+
Elo and Rapidata Elo from pairwise preference. Price is USD per second of
|
| 125 |
+
output video. Time per second of video is provider wall time, except Pruna
|
| 126 |
+
models which use model execution time. Sample media is not yet available.
|
| 127 |
+
|
| 128 |
### Pruna Internal Video-Edit Benchmark
|
| 129 |
Pruna's internal video-to-video editing benchmark, collected by our
|
| 130 |
research engineers. It combines prompts from public video-editing
|
|
|
|
| 181 |
- **Generation time**: median and minimum generation time in seconds for
|
| 182 |
images, as reported in the evaluation table. For video, generation time
|
| 183 |
per second of output video is the more informative figure. On the
|
| 184 |
+
text-to-video and image-to-video benchmarks this is provider wall time, except Pruna models which
|
| 185 |
use model execution time. On video-edit it is end-to-end wall time. This
|
| 186 |
is not a p95, and we do not state warm vs cold or concurrent load. Not
|
| 187 |
available for Arena AI.
|
| 188 |
- **Price**: USD per image for text-to-image, or USD per second of output
|
| 189 |
+
video for text-to-video, image-to-video, and video-to-video. We do not state list price vs
|
| 190 |
amount paid, or whether failed generations are included. Not available
|
| 191 |
for Arena AI.
|
| 192 |
|
|
|
|
| 196 |
|
| 197 |
## Setup
|
| 198 |
|
| 199 |
+
- **Evaluation snapshots:** the current video-generation exports include
|
| 200 |
+
P-Video-2 Pro. Other benchmark tables retain their existing snapshots.
|
| 201 |
- **Update policy:** numbers come from evaluation snapshots in the tables,
|
| 202 |
not a live API poll.
|
| 203 |
- **Prompt counts:** OneIG Alignment uses 100 anime, 100 human, and 99 object
|
|
|
|
| 215 |
same prompt. Steps, CFG, prompt rewrite, and safety filters follow each
|
| 216 |
endpoint's default. This does not describe Artificial Analysis or Arena AI.
|
| 217 |
- **Generation (Text-to-Video):** one clip per prompt per endpoint when the
|
| 218 |
+
run exists. The export reports 10 generations for Pro Cost variants and
|
| 219 |
+
about 90 for other models.
|
| 220 |
+
- **Generation (Image-to-Video):** the export reports 10 generations for
|
| 221 |
+
Pro Cost variants and 85–100 for other models, with five-second output clips.
|
| 222 |
- **Generation (Video-Edit):** one edited clip per prompt per endpoint when
|
| 223 |
the run exists. Every model sees the same source video for a prompt.
|
| 224 |
- **Datapoint (Qwen and OneIG):** every model pair is compared on every
|
|
|
|
| 678 |
css = _leaderboard_cell_class(column)
|
| 679 |
sort_value = escape(_leaderboard_sort_value(column, row[column]), quote=True)
|
| 680 |
if column == "Model":
|
| 681 |
+
model_name = _format_leaderboard_cell(column, row[column])
|
| 682 |
cells.append(
|
| 683 |
+
f'<td class="{css}" data-sort-value="{sort_value}" '
|
| 684 |
+
f'data-model-name="{model_name}" tabindex="0">'
|
| 685 |
+
f"<strong>{model_name}</strong></td>"
|
| 686 |
)
|
| 687 |
else:
|
| 688 |
cells.append(
|