Clean history without videos

.gitattributes

@@ -0,0 +1 @@
+

.gitignore

@@ -0,0 +1,9 @@
+# IDE configuration
+.idea/
+
+# Local data and results
+data/
+
+# Python cache files
+__pycache__/
+*.pyc

README.md

@@ -0,0 +1,10 @@
+---
+title: Sequential Driving World Model evaluation
+emoji: 🎬
+colorFrom: indigo
+colorTo: purple
+sdk: gradio
+sdk_version: "4.44.1"
+app_file: app.py
+pinned: false
+---

app.py

@@ -0,0 +1,348 @@
+import gradio as gr
+import pandas as pd
+import os
+import random
+from datetime import datetime
+from huggingface_hub import list_repo_files
+
+# --- 1. CONFIGURATION ---
+DATASET_REPO = "AhmadRH/SeqWMVideos"
+DATASET_BRANCH = "main"  # you can pin to a commit SHA for stability
+# VIDEO_DIR = "videos/opendv"
+VIDEO_BASE_URL = f"https://huggingface.co/datasets/{DATASET_REPO}/resolve/{DATASET_BRANCH}"
+RESULTS_FILE = "data/results.csv"
+ADMIN_PASSWORD = os.environ.get('ADMIN_PASSWORD', 'password_for_local_test')
+
+# Create the data directory if it doesn't exist
+os.makedirs("data", exist_ok=True)
+
+
+# --- 2. LOAD AND PREPARE VIDEO PAIRS ---
+def load_video_pairs():
+    """Scans the video directory and creates a list of video pairs."""
+    # videos = os.listdir(os.path.join(VIDEO_DIR, "ltx_2B_2step"))
+    files = list_repo_files(DATASET_REPO, repo_type="dataset", revision=DATASET_BRANCH)
+    # Keep only the files under /videos/... with typical video extensions
+    video_exts = {".mp4", ".webm", ".mov"}
+    dataset_videos = [f for f in files if os.path.splitext(f)[1].lower() in video_exts]
+
+    # Build: model -> set of filenames in that model folder (e.g., "scene1.mp4")
+    # Also remember full relative path per file.
+    from collections import defaultdict
+    model_to_files = defaultdict(set)
+    model_to_paths = defaultdict(dict)  # (filename -> "videos/<model>/<filename>")
+
+    for relpath in dataset_videos:
+        # relpath like: "ltx_2B_2step/scene1.mp4"
+        parts = relpath.split("/")  # ["videos", "<model>", "<filename>"]
+        model_name, filename = parts
+        model_to_files[model_name].add(filename)
+        model_to_paths[model_name][filename] = relpath
+
+    models_to_compare = [("ltx_2B_2step", "ltx_2B_unconditional"),
+                         ("ltx_13B_2step", "ltx_13B_unconditional"),
+                         ("ltx_2B_2step", "predict1-7B"),
+                         ("ltx_13B_2step", "predict2-2B"),
+                         ("ltx_2B_2step", "ltx_2B_2step_from13B"),
+                         ("ltx_2B_2step", "ltx_13B_2step_from2B"),
+                         ("ltx_2B_2step", "ltx_13B_2step"),
+                         ("ltx_2B_2step_from13B", "ltx_13B_2step_from2B"),
+                         ("ltx_2B_2step_from13B", "ltx_13B_2step"),
+                         ("ltx_13B_2step_from2B", "ltx_13B_2step")]
+    video_pairs = []
+    for model_a, model_b in models_to_compare:
+        if model_a not in model_to_files or model_b not in model_to_files:
+            continue
+
+        common_filenames = sorted(model_to_files[model_a].intersection(model_to_files[model_b]))
+        for filename in common_filenames:
+            path_a = model_to_paths[model_a][filename]  # e.g., "videos/model_a/scene1.mp4"
+            path_b = model_to_paths[model_b][filename]
+
+            url_a = f"{VIDEO_BASE_URL}/{path_a}"  # direct HTTPS URL to the file in the dataset
+            url_b = f"{VIDEO_BASE_URL}/{path_b}"
+
+            scene_name = os.path.splitext(filename)[0]
+            pair_key = f"{scene_name}${model_a} vs {model_b}"
+
+            video_pairs.append((url_a, url_b, pair_key, model_a, model_b, scene_name))
+
+    print(f"Loaded {len(video_pairs)} video pairs.")
+    return video_pairs
+
+
+# Load the pairs once when the app starts
+ALL_PAIRS = load_video_pairs()
+ALL_PAIR_KEYS = {key for _, _, key, _, _, _ in ALL_PAIRS}
+
+# --- 3. CORE APP LOGIC ---
+def start_session(email):
+    """
+    Initializes a user session. Hides login, shows eval UI, and loads the first pair.
+    """
+    if not email:
+        # You can add more robust email validation here if needed
+        raise gr.Error("Please enter your email to start.")
+
+    # NEW: Check for previously completed pairs for this user.
+    seen_pair_keys = set()
+    if os.path.exists(RESULTS_FILE):
+        df_results = pd.read_csv(RESULTS_FILE)
+        # Filter results for the current user's email
+        user_results = df_results[df_results["email"] == email]
+        seen_pair_keys = set(user_results["pair_key"])
+
+    # NEW: Determine which pairs the user has not seen yet.
+    unseen_pair_keys = ALL_PAIR_KEYS - seen_pair_keys
+
+    if not unseen_pair_keys:
+        return None, gr.update(visible=False), gr.update(visible=False), gr.update(
+            visible=True), None, None, "You have already completed all evaluations. Thank you!"
+
+    unseen_pairs = [pair for pair in ALL_PAIRS if pair[2] in unseen_pair_keys]
+    random.shuffle(unseen_pairs)
+
+    user_state = [email, unseen_pairs, []]
+
+    first_pair = user_state[1].pop()
+    path_a, path_b, pair_key, model_a_name, model_b_name, scene_name = first_pair
+
+    # Random assignment: which one goes to Video A / Video B in the UI
+    a_first = random.choice([True, False])
+    if a_first:
+        video_a_path, video_b_path = path_a, path_b
+        video_a_model, video_b_model = model_a_name, model_b_name
+    else:
+        video_a_path, video_b_path = path_b, path_a
+        video_a_model, video_b_model = model_b_name, model_a_name
+
+    # Store the actual assignment in the state for saving results
+    user_state.append({
+        "current_pair_key": pair_key,
+        "scene_name": scene_name,
+        "video_a_model": video_a_model,
+        "video_b_model": video_b_model,
+        "video_a_path": video_a_path,
+        "video_b_path": video_b_path,
+    })
+
+    progress = f"Progress: {len(seen_pair_keys) + 1} / {len(ALL_PAIRS)}"
+
+    return (
+        user_state,
+        gr.update(visible=False),  # Hide login UI
+        gr.update(visible=True),  # Show evaluation UI
+        gr.update(visible=False),  # Hide thank you message
+        video_a_path,
+        video_b_path,
+        progress
+    )
+
+
+# def record_choice(user_state, choice):
+#     """
+#     Records the user's choice, saves it, and loads the next pair.
+#     """
+#     # Unpack user state
+#     email, unseen_pairs, results = user_state[:3]
+#     current_pair_info = user_state[3]
+#
+#     is_baseline_a = current_pair_info["is_baseline_a"]
+#
+#     # Determine which video type was preferred
+#     if choice == "A":
+#         preferred_type = "baseline" if is_baseline_a else "mymethod"
+#     elif choice == "B":
+#         preferred_type = "mymethod" if is_baseline_a else "baseline"
+#     else:  # choice == "None"
+#         preferred_type = "no_preference"
+#     # print("User choice:", choice, "-> preferred_type:", preferred_type)
+#
+#     # Create a result record
+#     result = {
+#         "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+#         "email": email,
+#         "pair_key": current_pair_info["current_pair_key"],
+#         "preferred_type": preferred_type,
+#         "video_a": "baseline" if is_baseline_a else "mymethod",
+#         "video_b": "mymethod" if is_baseline_a else "baseline",
+#     }
+#     results.append(result)
+#
+#     # Save to CSV immediately
+#     df_new = pd.DataFrame([result])
+#     df_new.to_csv(RESULTS_FILE, mode='a', header=not os.path.exists(RESULTS_FILE), index=False)
+#
+#     # Check if there are more pairs
+#     if not unseen_pairs:
+#         # No more pairs, end the session
+#         return user_state, gr.update(visible=False), gr.update(visible=True), None, None, "Complete!"
+#
+#     # Load the next pair
+#     next_pair = unseen_pairs.pop()
+#
+#     # Randomly decide if baseline is video A or B
+#     is_baseline_a = random.choice([True, False])
+#     video_a, video_b = (next_pair[1], next_pair[0]) if is_baseline_a else (next_pair[0], next_pair[1])
+#
+#     # Update the state with the new pair's info
+#     user_state[3] = {"current_pair_key": next_pair[2], "is_baseline_a": is_baseline_a}
+#
+#     progress = f"Progress: {len(ALL_PAIRS) - len(unseen_pairs)} / {len(ALL_PAIRS)}"
+#
+#     return user_state, gr.update(visible=True), gr.update(visible=False), video_a, video_b, progress
+def record_choice(user_state, choice):
+    """
+    Records the user's choice, saves it, and loads the next pair.
+    """
+    # Unpack user state
+    email, unseen_pairs, results = user_state[:3]
+    current = user_state[3]
+
+    video_a_model = current["video_a_model"]
+    video_b_model = current["video_b_model"]
+    video_a_path = current["video_a_path"]
+    video_b_path = current["video_b_path"]
+    pair_key = current["current_pair_key"]
+    scene_name = current.get("scene_name", "")
+
+    # Determine which model was preferred
+    if choice == "A":
+        winner_model, loser_model = video_a_model, video_b_model
+    elif choice == "B":
+        winner_model, loser_model = video_b_model, video_a_model
+    else:  # "None"
+        winner_model, loser_model = "no_preference", "no_preference"
+
+    # Create a result record
+    result = {
+        "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+        "email": email,
+        "pair_key": pair_key,
+        "scene": scene_name,
+        "choice": choice,  # "A" / "B" / "None"
+        "winner_model": winner_model,
+        "loser_model": loser_model,
+        "video_a_model": video_a_model,
+        "video_b_model": video_b_model,
+        "video_a_path": video_a_path,
+        "video_b_path": video_b_path,
+    }
+    results.append(result)
+
+    # Save to CSV immediately
+    df_new = pd.DataFrame([result])
+    df_new.to_csv(RESULTS_FILE, mode='a', header=not os.path.exists(RESULTS_FILE), index=False)
+
+    # Check if there are more pairs
+    if not unseen_pairs:
+        # No more pairs, end the session
+        return user_state, gr.update(visible=False), gr.update(visible=True), None, None, "Complete!"
+
+    # Load the next pair
+    next_pair = unseen_pairs.pop()
+    path_a, path_b, pair_key, model_a_name, model_b_name, scene_name = next_pair
+
+    # Random assignment for the next pair
+    a_first = random.choice([True, False])
+    if a_first:
+        video_a_path, video_b_path = path_a, path_b
+        video_a_model, video_b_model = model_a_name, model_b_name
+    else:
+        video_a_path, video_b_path = path_b, path_a
+        video_a_model, video_b_model = model_b_name, model_a_name
+
+    # Update the state with the new pair's info
+    user_state[3] = {
+        "current_pair_key": pair_key,
+        "scene_name": scene_name,
+        "video_a_model": video_a_model,
+        "video_b_model": video_b_model,
+        "video_a_path": video_a_path,
+        "video_b_path": video_b_path,
+    }
+
+    progress = f"Progress: {len(ALL_PAIRS) - len(unseen_pairs)} / {len(ALL_PAIRS)}"
+
+    return user_state, gr.update(visible=True), gr.update(visible=False), video_a_path, video_b_path, progress
+
+def get_results_file(password):
+    if password == ADMIN_PASSWORD:
+        if os.path.exists(RESULTS_FILE):
+            return gr.File(value=RESULTS_FILE, visible=True)
+        else:
+            gr.Warning("Results file not found. Has anyone submitted an evaluation yet?")
+            return gr.File(visible=False)
+    else:
+        gr.Warning("Incorrect password.")
+        return gr.File(visible=False)
+
+# --- 4. GRADIO UI DEFINITION ---
+with gr.Blocks(theme=gr.themes.Soft(), css="footer {display: none !important}") as demo:
+    # This component holds all the user-specific data during a session
+    user_state = gr.State()
+
+    gr.Markdown("# 🎬 Video Generation Model Evaluation")
+    with gr.Tabs():
+        with gr.TabItem("Evaluation"):
+            gr.Markdown(
+                "Thank you for participating! Please enter your email to begin. You will be shown pairs of videos that loop automatically. Please select the one you prefer or choose 'No Preference'.")
+
+            with gr.Column(visible=True) as login_ui:
+                email_input = gr.Textbox(label="Your Email", placeholder="user@example.com")
+                start_button = gr.Button("Start / Resume Evaluation", variant="primary")
+
+            with gr.Column(visible=False) as eval_ui:
+                progress_text = gr.Markdown(f"Progress: 1 / {len(ALL_PAIRS)}")
+                with gr.Row():
+                    video_a = gr.Video(label="Video A", width=1280, height=704, autoplay=True, loop=True)
+                    video_b = gr.Video(label="Video B", width=1280, height=704, autoplay=True, loop=True)
+                with gr.Row():
+                    choice_a_btn = gr.Button("I prefer Video A", variant="primary")
+                    no_pref_btn = gr.Button("No Preference")
+                    choice_b_btn = gr.Button("I prefer Video B", variant="primary")
+
+            with gr.Column(visible=False) as thanks_ui:
+                thanks_message = gr.Markdown(
+                    "## ✅ Thank You! \n You have completed the evaluation. Your responses have been saved. You can now close this window.")
+
+        # --- Admin tab for downloading results ---
+        with gr.TabItem("Admin"):
+            gr.Markdown("Enter the password to download the current results file.")
+            admin_password = gr.Textbox(label="Password", type="password")
+            download_button = gr.Button("Download Results", variant="primary")
+            results_file_output = gr.File(label="Results File", visible=False)
+
+    # --- 5. CONNECTING UI TO FUNCTIONS ---
+    start_button.click(
+        fn=start_session,
+        inputs=[email_input],
+        outputs=[user_state, login_ui, eval_ui, thanks_ui, video_a, video_b, progress_text]
+    )
+
+    choice_a_btn.click(
+        fn=record_choice,
+        inputs=[user_state, gr.Textbox("A", visible=False)],
+        outputs=[user_state, eval_ui, thanks_ui, video_a, video_b, progress_text]
+    ).then(fn=None, inputs=None, outputs=None, js="() => { window.scrollTo(0, 0); }")
+
+    no_pref_btn.click(
+        fn=record_choice,
+        inputs=[user_state, gr.Textbox("None", visible=False)],
+        outputs=[user_state, eval_ui, thanks_ui, video_a, video_b, progress_text]
+    ).then(fn=None, inputs=None, outputs=None, js="() => { window.scrollTo(0, 0); }")
+
+    choice_b_btn.click(
+        fn=record_choice,
+        inputs=[user_state, gr.Textbox("B", visible=False)],
+        outputs=[user_state, eval_ui, thanks_ui, video_a, video_b, progress_text]
+    ).then(fn=None, inputs=None, outputs=None, js="() => { window.scrollTo(0, 0); }")
+
+    download_button.click(
+        fn=get_results_file,
+        inputs=[admin_password],
+        outputs=[results_file_output]
+    )
+
+# Launch the app
+demo.launch()

requirements.txt

@@ -0,0 +1,3 @@
+gradio
+pandas
+huggingface_hub