init

.gitignore

@@ -0,0 +1,2 @@
+survey-hf-dataset/
+survey-hf-dataset

.vscode/settings.json

@@ -0,0 +1,21 @@
+{
+    "files.watcherExclude": {
+        "**/datadir/**": true,
+        "**/log*/**": true,
+        "**/survey-hf-dataset/**": true
+    },
+    "search.exclude": {
+        "**/datadir/**": true,
+        "**/log*/**": true,
+        "datadir/**": true,
+        "log*/**": true,
+        "**/survey-hf-dataset/**": true
+    },
+    "python.analysis.exclude": [
+        "**/datadir/**",
+        "**/survey-hf-dataset/**"
+    ],
+    "black-formatter.args": [
+        "--line-length=100"
+    ]
+}

README.md

@@ -0,0 +1,13 @@
+---
+title: Video Quality Survey
+emoji: 🧐
+colorFrom: green
+colorTo: yellow
+sdk: gradio
+sdk_version: 4.41.0
+python_version: 3.10.14
+app_file: app.py
+pinned: true
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,7 +1,525 @@
+from argparse import ArgumentParser
+import os, threading, time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pprint import pprint
+import numpy as np
 import gradio as gr
+from huggingface_hub import HfApi
 
-def greet(name):
-    return "Hello " + name + "!!"
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()
+@dataclass
+class VideoCfg:
+    value: str
+    label: str
+    visible: bool
+    interactive: bool = False
+    autoplay: bool = True
+    show_share_button: bool = False
+    show_download_button: bool = False
+    loop: bool = False
+    show_label: bool = True
+    width: int | str | None = None
+
+
+def time_now():
+    return datetime.now(timezone.utc).strftime("%Y-%m-%d_%H-%M-%s")
+
+
+class SurveyEngine:
+    def __init__(self, args):
+        self.args = args
+        self.api = HfApi()
+
+        # pull videos from hf dataset to hf space
+        self.repo_id = "zirui-wang/survey-hf-dataset"
+        self.local_dir = "./survey-hf-dataset"
+        self.api.snapshot_download(self.repo_id, local_dir=self.local_dir, repo_type="dataset")
+
+        # create a file to record the user study
+        response_file_name = f"{time_now()}.csv"
+        self.response_file_path_local = f"{self.local_dir}/response/{response_file_name}"
+        self.response_file_path_remote = f"response/{response_file_name}"
+        os.makedirs(f"{self.local_dir}/response", exist_ok=True)
+        self._update_local_file("timestamp,path1,path2,selection\n", self.response_file_path_local)
+
+        # create a file to record optional feedback
+        feedback_file_name = f"{time_now()}.txt"
+        self.feedback_file_path_local = f"{self.local_dir}/optional_feedback/{feedback_file_name}"
+        self.feedback_file_path_remote = f"optional_feedback/{feedback_file_name}"
+        os.makedirs(f"{self.local_dir}/optional_feedback", exist_ok=True)
+        self._update_local_file("", self.feedback_file_path_local)
+
+        self.video_paths, self.N_prompts, self.N_methods = self._get_all_video_paths()
+
+        self.theme = gr.themes.Default(
+            text_size=gr.themes.sizes.text_lg,
+            spacing_size=gr.themes.sizes.spacing_sm,
+            radius_size=gr.themes.sizes.radius_md,
+        )
+
+        self._start_periodic_sync()
+
+    def _start_periodic_sync(self):
+        def _upload_periodically(path_local, path_remote):
+            while True:
+                print(time_now())
+                print(f"Uploading {path_local}.")
+                try:
+                    self._update_remote_file(path_local, path_remote)
+                except Exception as e:
+                    print(e)
+                time.sleep(args.period_upload)
+
+        def _squash_commits_periodically():
+            while True:
+                print(time_now())
+                print("Squashing commits.")
+                try:
+                    self.api.super_squash_history("zirui-wang/survey-videos", repo_type="dataset")
+                except Exception as e:
+                    print(e)
+                time.sleep(args.period_squash)
+
+        thread_upload_response = threading.Thread(
+            target=_upload_periodically,
+            args=(self.response_file_path_local, self.response_file_path_remote),
+        )
+        thread_upload_response.daemon = True
+        thread_upload_response.start()
+
+        thread_upload_feedback = threading.Thread(
+            target=_upload_periodically,
+            args=(self.feedback_file_path_local, self.feedback_file_path_remote),
+        )
+        thread_upload_feedback.daemon = True
+        thread_upload_feedback.start()
+
+        thread_squash_commits = threading.Thread(target=_squash_commits_periodically)
+        thread_squash_commits.daemon = True
+        thread_squash_commits.start()
+
+    def _get_all_video_paths(self):
+        video_dir = f"{self.local_dir}/video"
+        method_list = sorted(os.listdir(video_dir))
+        video_name_list = sorted(os.listdir(f"{video_dir}/{method_list[0]}"))
+
+        N_prompts = len(video_name_list)
+        N_methods = len(method_list)
+
+        video_paths = []  # (N_prompts, N_methods)
+        for video_name in video_name_list:
+            paths = [os.path.join(video_dir, method, video_name) for method in method_list]
+            video_paths.append(paths)
+        video_paths = np.array(video_paths)
+        return video_paths, N_prompts, N_methods
+
+    def _sample_video_pair(self):
+        # random choose two prompts
+        prompt_ids = np.random.choice(self.N_prompts, 2, replace=False)
+
+        video_pair1 = self.video_paths[prompt_ids[0]]
+        video_pair1 = np.random.permutation(video_pair1)
+
+        video_pair2 = self.video_paths[prompt_ids[1]]
+        video_pair2 = np.random.permutation(video_pair2)
+
+        radio_select_to_path_lut = {
+            "Video 1 👍": str(video_pair1[0]),
+            "Video 2 👍": str(video_pair1[1]),
+            "Video 3 👍": str(video_pair2[0]),
+            "Video 4 👍": str(video_pair2[1]),
+            "Similar 🤔": "Similar",
+            "Seen & Next ⏭️": "Seen",
+        }
+        print("---------------")
+        print(time_now())
+        pprint(radio_select_to_path_lut)
+        return video_pair1, video_pair2, radio_select_to_path_lut
+
+    def _update_local_file(self, message, file_path_local):
+        with open(file_path_local, "a") as f:
+            f.write(message)
+
+    def _update_remote_file(self, file_path_local, file_path_remote):
+        self.api.upload_file(
+            path_or_fileobj=file_path_local,
+            path_in_repo=file_path_remote,
+            repo_id=self.repo_id,
+            repo_type="dataset",
+        )
+
+    def _setup_video(self, path_a, path_b, label_a, label_b, visible):
+        cfg_a = VideoCfg(value=path_a, label=label_a, visible=visible)
+        cfg_b = VideoCfg(value=path_b, label=label_b, visible=visible)
+        video_a = gr.Video(**cfg_a.__dict__)
+        video_b = gr.Video(**cfg_b.__dict__)
+        return video_a, video_b
+
+    def _load_callback(self):
+        video_pair1, video_pair2, radio_select_to_path_lut = self._sample_video_pair()
+        update_video1 = gr.update(value=video_pair1[0])
+        update_video2 = gr.update(value=video_pair1[1])
+        update_video3 = gr.update(value=video_pair2[0])
+        update_video4 = gr.update(value=video_pair2[1])
+        return (
+            video_pair1,
+            video_pair2,
+            radio_select_to_path_lut,
+            update_video1,
+            update_video2,
+            update_video3,
+            update_video4,
+        )
+
+    def _click_select_radio(self):
+        update1 = gr.update(visible=True)
+        return update1
+
+    def _click_button_confirm1(self, radio_select, radio_select_to_path_lut):
+        # update response file with the acutal file path
+        selected_path = radio_select_to_path_lut[radio_select]
+        path1 = radio_select_to_path_lut["Video 1 👍"]
+        path2 = radio_select_to_path_lut["Video 2 👍"]
+        message = f"{time_now()},{path1},{path2},{selected_path}\n"
+        self._update_local_file(message, self.response_file_path_local)
+
+        confirm_message = f"""
+            Your selection was:  
+            <span style="font-size:20px; color:orange "> {radio_select} </span> \n\n
+        """
+        # display confirm message
+        update_md_confirm1 = gr.update(visible=True, value=confirm_message)
+
+        # hide the radio and button for video 1-2
+        update_button_confirm1 = gr.update(visible=False)
+        update_ratio_select1 = gr.update(visible=False)
+
+        # show video 3-4 and radio
+        update_md_pair_34 = gr.update(visible=True)
+        update_video_3 = gr.update(visible=True)
+        update_video_4 = gr.update(visible=True)
+        update_radio_select2 = gr.update(visible=True)
+        return (
+            update_md_confirm1,
+            update_button_confirm1,
+            update_ratio_select1,
+            update_md_pair_34,
+            update_video_3,
+            update_video_4,
+            update_radio_select2,
+        )
+
+    def _click_button_confirm2(self, radio_select, radio_select_to_path_lut):
+        # update response file with the acutal file path
+        selected_path = radio_select_to_path_lut[radio_select]
+        path1 = radio_select_to_path_lut["Video 3 👍"]
+        path2 = radio_select_to_path_lut["Video 4 👍"]
+        message = f"{time_now()},{path1},{path2},{selected_path}\n"
+        self._update_local_file(message, self.response_file_path_local)
+
+        confirm_message = f"""
+            Your selection was:&nbsp;&nbsp;
+            <span style="font-size:20px; color:orange "> {radio_select} </span> \n\n
+            ## Study Done! 
+            Click the button below 🎲 if you'd like to evaluate another set. \n\n
+            You can exit this study by closing this page. 
+            For more details about this study, click 
+            [here](https://huggingface.co/spaces/zirui-wang/toy/blob/main/detail.md).
+        """
+
+        # display confirm message
+        update_md_confirm2 = gr.update(visible=True, value=confirm_message)
+
+        # hide the radio and button for video 3-4
+        update_button_confirm2 = gr.update(visible=False)
+        update_radio_select2 = gr.update(visible=False)
+
+        # show button_new
+        update_button_new = gr.update(visible=True)
+
+        # show textbox and button for optional feedback
+        update_textbox_optional = gr.update(visible=True)
+        update_button_submit_optional = gr.update(visible=True)
+        return (
+            update_md_confirm2,
+            update_button_confirm2,
+            update_radio_select2,
+            update_button_new,
+            update_textbox_optional,
+            update_button_submit_optional,
+        )
+
+    def _click_button_new(self):
+        video_pair1, video_pair2, radio_select_to_path_lut = self._sample_video_pair()
+
+        update_video1 = gr.update(value=video_pair1[0])
+        update_video2 = gr.update(value=video_pair1[1])
+        update_radio_select1 = gr.update(visible=True, value=None)
+        update_button_confirm1 = gr.update(visible=False)
+        update_md_confirm1 = gr.update(visible=False)
+
+        update_md_pair_34 = gr.update(visible=False)
+        update_video3 = gr.update(value=video_pair2[0], visible=False)
+        update_video4 = gr.update(value=video_pair2[1], visible=False)
+        update_radio_select2 = gr.update(visible=False, value=None)
+        update_button_confirm2 = gr.update(visible=False)
+        update_md_confirm2 = gr.update(visible=False)
+
+        update_button_new = gr.update(visible=False)
+
+        update_textbox_optional = gr.update(visible=False, value=None)
+        update_button_submit_optional = gr.update(visible=False)
+        update_md_optional_feedback = gr.update(visible=False)
+        return (
+            video_pair1,
+            video_pair2,
+            radio_select_to_path_lut,
+            update_video1,
+            update_video2,
+            update_radio_select1,
+            update_button_confirm1,
+            update_md_confirm1,
+            update_md_pair_34,
+            update_video3,
+            update_video4,
+            update_radio_select2,
+            update_button_confirm2,
+            update_md_confirm2,
+            update_button_new,
+            update_textbox_optional,
+            update_button_submit_optional,
+            update_md_optional_feedback,
+        )
+
+    def _click_button_optional_feedback(self, textbox_optional_feedback):
+        if textbox_optional_feedback == "":
+            return gr.skip(), gr.skip()
+        message = f"{time_now()}\n{textbox_optional_feedback}\n\n"
+        self._update_local_file(message, self.feedback_file_path_local)
+
+        update_md_optional_feedback = gr.update(visible=True)
+        update_button_submit_optional = gr.update(visible=False)
+        return update_md_optional_feedback, update_button_submit_optional
+
+    def main(self):
+        # read in md file
+        with open("start.md", "r") as f:
+            md_start = f.read()
+
+        with gr.Blocks(theme=self.theme, title="Video Quality User Study") as demo:
+
+            # set up session states
+            video_pair1 = gr.State(value=["path1", "path2"], time_to_live=900)
+            video_pair2 = gr.State(value=["path2", "path4"], time_to_live=900)
+            radio_select_to_path_lut = gr.State(value={}, time_to_live=900)  # hold a dict
+
+            # set up layout
+            with gr.Column():
+                gr.Markdown(md_start)
+
+                # a debug button
+                if self.args.debug:
+
+                    def _click_button_debug(video_pair1, video_pair2, radio_select_to_path_lut):
+                        print(f"video_pair1: {video_pair1}")
+                        print(f"video_pair2: {video_pair2}")
+                        print(f"radio_select_to_path_lut: {radio_select_to_path_lut}")
+
+                    button_debug = gr.Button("debug", variant="primary", scale=1)
+                    button_debug.click(
+                        _click_button_debug,
+                        inputs=[video_pair1, video_pair2, radio_select_to_path_lut],
+                    )
+
+                # ---------------- video 1-2 ----------------
+                gr.Markdown("## Video Pair 1")
+                with gr.Row():
+                    video1, video2 = self._setup_video(
+                        path_a=video_pair1.value[0],
+                        path_b=video_pair1.value[1],
+                        label_a="Video 1",
+                        label_b="Video 2",
+                        visible=True,
+                    )
+                with gr.Row():
+                    radio_select1 = gr.Radio(
+                        choices=["Video 1 👍", "Video 2 👍", "Similar 🤔", "Seen & Next ⏭️"],
+                        label="Your Preference:",
+                        scale=2,
+                    )
+
+                    button_confirm1 = gr.Button(
+                        value="Confirm",
+                        variant="primary",
+                        scale=1,
+                        visible=False,
+                    )
+
+                md_confirm1 = gr.Markdown(visible=False, inputs=button_confirm1)
+
+                # ---------------- video 3-4 ----------------
+                md_pair_34 = gr.Markdown("## Video Pair 2", visible=False)
+                with gr.Row():
+                    video3, video4 = self._setup_video(
+                        path_a=video_pair2.value[0],
+                        path_b=video_pair2.value[1],
+                        label_a="Video 3",
+                        label_b="Video 4",
+                        visible=False,
+                    )
+                with gr.Row():
+                    radio_select2 = gr.Radio(
+                        choices=["Video 3 👍", "Video 4 👍", "Similar 🤔", "Seen & Next ⏭️"],
+                        label="Your Preference:",
+                        scale=2,
+                        visible=False,
+                    )
+
+                    button_confirm2 = gr.Button(
+                        value="Confirm",
+                        variant="primary",
+                        scale=1,
+                        visible=False,
+                    )
+
+                md_confirm2 = gr.Markdown(visible=False, inputs=button_confirm2)
+
+                # ---------------- new button ----------------
+                button_new = gr.Button(
+                    value="New One 🎲",
+                    variant="primary",
+                    visible=False,
+                )
+
+                # ---------------- optional feedback ----------------
+                with gr.Row():
+                    textbox_optional_feedback = gr.Textbox(
+                        label="Optional Comments:",
+                        placeholder="Type here...",
+                        lines=1,
+                        scale=4,
+                        interactive=True,
+                        visible=False,
+                    )
+                    button_submit_optional_feedback = gr.Button(
+                        value="Submit Comments",
+                        variant="secondary",
+                        scale=1,
+                        visible=False,
+                    )
+
+                md_optional_feedback = gr.Markdown(
+                    "Thank you for providing additional comments! 🙏",
+                    visible=False,
+                )
+
+            # set up callbacks
+            radio_select1.select(
+                self._click_select_radio,
+                trigger_mode="once",
+                outputs=button_confirm1,
+            )
+
+            button_confirm1.click(
+                self._click_button_confirm1,
+                trigger_mode="once",
+                inputs=[radio_select1, radio_select_to_path_lut],
+                outputs=[
+                    md_confirm1,
+                    button_confirm1,
+                    radio_select1,
+                    md_pair_34,
+                    video3,
+                    video4,
+                    radio_select2,
+                ],
+            )
+
+            radio_select2.select(
+                self._click_select_radio,
+                trigger_mode="once",
+                outputs=button_confirm2,
+            )
+
+            button_confirm2.click(
+                self._click_button_confirm2,
+                trigger_mode="once",
+                inputs=[radio_select2, radio_select_to_path_lut],
+                outputs=[
+                    md_confirm2,
+                    button_confirm2,
+                    radio_select2,
+                    button_new,
+                    textbox_optional_feedback,
+                    button_submit_optional_feedback,
+                ],
+            )
+
+            button_new.click(
+                self._click_button_new,
+                trigger_mode="once",
+                outputs=[
+                    video_pair1,
+                    video_pair2,
+                    radio_select_to_path_lut,
+                    video1,
+                    video2,
+                    radio_select1,
+                    button_confirm1,
+                    md_confirm1,
+                    md_pair_34,
+                    video3,
+                    video4,
+                    radio_select2,
+                    button_confirm2,
+                    md_confirm2,
+                    button_new,
+                    textbox_optional_feedback,
+                    button_submit_optional_feedback,
+                    md_optional_feedback,
+                ],
+            )
+
+            button_submit_optional_feedback.click(
+                self._click_button_optional_feedback,
+                trigger_mode="once",
+                inputs=textbox_optional_feedback,
+                outputs=[md_optional_feedback, button_submit_optional_feedback],
+            )
+
+            demo.load(
+                self._load_callback,
+                outputs=[
+                    video_pair1,
+                    video_pair2,
+                    radio_select_to_path_lut,
+                    video1,
+                    video2,
+                    video3,
+                    video4,
+                ],
+            )
+        demo.launch(share=self.args.share, show_api=False)
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("--debug", action="store_true")
+    parser.add_argument("--share", action="store_true")
+    parser.add_argument(
+        "--period_upload",
+        type=int,
+        default=300,
+        help="Period (sec) to upload response files in seconds, 300sec=5min",
+    )
+    parser.add_argument(
+        "--period_squash",
+        type=int,
+        default=43200,
+        help="Period (sec) to squash commits in seconds, 43200sec=12h",
+    )
+    args = parser.parse_args()
+
+    survey_engine = SurveyEngine(args)
+    survey_engine.main()

detail.md

@@ -0,0 +1,18 @@
+# A Video Quality User Study for Video Diffusion Models
+Are you interested in the latest developments in video diffusion models? We invite you to participate in our user study, where you can watch a few video samples and provide your ratings. By participating, you will have the chance to experience the latest video diffusion models firsthand. The study should take as little as 1 minute to complete.
+
+### What Are We Evaluating?
+This study aims to assess the quality of videos generated by two different diffusion-based generative models. We are focusing on key aspects such as spatial and temporal consistency, as well as multi-view coherence. While these models produce impressive results, they can also generate unexpected artifacts, such as strange morphing effects, abrupt changes in subjects, or unusual occlusions and geometries. We have created 50 videos for this study, each video is generated using identical text prompts across these two models, and we may expand the study to include more videos and additional models in the future. 
+
+| ![GIF 1](example/good.gif) | ![GIF 2](example/bad.gif) |
+|:---------------------------------------------------------------------------:|:---------------------------------------------------------------------------:|
+| 🙂 Pretty Good                                                           |  🤔 Subject changed and weird books.                                                              |
+
+### Your Contribution Matters
+The results of this study, along with the videos and prompts used, will be made publicly available as part of a dataset. This dataset will contribute to a research report aimed at advancing the understanding of video diffusion models. Your feedback is crucial in helping us identify and understand these phenomena. Rest assured, the study is fully anonymous, and no personal information will be collected.
+
+### How to Participate
+You can join the study using any device, whether it is a phone, tablet, laptop, or desktop computer. For the best experience, we recommend using a larger screen to observe potential artifacts more clearly. Please note that some mobile devices may have energy-saving settings that prevent videos from autoplaying; if this happens, simply click on the video boxes to play them manually.
+
+Your participation is invaluable to us. Thank you for contributing to the advancement of video diffusion technology. 
+Feel free to contact me Zirui Wang (zirui dot wang @eng.ox.ac.uk) for additional feedbacks.

start.md

@@ -0,0 +1,6 @@
+# 🧐 Video Quality Study
+🙏 Thank you for participating our study! This study should take ~1 minute.
+
+💼 **Task: You'll watch 2 pairs of videos. For each pair, select the video that has better spatial and temporal consistency, as well as multi-view coherence, and less noise.**
+
+🚗 On mobile devices: you might need to manually 🕹️ **click**  video boxes to play videos.

sync_dataset.py

@@ -0,0 +1,59 @@
+from argparse import ArgumentParser
+from huggingface_hub import HfApi
+
+
+def upload_videos(api, upload_dir):
+    api.upload_folder(
+        folder_path=upload_dir,
+        repo_id="zirui-wang/survey-videos",
+        repo_type="dataset",
+    )
+
+
+def delete_response(api):
+    api.delete_folder("response", "zirui-wang/survey-videos", repo_type="dataset")
+
+
+def delete_optional_feedback(api):
+    api.delete_folder("optional_feedback", "zirui-wang/survey-videos", repo_type="dataset")
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--upload_dir",
+        type=str,
+        default="~/projects/mview/storage/data/video_diffusion/text-to-video-debug-anonymised",
+    )
+    parser.add_argument("--upload_videos", type=eval, default=False, choices=[True, False])
+    parser.add_argument("--delete_response", type=eval, default=False, choices=[True, False])
+    parser.add_argument(
+        "--delete_optional_feedback", type=eval, default=False, choices=[True, False]
+    )
+    args = parser.parse_args()
+
+    api = HfApi()
+    if args.upload_videos:
+        upload_videos(api, args.upload_dir)
+
+    if args.delete_response:
+        answer = input("Are you sure you want to delete the response folder? (y/n)")
+        if answer == "y":
+            try:
+                delete_response(api)
+            except Exception as e:
+                print(e)
+
+    if args.delete_optional_feedback:
+        answer = input("Are you sure you want to delete the optional_feedback folder? (y/n)")
+        if answer == "y":
+            try:
+                delete_optional_feedback(api)
+            except Exception as e:
+                print(e)
+
+    api.super_squash_history("zirui-wang/survey-videos", repo_type="dataset")
+
+
+if __name__ == "__main__":
+    main()