human eval3

CONTROLS_file_pairs.csv

@@ -0,0 +1,11 @@
+file_name,vista_id,gem_id
+1,1051245161,1051244884
+11,1051245233,1051244979
+13,1051245253,1051245004
+17,1051245269,1051245040
+2,1051245188,1051244923
+21,1051245289,1051245077
+22,1051245311,1051245091
+25,1051245334,1051245112
+27,1051245352,1051245139
+3,1051245204,1051244950

app.py

@@ -12,7 +12,7 @@ import pandas as pd
 
 load_dotenv()
 
-video_pairs = pd.read_csv('file_pairs.csv')[['file_name', 'vista_id', 'gem_id']].values.tolist()
+video_pairs = pd.read_csv('file_pairs.csv')[['file_name', 'vista_id', 'gem_id', 'rgb_id']].values.tolist()
 random.seed(42)
 random.shuffle(video_pairs)
 
@@ -45,8 +45,9 @@ def get_video_pair(state):
     shuffled_pairs = state['shuffled_pairs']
     user_votes = state['user_votes']
     while pair_index < len(shuffled_pairs):
-        video_name, vista_id, gem_id = shuffled_pairs[pair_index]
+        video_name, vista_id, gem_id, rgb_id = shuffled_pairs[pair_index]
         pair_key = f"{vista_id}_{gem_id}"
+        state['rgb_id'] = rgb_id
         if pair_key not in user_votes:
             # Randomize left-right positions
             if random.choice([True, False]):
@@ -125,6 +126,7 @@ def update_interface(responses, state):
             gr.update(visible=False),  # video_column
             gr.update(value=""),       # video1
             gr.update(value=""),       # video2
+            gr.update(value=""),       # rgb_video
             gr.update(visible=False),  # question_column
             gr.update(visible=False),  # button_row
             output_message,            # output
@@ -136,12 +138,14 @@ def update_interface(responses, state):
     else:
         video1_html = generate_video_html(video1_url)
         video2_html = generate_video_html(video2_url)
+        rgb_html = generate_video_html(get_embed_link(state['rgb_id']))
         # Update videos and reset questions
         return (
             gr.update(visible=True),   # video_column
             gr.update(visible=True),   # video_column
             gr.update(value=video1_html),  # video1
             gr.update(value=video2_html),  # video2
+            gr.update(value=rgb_html),       # rgb_video
             gr.update(visible=True),   # question_column
             gr.update(visible=True),   # button_row
             "",                        # output
@@ -172,6 +176,7 @@ def authenticate_user(email, state):
             gr.update(visible=False),  # video_column
             gr.update(value=""),       # video1
             gr.update(value=""),       # video2
+            gr.update(value=""),       # rgb_video
             gr.update(visible=False),  # question_column
             gr.update(visible=False),  # button_row
             "",                      # output
@@ -208,6 +213,7 @@ def authenticate_user(email, state):
                 gr.update(visible=False),  # video_column
                 gr.update(value=""),       # video1
                 gr.update(value=""),       # video2
+                gr.update(value=""),       # rgb_video
                 gr.update(visible=False),  # question_column
                 gr.update(visible=False),  # button_row
                 output_message,         # output
@@ -216,6 +222,7 @@ def authenticate_user(email, state):
         else:
             video1_html = generate_video_html(video1_url)
             video2_html = generate_video_html(video2_url)
+            rgb_html = generate_video_html(get_embed_link(state['rgb_id']))
             return (
                 gr.update(visible=False),  # email_input
                 gr.update(visible=False),  # submit_email
@@ -224,6 +231,7 @@ def authenticate_user(email, state):
                 gr.update(visible=True),   # video_column
                 gr.update(value=video1_html),  # video1
                 gr.update(value=video2_html),  # video2
+                gr.update(value=rgb_html),       # rgb_video
                 gr.update(visible=True),    # question_column
                 gr.update(visible=True),    # button_row
                 "",                         # output
@@ -235,12 +243,13 @@ with gr.Blocks() as demo:
 
     gr.Markdown(
         """
-        You'll be seeing pairs of videos and will be asked to compare them.
-        The videos were generated by AI to move a car from one point to another (green boxes in the video).
-        Choose the video that shows more realistic motion of the car from a box to another.
+        You'll be seeing three videos per question.
+        At the top, you'll see a video with the RGB view.
+        Below, you'll see two depth videos.
+        You'll be asked to select which depth video seems to be better quality with respect to the RGB video.
 
         **There are 10 videos in total.**
-        **If both videos are the same, select "No preference".**
+        **Avoid "No preference" answers as much as possible.**
         """
     )
 
@@ -250,26 +259,31 @@ with gr.Blocks() as demo:
     email_output = gr.Markdown()
 
     # Video components (initially hidden)
-    with gr.Row():
-        with gr.Column(visible=False) as video1_column:
-            gr.Markdown("### Video 1")
-            video1 = gr.HTML()
+    with gr.Column():
+        with gr.Column() as video1_column:
+            gr.Markdown("### RGB Video")
+            rgb_video = gr.HTML()
 
-        with gr.Column(visible=False) as video2_column:
-            gr.Markdown("### Video 2")
-            video2 = gr.HTML()
+        with gr.Row():
+            with gr.Column(visible=False) as video1_column:
+                gr.Markdown("### Video 1")
+                video1 = gr.HTML()
+
+            with gr.Column(visible=False) as video2_column:
+                gr.Markdown("### Video 2")
+                video2 = gr.HTML()
 
     # Questions (initially hidden)
     with gr.Column(visible=False) as question_column:
         gr.Markdown("## Please answer the following questions:")
         q1 = gr.Radio(
             choices=["Video 1", "Video 2", "No preference"],
-            label="1. Which video shows more natural or consistent motion?",
+            label="1. Which depth video is more aligned with RGB?",
             type="value"
         )
         q2 = gr.Radio(
             choices=["Video 1", "Video 2", "No preference"],
-            label="2. Which video follows the controls better?",
+            label="2. Which depth video is more consistent and has less flickering?",
             type="value"
         )
 
@@ -295,6 +309,7 @@ with gr.Blocks() as demo:
             video2_column,        # Update video_column
             video1,           # Update video1
             video2,           # Update video2
+            rgb_video,
             question_column,  # Update question_column
             button_row,       # Update button_row
             output,           # Update output message
@@ -329,6 +344,7 @@ with gr.Blocks() as demo:
             video2_column,    # Update video_column
             video1,           # Update video1
             video2,           # Update video2
+            rgb_video,
             question_column,  # Update question_column
             button_row,       # Update button_row
             output,           # Update output message

file_pairs.csv

@@ -1,11 +1,31 @@
-file_name,vista_id,gem_id
-1,1051245161,1051244884
-11,1051245233,1051244979
-13,1051245253,1051245004
-17,1051245269,1051245040
-2,1051245188,1051244923
-21,1051245289,1051245077
-22,1051245311,1051245091
-25,1051245334,1051245112
-27,1051245352,1051245139
-3,1051245204,1051244950
+file_name,vista_id,gem_id,rgb_id
+depth_0,1051869114,1051868252,1051868659
+depth_1,1051869131,1051868261,1051868677
+depth_10,1051869294,1051868402,1051868797
+depth_11,1051869311,1051868422,1051868815
+depth_12,1051869328,1051868431,1051868834
+depth_13,1051869340,1051868440,1051868848
+depth_14,1051869351,1051868451,1051868862
+depth_15,1051869373,1051868464,1051868881
+depth_16,1051869389,1051868477,1051868892
+depth_17,1051869407,1051868495,1051868912
+depth_18,1051869418,1051868506,1051868922
+depth_19,1051869430,1051868515,1051868936
+depth_2,1051869155,1051868274,1051868693
+depth_20,1051869443,1051868536,1051868954
+depth_21,1051869464,1051868550,1051868968
+depth_22,1051869476,1051868561,1051868988
+depth_23,1051869491,1051868570,1051869003
+depth_24,1051869507,1051868582,1051869021
+depth_25,1051869527,1051868595,1051869033
+depth_26,1051869550,1051868603,1051869043
+depth_27,1051869570,1051868613,1051869061
+depth_28,1051869584,1051868631,1051869073
+depth_3,1051869166,1051868288,1051868710
+depth_4,1051869183,1051868302,1051868726
+depth_5,1051869203,1051868323,1051868745
+depth_6,1051869219,1051868339,1051868756
+depth_7,1051869243,1051868357,1051868771
+depth_8,1051869262,1051868375,1051868781
+depth_9,1051869278,1051868387,1051868789
+depth_99,1051869604,1051868642,1051869092

get_vimeo.py

@@ -13,8 +13,12 @@ v = vimeo.VimeoClient(
 user_id = 'pedroooombr'  # Replace with the actual user ID
 endpoint = f'/users/{user_id}/videos'
 # Make the GET request
-response1 = v.get(f'{endpoint}?per_page=50').json()
+response1 = v.get(f'{endpoint}?per_page=100').json()
+# response2 = v.get(f'{endpoint}?per_page=100&page=2').json()
+# response3 = v.get(f'{endpoint}?per_page=100&page=3').json()
 response = response1
+# response['data'].extend(response2['data'])
+# response['data'].extend(response3['data'])
 
 gem_videos = [
     ("_".join(v['name'].split("_")[1:]), v['uri'].split("/")[-1]) for v in response['data'] if v['name'].startswith('GEM')
@@ -24,7 +28,11 @@ vista_videos = [
     ("_".join(v['name'].split("_")[1:]), v['uri'].split("/")[-1]) for v in response['data'] if v['name'].startswith('Vista')
 ]
 
-print(len(gem_videos), len(vista_videos))
+rgb_videos = [
+    ("_".join(v['name'].split("_")[1:]), v['uri'].split("/")[-1]) for v in response['data'] if v['name'].startswith('rgb_')
+]
+
+print(len(gem_videos), len(vista_videos), len(rgb_videos))
 assert len(gem_videos) == len(vista_videos)
 
 # Match files by title and create a csv
@@ -32,10 +40,11 @@ data = []
 
 for vista_file in vista_videos:
     for gem_file in gem_videos:
-        if vista_file[0] == gem_file[0]:
-            data.append((vista_file[0], vista_file[1], gem_file[1]))
+        for rgb_file in rgb_videos:
+            if vista_file[0] == gem_file[0] and vista_file[0] == "depth_" + rgb_file[0]:
+                data.append((vista_file[0], vista_file[1], gem_file[1], rgb_file[1]))
 
 # sort by file_name
 data.sort(key=lambda x: x[0])
-df = pd.DataFrame(data, columns=['file_name', 'vista_id', 'gem_id'])
+df = pd.DataFrame(data, columns=['file_name', 'vista_id', 'gem_id', 'rgb_id'])
 df.to_csv('file_pairs.csv', index=False)

results.csv

@@ -1,3 +1,3 @@
 category,gem,vista,no_preference
-q1,10,0,0
-q2,8,0,2
+q1,59,8,11
+q2,52,12,14