Update app.py

app.py

@@ -20,11 +20,23 @@ HF_INPUT_DATASET_URL_COLUMN = os.getenv("HF_INPUT_DATASET_URL_COLUMN")
 HF_OUTPUT_DATASET = os.getenv("HF_OUTPUT_DATASET")
 HF_OUTPUT_DATASET_DIR = os.getenv("HF_OUTPUT_DATASET_DIR")
 INSTRUCTIONS = """
-# Pairwise Model Output Labeling
+# Pairwise Model Output Labeling Space
+
+### How-to:
+
+* Duplicate this Space
+* Add a `HF_TOKEN` secret in the Space settings (it saves the results to a given dataset, so it requires write access)
+* Set the environment variables in the [`.env`](https://huggingface.co/spaces/saridormi/labeling-template/blob/main/.env) file to point to correct datasets/columns/paths for your input and output data
+* Adjust `INSTRUCTIONS` (it's what's rendered here) and `SAVE_EVERY_N_EXAMPLES` (how often should answers be saved to the output dataset) in the [`app.py`](https://huggingface.co/spaces/saridormi/labeling-template/blob/main/app.py) file
+
+Done! 💛
+
+### About:
+
 Please compare the two model outputs shown below and select which one you think is better.
 - Choose "A is better" if the output from Model A (left) is superior
 - Choose "B is better" if the output from Model B (right) is superior
-- Choose "Tie" if you think they are equally good or bad
+- Choose "Tie" if they are equally good or bad
 - Choose "Can't choose" if you cannot make a determination
 """
 SAVE_EVERY_N_EXAMPLES = 5
@@ -34,11 +46,12 @@ class PairwiseLabeler:
     def __init__(self):
         self.current_index = defaultdict(int)
         self.results = defaultdict(list)
-        self.df = self.read_hf_dataset()
-        self.has_url_column = HF_INPUT_DATASET_URL_COLUMN and HF_INPUT_DATASET_URL_COLUMN in self.df.columns
+        self._df = self.read_hf_dataset()
+        self.df = {}
+        self.has_url_column = HF_INPUT_DATASET_URL_COLUMN and HF_INPUT_DATASET_URL_COLUMN in self._df.columns
     
     def __len__(self):
-        return len(self.df)
+        return len(self._df)
 
     def read_hf_dataset(self) -> pd.DataFrame:
         try:
@@ -69,13 +82,16 @@ class PairwiseLabeler:
             return pd.DataFrame(sample_data)
     
     def get_current_pair(self, session_id):
-        if self.current_index[session_id] >= len(self.df):
+        if session_id not in self.df:
+            self.df[session_id] = self._df.sample(frac=1).reset_index(drop=True)
+        
+        if self.current_index[session_id] >= len(self.df[session_id]):
             if self.has_url_column:
                 return None, None, None, None
             else:
                 return None, None, None
         
-        item = self.df.iloc[self.current_index[session_id]]
+        item = self.df[session_id].iloc[self.current_index[session_id]]
         item_id = item.get(HF_INPUT_DATASET_ID_COLUMN, f"item_{self.current_index[session_id]}")
         left_text = item.get(HF_INPUT_DATASET_COLUMN_A, "")
         right_text = item.get(HF_INPUT_DATASET_COLUMN_B, "")
@@ -93,11 +109,6 @@ class PairwiseLabeler:
             else:
                 return item_id, left_text, right_text, self.current_index[session_id]
         
-        # Get the current URL if available
-        current_url = None
-        if self.has_url_column:
-            current_url = self.df.iloc[self.current_index[session_id]].get(HF_INPUT_DATASET_URL_COLUMN, "")
-            
         # Record the judgment
         result = {
             "item_id": item_id,
@@ -105,7 +116,7 @@ class PairwiseLabeler:
             "timestamp": datetime.datetime.now().isoformat(),
             "labeler_id": session_id
         }
-        
+  
         self.results[session_id].append(result)
         
         # Move to next item
@@ -158,27 +169,44 @@ class PairwiseLabeler:
 # Initialize the labeler
 labeler = PairwiseLabeler()
 
-# Create a unique session ID
-def create_new_session():
+# Create a default session ID
+def create_default_session():
     return str(uuid.uuid4())[:8]
 
-with gr.Blocks() as app:
+# Create a demo session ID for initial display
+demo_session = create_default_session()
+demo_item_id, demo_left_text, demo_right_text = None, "", ""
+demo_url = None
+if labeler.has_url_column:
+    demo_item_id, demo_left_text, demo_right_text, demo_url = labeler.get_current_pair(demo_session)
+else:
+    demo_item_id, demo_left_text, demo_right_text = labeler.get_current_pair(demo_session)
+
+with gr.Blocks(css="footer {visibility: hidden}") as app:
     # State for the session ID
     session_id = gr.State(value=None)
+    is_session_started = gr.State(value=False)
     
-    # The actual interface components will be created here
+    # Header with instructions
     gr.Markdown(INSTRUCTIONS)
     
+    # Session and progress bar in one line
+    user_session_id = gr.Textbox(
+        placeholder="Enter your unique ID or leave blank for random",
+        label="Session ID",
+        scale=2
+    )
+    start_btn = gr.Button("Start Session", variant="primary", scale=1)
+    
     # URL display component - only shown if URL column is defined
-    url_display = None
     if labeler.has_url_column:
-        url_display = gr.HTML(label="Reference URL")
-
-    session_id_display = gr.Textbox(label="Session Information", interactive=False)
+        url_display = gr.HTML(label="Reference URL", value="", visible=True)
     
+    # Main content area (shown with placeholder text)
     with gr.Row():
         with gr.Column():
             left_output = gr.Textbox(
+                value=demo_left_text,
                 label="Model A Output",
                 lines=10,
                 interactive=False
@@ -186,68 +214,109 @@ with gr.Blocks() as app:
         
         with gr.Column():
             right_output = gr.Textbox(
+                value=demo_right_text,
                 label="Model B Output", 
                 lines=10,
                 interactive=False
             )
     
-    item_id = gr.Textbox(visible=False)
+    item_id = gr.Textbox(value=demo_item_id, visible=False)
     
+    # Buttons row (initially disabled)
     with gr.Row():
-        left_btn = gr.Button("⬅️ A is better", variant="primary")
-        right_btn = gr.Button("➡️ B is better", variant="primary")
-        tie_btn = gr.Button("🤝 Tie", variant="primary")
-        cant_choose_btn = gr.Button("🤔 Can't choose")
-    
-    current_sample_sld = gr.Slider(minimum=0, maximum=len(labeler), step=1,
-                                  interactive=False,
-                                  label='sample_ind',
-                                  info=f"Samples labeled (out of {len(labeler)})",
-                                  show_label=False,
-                                  container=False,
-                                  scale=5)
+        left_btn = gr.Button("⬅️ A is better", variant="primary", interactive=False)
+        right_btn = gr.Button("➡️ B is better", variant="primary", interactive=False)
+        tie_btn = gr.Button("🤝 Tie", variant="primary", interactive=False)
+        cant_choose_btn = gr.Button("🤔 Can't choose", interactive=False)
     
+    # Progress slider (initially disabled)
+    current_sample_sld = gr.Slider(
+        minimum=0, 
+        maximum=len(labeler), 
+        step=1,
+        value=0,
+        interactive=False,
+        label='Progress',
+        show_label=False
+    )
+
     # Initialize the session and get the first pair
-    def init_session():
-        new_session_id = create_new_session()
+    def init_session(entered_id):
+        # Use entered ID or generate a new one if empty
+        session_id_value = entered_id.strip() if entered_id and entered_id.strip() else create_default_session()
         
+        # Get the initial data for this session ID
         if labeler.has_url_column:
-            initial_id, initial_left, initial_right, initial_url = labeler.get_current_pair(new_session_id)
+            initial_id, initial_left, initial_right, initial_url = labeler.get_current_pair(session_id_value)
             url_html = f'<a href="{initial_url}" target="_blank">{initial_url}</a>' if initial_url else ""
             
             return (
-                new_session_id,                            # session_id state
-                f"Session ID: {new_session_id}",           # session_id_display
-                url_html,                                  # url_display
-                initial_left,                              # left_output
-                initial_right,                             # right_output
-                initial_id,                                # item_id
-                labeler.current_index[new_session_id]      # current_sample_sld
+                session_id_value,                 # session_id state
+                True,                             # is_session_started state
+                labeler.current_index[session_id_value], # current_sample_sld
+                url_html,                         # url_display
+                initial_left,                     # left_output
+                initial_right,                    # right_output
+                initial_id,                       # item_id
+                gr.update(interactive=True),      # left_btn
+                gr.update(interactive=True),      # right_btn
+                gr.update(interactive=True),      # tie_btn
+                gr.update(interactive=True)       # cant_choose_btn
             )
         else:
-            initial_id, initial_left, initial_right = labeler.get_current_pair(new_session_id)
+            initial_id, initial_left, initial_right = labeler.get_current_pair(session_id_value)
+            
+            # Create session info text
+            progress_text = f"Session: {session_id_value} | {labeler.current_index[session_id_value] + 1}/{len(labeler)}"
             
             return (
-                new_session_id,                            # session_id state
-                f"Session ID: {new_session_id}",           # session_id_display
-                initial_left,                              # left_output
-                initial_right,                             # right_output
-                initial_id,                                # item_id
-                labeler.current_index[new_session_id]      # current_sample_sld
+                session_id_value,                 # session_id state
+                True,                             # is_session_started state
+                labeler.current_index[session_id_value], # current_sample_sld
+                initial_left,                     # left_output
+                initial_right,                    # right_output
+                initial_id,                       # item_id
+                gr.update(interactive=True),      # left_btn
+                gr.update(interactive=True),      # right_btn
+                gr.update(interactive=True),      # tie_btn
+                gr.update(interactive=True)       # cant_choose_btn
             )
     
-    # Run the initialization when the app loads
+    # Connect the start button
     if labeler.has_url_column:
-        app.load(
+        start_btn.click(
             init_session,
-            inputs=None,
-            outputs=[session_id, session_id_display, url_display, left_output, right_output, item_id, current_sample_sld]
+            inputs=[user_session_id],
+            outputs=[
+                session_id, 
+                is_session_started,
+                current_sample_sld,
+                url_display, 
+                left_output, 
+                right_output, 
+                item_id,
+                left_btn,
+                right_btn,
+                tie_btn,
+                cant_choose_btn
+            ]
         )
     else:
-        app.load(
+        start_btn.click(
             init_session,
-            inputs=None,
-            outputs=[session_id, session_id_display, left_output, right_output, item_id, current_sample_sld]
+            inputs=[user_session_id],
+            outputs=[
+                session_id, 
+                is_session_started,
+                current_sample_sld,
+                left_output, 
+                right_output, 
+                item_id,
+                left_btn,
+                right_btn,
+                tie_btn,
+                cant_choose_btn
+            ]
         )
     
     def judge_left(session_id, item_id, left_text, right_text):
@@ -263,17 +332,29 @@ with gr.Blocks() as app:
         return judge("Can't choose", session_id, item_id, left_text, right_text)
     
     def judge(choice, session_id, item_id, left_text, right_text):
+        if not session_id:
+            # Session not initialized, do nothing
+            return item_id, left_text, right_text, 0, "Not started"
+            
         if labeler.has_url_column:
             new_id, new_left, new_right, new_url, new_index = labeler.submit_judgment(
                 item_id, left_text, right_text, choice, session_id
             )
             url_html = f'<a href="{new_url}" target="_blank">{new_url}</a>' if new_url else ""
-            return new_id, new_left, new_right, url_html, new_index
+            
+            # Update session info text with new progress
+            progress_text = f"Session: {session_id} | {new_index + 1}/{len(labeler)}" if new_index < len(labeler) else f"Session: {session_id} | Complete! {len(labeler)}/{len(labeler)}"
+            
+            return new_id, new_left, new_right, url_html, new_index, progress_text
         else:
             new_id, new_left, new_right, new_index = labeler.submit_judgment(
                 item_id, left_text, right_text, choice, session_id
             )
-            return new_id, new_left, new_right, new_index
+            
+            # Update session info text with new progress
+            progress_text = f"Session: {session_id} | {new_index + 1}/{len(labeler)}" if new_index < len(labeler) else f"Session: {session_id} | Complete! {len(labeler)}/{len(labeler)}"
+            
+            return new_id, new_left, new_right, new_index, progress_text
     
     if labeler.has_url_column:
         left_btn.click(
@@ -325,4 +406,4 @@ with gr.Blocks() as app:
         )
 
 if __name__ == "__main__":
-    app.launch()
+    app.launch()