Add application file

app.py

@@ -1,88 +1,143 @@
 import gradio as gr
+import datasets
+from huggingface_hub import HfApi
 
-# Function to handle evaluation
-def evaluate_texts(*args):
-    num_columns = (len(args) - 1) // 2  # Exclude preference from the count
-    evaluations = []
+# Load dataset from Hugging Face
+ds_name = "anna-tch/generation-results"
+dataset = datasets.load_dataset(ds_name)["train"]
 
-    for i in range(num_columns):
-        grammar = args[i * 2]
-        coherence = args[i * 2 + 1]
+# Identify generation columns (all except 'comment_id' and 'manual_annotation')
+generation_columns = [col for col in dataset.column_names if col not in ["comment_id", "manual_annotation"]]
 
-        # Ensure required fields are filled
-        if grammar is None or coherence is None:
-            return "Error: Please fill out all fields before submitting."
-        
-        evaluations.append(f"Adapter {i+1}: Grammar {grammar}, Coherence {coherence}")
+# Track annotation progress
+current_index = 0
+annotations = {}  # Store annotations locally before pushing
 
-    # Preferred adapter
-    preference = args[-1]
-    evaluations.append(f"Preferred Text: {preference}")
 
-    return "\n".join(evaluations)
+def get_next_sample():
+    """Fetch the next unannotated sample"""
+    global current_index
 
-# Define the number of adapters (change dynamically)
-num_columns = 3  # Change to 2, 3, or 4 as needed
+    # Find the next unannotated comment
+    while current_index < len(dataset):
+        sample = dataset[current_index]
+        if sample["manual_annotation"] is None:
+            return sample
+        current_index += 1
 
-with gr.Blocks() as demo:
-    gr.Markdown("## Compare and Rate Generated Texts")
+    return None  # No more samples to annotate
+
+
+def annotate_text(grammar_scores, coherence_scores, preferred_text):
+    """Save annotation and move to next sample"""
+    global current_index
+
+    sample = dataset[current_index]
+
+    # Store annotation locally
+    annotations[sample["comment_id"]] = {
+        "grammar": dict(zip(generation_columns, grammar_scores)),
+        "coherence": dict(zip(generation_columns, coherence_scores)),
+        "preferred_text": preferred_text,
+    }
+
+    # update the dataset
+    dataset[current_index]["manual_annotation"] = annotations[sample["comment_id"]]
+
+    # Push updated dataset to Hugging Face
+    dataset.push_to_hub(ds_name)
+
+    # Move to next sample
+    current_index += 1
+    return update_ui()
+
+
+def update_ui():
+    """Update UI with the next sample or display completion message"""
+    sample = get_next_sample()
+    if sample is None:
+        return gr.update(visible=False), gr.update(value="All comments annotated!"), gr.update(visible=False)
+
+    textboxes = [gr.update(value=sample[col]) for col in generation_columns]
+    return textboxes + [gr.update(value=None)] * len(generation_columns) + [gr.update(value=None)]
 
-    # Description of scoring system
-    gr.Markdown("""
-    **Scoring System**  
-    - **Grammar Score**: -2 (Poor), -1 (Fair), 0 (Neutral), 1 (Good), 2 (Excellent)  
-    - **Coherence Score**: -2 (Poor), -1 (Fair), 0 (Neutral), 1 (Good), 2 (Excellent)  
-    """)
 
-    adapters = []  # Store inputs dynamically
+# def push_annotations():
+#     """Push annotations to Hugging Face"""
+#     for idx, sample in enumerate(dataset):
+#         if sample["comment_id"] in annotations:
+#             dataset[idx]["manual_annotation"] = annotations[sample["comment_id"]]
 
-    for i in range(num_columns):
+#     dataset.push_to_hub(ds_name)
+#     return "Annotations successfully uploaded to Hugging Face!"
+
+
+# Gradio Interface
+with gr.Blocks() as demo:
+    gr.Markdown("## Annotate Generated Texts")
+    gr.Markdown("#### Rate each generation and select the best one")
+
+    textboxes = []
+    grammar_radios = []
+    coherence_radios = []
+
+    for col in generation_columns:
         with gr.Row():
-            # Left: Generated text (in a larger column)
             with gr.Column(scale=3):
-                gr.Markdown(f"### Adapter {i+1}")
-                adapter_text = gr.Textbox(label=f"Generated Text by Adapter {i+1}", lines=5, max_lines=6)
+                gr.Markdown(f"### {col}")
+                text_input = gr.Textbox(label=f"Generated Text ({col})", lines=5)
+                textboxes.append(text_input)
 
-            # Right: Scoring section with **extra padding for spacing**
-            with gr.Column(scale=1, min_width=150):
-                gr.Markdown("### Scoring")  # Adds a title for clarity
+            with gr.Column(scale=1):
                 grammar = gr.Radio(
-                    choices=["-2", "-1", "0", "1", "2"], 
-                    label="Grammar", 
+                    choices=["-2", "-1", "0", "1", "2"],
+                    label="Grammar",
                     interactive=True
                 )
                 coherence = gr.Radio(
-                    choices=["-2", "-1", "0", "1", "2"], 
-                    label="Coherence", 
+                    choices=["-2", "-1", "0", "1", "2"],
+                    label="Coherence",
                     interactive=True
                 )
+                grammar_radios.append(grammar)
+                coherence_radios.append(coherence)
 
-            # Store inputs for later
-            adapters.append(adapter_text)
-            adapters.append(grammar)
-            adapters.append(coherence)
-
-    # User preference dropdown
-    preference = gr.Dropdown(
-        choices=[f"Adapter {i+1}" for i in range(num_columns)], 
-        label="Choose Your Preferred Text", 
-        value=f"Adapter 1"
-    )
+    preferred_text = gr.Dropdown(choices=generation_columns, label="Select Best Generation")
 
-    # Submit button + result box
     with gr.Row():
-        submit_btn = gr.Button("Submit Ratings")
-        result_output = gr.Textbox(label="Evaluation Results", lines=6)
+        submit_btn = gr.Button("Submit Annotation")
+        next_btn = gr.Button("Skip (Next)")
+        push_btn = gr.Button("Push to Hugging Face")
 
-    # Connect function to UI
-    submit_btn.click(evaluate_texts, inputs=adapters + [preference], outputs=result_output)
+    output_message = gr.Textbox(label="Status", interactive=False)
 
-# Launch the Gradio app
+    # Set initial values
+    demo.load(update_ui, outputs=textboxes)
+
+    # Actions
+    submit_btn.click(annotate_text, inputs=grammar_radios + coherence_radios + [preferred_text], outputs=textboxes + [output_message])
+    next_btn.click(lambda: update_ui(), outputs=textboxes + [output_message])
+    # push_btn.click(push_annotations, outputs=output_message)
+
+# Launch the app
 demo.launch()
 
+
 # import gradio as gr
 
-# # Define the function to display and rate multiple generations
+# # Load dataset from Hugging Face
+# ds_name = "anna-tch/generation-results"
+# dataset = datasets.load_dataset(ds_name)["train"]
+
+# # Identify generation columns (all except 'comment_id' and 'manual_annotation')
+# generation_columns = [col for col in dataset.column_names if col not in ["comment_id", "manual_annotation"]]
+
+# # Track annotation progress
+# current_index = 0
+# annotations = {}  # Store annotations locally before pushing to the dataset
+
+
+# # Function to handle evaluation
 # def evaluate_texts(*args):
 #     num_columns = (len(args) - 1) // 2  # Exclude preference from the count
 #     evaluations = []
@@ -91,25 +146,25 @@ demo.launch()
 #         grammar = args[i * 2]
 #         coherence = args[i * 2 + 1]
 
-#         # Validate required fields
+#         # Ensure required fields are filled
 #         if grammar is None or coherence is None:
 #             return "Error: Please fill out all fields before submitting."
         
 #         evaluations.append(f"Adapter {i+1}: Grammar {grammar}, Coherence {coherence}")
 
-#     # Get preferred adapter
+#     # Preferred adapter
 #     preference = args[-1]
 #     evaluations.append(f"Preferred Text: {preference}")
 
 #     return "\n".join(evaluations)
 
-# # Define the number of generated texts (adapters)
-# num_columns = 3  # Change this to 2, 3, or 4 dynamically
+# # Define the number of adapters (change dynamically)
+# num_columns = 3  # Change to 2, 3, or 4 as needed
 
 # with gr.Blocks() as demo:
 #     gr.Markdown("## Compare and Rate Generated Texts")
 
-#     # Description of scores before the interface
+#     # Description of scoring system
 #     gr.Markdown("""
 #     **Scoring System**  
 #     - **Grammar Score**: -2 (Poor), -1 (Fair), 0 (Neutral), 1 (Good), 2 (Excellent)  
@@ -117,45 +172,127 @@ demo.launch()
 #     """)
 
 #     adapters = []  # Store inputs dynamically
+
 #     for i in range(num_columns):
 #         with gr.Row():
-#             # Left: Generated text
-#             with gr.Column(scale=3):  # Adjust width ratio (3:1)
+#             # Left: Generated text (in a larger column)
+#             with gr.Column(scale=3):
 #                 gr.Markdown(f"### Adapter {i+1}")
 #                 adapter_text = gr.Textbox(label=f"Generated Text by Adapter {i+1}", lines=5, max_lines=6)
-            
-#             # Right: Scoring section
-#             with gr.Column(scale=1):
+
+#             # Right: Scoring section with **extra padding for spacing**
+#             with gr.Column(scale=1, min_width=150):
+#                 gr.Markdown("### Scoring")  # Adds a title for clarity
 #                 grammar = gr.Radio(
 #                     choices=["-2", "-1", "0", "1", "2"], 
-#                     label="Grammar Score", 
+#                     label="Grammar", 
 #                     interactive=True
 #                 )
 #                 coherence = gr.Radio(
 #                     choices=["-2", "-1", "0", "1", "2"], 
-#                     label="Coherence Score", 
+#                     label="Coherence", 
 #                     interactive=True
 #                 )
 
-#             # Store inputs for submission
+#             # Store inputs for later
 #             adapters.append(adapter_text)
 #             adapters.append(grammar)
 #             adapters.append(coherence)
 
-#     # Preference dropdown
+#     # User preference dropdown
 #     preference = gr.Dropdown(
 #         choices=[f"Adapter {i+1}" for i in range(num_columns)], 
 #         label="Choose Your Preferred Text", 
 #         value=f"Adapter 1"
 #     )
 
-#     # Submit button and output
+#     # Submit button + result box
 #     with gr.Row():
 #         submit_btn = gr.Button("Submit Ratings")
 #         result_output = gr.Textbox(label="Evaluation Results", lines=6)
 
-#     # Link function to interface
+#     # Connect function to UI
 #     submit_btn.click(evaluate_texts, inputs=adapters + [preference], outputs=result_output)
 
-# # Launch the app
+# # Launch the Gradio app
 # demo.launch()
+
+# # import gradio as gr
+
+# # # Define the function to display and rate multiple generations
+# # def evaluate_texts(*args):
+# #     num_columns = (len(args) - 1) // 2  # Exclude preference from the count
+# #     evaluations = []
+
+# #     for i in range(num_columns):
+# #         grammar = args[i * 2]
+# #         coherence = args[i * 2 + 1]
+
+# #         # Validate required fields
+# #         if grammar is None or coherence is None:
+# #             return "Error: Please fill out all fields before submitting."
+        
+# #         evaluations.append(f"Adapter {i+1}: Grammar {grammar}, Coherence {coherence}")
+
+# #     # Get preferred adapter
+# #     preference = args[-1]
+# #     evaluations.append(f"Preferred Text: {preference}")
+
+# #     return "\n".join(evaluations)
+
+# # # Define the number of generated texts (adapters)
+# # num_columns = 3  # Change this to 2, 3, or 4 dynamically
+
+# # with gr.Blocks() as demo:
+# #     gr.Markdown("## Compare and Rate Generated Texts")
+
+# #     # Description of scores before the interface
+# #     gr.Markdown("""
+# #     **Scoring System**  
+# #     - **Grammar Score**: -2 (Poor), -1 (Fair), 0 (Neutral), 1 (Good), 2 (Excellent)  
+# #     - **Coherence Score**: -2 (Poor), -1 (Fair), 0 (Neutral), 1 (Good), 2 (Excellent)  
+# #     """)
+
+# #     adapters = []  # Store inputs dynamically
+# #     for i in range(num_columns):
+# #         with gr.Row():
+# #             # Left: Generated text
+# #             with gr.Column(scale=3):  # Adjust width ratio (3:1)
+# #                 gr.Markdown(f"### Adapter {i+1}")
+# #                 adapter_text = gr.Textbox(label=f"Generated Text by Adapter {i+1}", lines=5, max_lines=6)
+            
+# #             # Right: Scoring section
+# #             with gr.Column(scale=1):
+# #                 grammar = gr.Radio(
+# #                     choices=["-2", "-1", "0", "1", "2"], 
+# #                     label="Grammar Score", 
+# #                     interactive=True
+# #                 )
+# #                 coherence = gr.Radio(
+# #                     choices=["-2", "-1", "0", "1", "2"], 
+# #                     label="Coherence Score", 
+# #                     interactive=True
+# #                 )
+
+# #             # Store inputs for submission
+# #             adapters.append(adapter_text)
+# #             adapters.append(grammar)
+# #             adapters.append(coherence)
+
+# #     # Preference dropdown
+# #     preference = gr.Dropdown(
+# #         choices=[f"Adapter {i+1}" for i in range(num_columns)], 
+# #         label="Choose Your Preferred Text", 
+# #         value=f"Adapter 1"
+# #     )
+
+# #     # Submit button and output
+# #     with gr.Row():
+# #         submit_btn = gr.Button("Submit Ratings")
+# #         result_output = gr.Textbox(label="Evaluation Results", lines=6)
+
+# #     # Link function to interface
+# #     submit_btn.click(evaluate_texts, inputs=adapters + [preference], outputs=result_output)
+
+# # # Launch the app
+# # demo.launch()