Add feedback buttons

app.py

@@ -7,16 +7,21 @@ from wiki_data_fetcher import (
     get_revisions_behind,
     get_random_wikipedia_title,
 )
-from models import classifier, judge
-import logfire
-from dotenv import load_dotenv
+from feedback import save_feedback_agree, save_feedback_disagree
 from contextlib import nullcontext
+from dotenv import load_dotenv
+import logfire
+import os
 
 # Load API keys
 load_dotenv()
 # Setup logging with Logfire
 logfire.configure()
 
+# Import this after logfire.configure() so we don't get
+# LogfireNotConfiguredWarning: Instrumentation will have no effect
+from models import classifier, judge
+
 
 def start_parent_span(title: str, number: int, units: str):
     """
@@ -419,6 +424,17 @@ with gr.Blocks(title="Noteworthy Differences") as demo:
                 )
             rerun_btn = gr.Button("Rerun Model")
 
+            gr.Markdown("### Your feedback")
+            feedback_status = gr.Textbox(
+                label="",
+                lines=1,
+                interactive=False,
+                visible=True,
+            )
+            with gr.Row():
+                thumbs_up_btn = gr.Button("👍 Agree", variant="primary")
+                thumbs_down_btn = gr.Button("👎 Disagree", variant="secondary")
+
     # States to store boolean values
     heuristic_noteworthy = gr.State()
     fewshot_noteworthy = gr.State()
@@ -510,6 +526,55 @@ with gr.Blocks(title="Noteworthy Differences") as demo:
         api_name=False,
     )
 
+    # Feedback button handlers
+    thumbs_up_btn.click(
+        fn=save_feedback_agree,
+        inputs=[
+            title_input,
+            number_input,
+            units_dropdown,
+            judge_mode_dropdown,
+            old_revision,
+            new_revision,
+            old_timestamp,
+            new_timestamp,
+            heuristic_rationale,
+            fewshot_rationale,
+            judge_reasoning,
+            noteworthy_text,
+            confidence,
+            heuristic_noteworthy,
+            fewshot_noteworthy,
+            judge_noteworthy,
+        ],
+        outputs=[feedback_status],
+        api_name=False,
+    )
+
+    thumbs_down_btn.click(
+        fn=save_feedback_disagree,
+        inputs=[
+            title_input,
+            number_input,
+            units_dropdown,
+            judge_mode_dropdown,
+            old_revision,
+            new_revision,
+            old_timestamp,
+            new_timestamp,
+            heuristic_rationale,
+            fewshot_rationale,
+            judge_reasoning,
+            noteworthy_text,
+            confidence,
+            heuristic_noteworthy,
+            fewshot_noteworthy,
+            judge_noteworthy,
+        ],
+        outputs=[feedback_status],
+        api_name=False,
+    )
+
 if __name__ == "__main__":
 
     # Setup theme without background image

feedback.py

@@ -0,0 +1,88 @@
+from huggingface_hub import HfApi, CommitScheduler
+from datetime import datetime
+from pathlib import Path
+import gradio as gr
+import logfire
+import json
+import os
+
+# Set repo ID for Hugging Face dataset
+REPO_ID = "jedick/noteworthy-differences-feedback"
+# Setup user feedback file for uploading to HF dataset
+# https://huggingface.co/spaces/Wauplin/space_to_dataset_saver
+# https://huggingface.co/docs/huggingface_hub/v0.16.3/en/guides/upload#scheduled-uploads
+USER_FEEDBACK_DIR = Path("user_feedback")
+USER_FEEDBACK_DIR.mkdir(parents=True, exist_ok=True)
+
+
+if gr.NO_RELOAD:
+    # Create dataset if one doesn't exist
+    if not HfApi().repo_exists(REPO_ID, repo_type="dataset"):
+        api.create_repo(REPO_ID, repo_type="dataset")
+
+    # Initialize commit scheduler if we're running on Hugging Face spaces
+    if "SPACE_ID" in os.environ:
+        scheduler = CommitScheduler(
+            repo_id=REPO_ID,
+            repo_type="dataset",
+            folder_path=USER_FEEDBACK_DIR,
+            path_in_repo="data",
+        )
+
+
+def save_feedback(*args, feedback_value: str) -> None:
+    """
+    Save complete app state and user feedback to a
+    JSON Lines file for upload to a Hugging Face dataset.
+    """
+    # Assign dict keys to positional arguments
+    keys = [
+        "title_input",
+        "number_input",
+        "units_dropdown",
+        "judge_mode_dropdown",
+        "old_revision",
+        "new_revision",
+        "old_timestamp",
+        "new_timestamp",
+        "heuristic_rationale",
+        "fewshot_rationale",
+        "judge_reasoning",
+        "noteworthy_text",
+        "confidence",
+        "heuristic_noteworthy",
+        "fewshot_noteworthy",
+        "judge_noteworthy",
+    ]
+    feedback_dict = dict(zip(keys, args))
+    # Add feedback value
+    feedback_dict["feedback"] = feedback_value
+    # Save feedback to file
+    feedback_file = f"train-{datetime.now().isoformat()}.json"
+    feedback_path = USER_FEEDBACK_DIR / feedback_file
+    with feedback_path.open("a") as f:
+        f.write(json.dumps(feedback_dict))
+        f.write("\n")
+    gr.Success(f"Saved your feedback: {feedback_value}", duration=2, title="Thank you!")
+
+
+@logfire.instrument("Save feedback: agree")
+def save_feedback_agree(*args) -> None:
+    """Wrapper to save feedback with 'agree' value."""
+    # Schedule feedback for commit if we're running on Hugging Face spaces
+    if "SPACE_ID" in os.environ:
+        # Use a thread lock to avoid concurrent writes from different users
+        with scheduler.lock:
+            save_feedback(*args, feedback_value="agree")
+    else:
+        save_feedback(*args, feedback_value="agree")
+
+
+@logfire.instrument("Save feedback: disagree")
+def save_feedback_disagree(*args) -> None:
+    """Wrapper to save feedback with 'disagree' value."""
+    if "SPACE_ID" in os.environ:
+        with scheduler.lock:
+            save_feedback(*args, feedback_value="disagree")
+    else:
+        save_feedback(*args, feedback_value="disagree")

models.py

@@ -16,9 +16,6 @@ import logfire
 # Load API keys
 load_dotenv()
 
-# Setup Logfire
-logfire.configure()
-
 # This wraps Google Gen AI client calls
 # to capture prompts, responses, and metadata
 logfire.instrument_google_genai()