Add special random button

app.py

@@ -19,15 +19,16 @@ from app_functions import (
     _run_heuristic_classifier,
     _run_fewshot_classifier,
     _run_judge,
+    find_interesting_example,
 )
 
 
-def start_parent_span(title: str, number: int, units: str):
+def start_parent_span(page_title: str, number_behind: int, units_behind: str):
     """
     Start a parent span and return the context for propagation to children.
     See https://logfire.pydantic.dev/docs/how-to-guides/distributed-tracing/#manual-context-propagation
     """
-    span_name = f"{title} - {number} {units}"
+    span_name = f"{page_title} - {number_behind} {units_behind}"
     with logfire.span(span_name) as span:
         span.__enter__()
         context = logfire.get_context()
@@ -93,15 +94,15 @@ with gr.Blocks(title="Noteworthy Differences") as demo:
           </colgroup>
           <tr>
             <td>
-              <i class="fa-brands fa-wikipedia-w"></i> Compare current and old revisions of a Wikipedia article.<br>
-              📅 You choose the number of revisions or days behind.
+              <i class="fa-brands fa-wikipedia-w"></i> Compare current and old revisions of a Wikipedia article<br>
+              📅 You choose the number of revisions or days behind
             </td>
             <td>
-              ◇ ∴ ⚖ Two classifier models and a judge predict the noteworthiness of the differences.
+              ◇ ∴ ⚖ Two classifier models and a judge predict the noteworthiness of the differences
             </td>
             <td>
-              <i class="fa-brands fa-github"></i> The <a href="https://github.com/jedick/noteworthy-differences">GitHub repository</a> describes how the judge was aligned with human preferences.<br>
-              👥 The <a href="https://huggingface.co/datasets/jedick/noteworthy-differences-feedback">feedback dataset</a> holds all user feedback collected to date.
+              <i class="fa-brands fa-github"></i> The <a href="https://github.com/jedick/noteworthy-differences">GitHub repository</a> describes how the judge was aligned with human preferences<br>
+              👥 The <a href="https://huggingface.co/datasets/jedick/noteworthy-differences-feedback">feedback dataset</a> holds all user feedback collected to date
               </td>
           </tr>
         </table>
@@ -203,6 +204,17 @@ with gr.Blocks(title="Noteworthy Differences") as demo:
                         thumbs_down_btn = gr.Button(
                             "👎 Disagree", size="md", variant="primary"
                         )
+                    with gr.Accordion("Find Interesting Example", open=True):
+                        special_random_btn = gr.Button(
+                            "🎲 Special Random", size="md", variant="secondary"
+                        )
+                        gr.Markdown(
+                            """
+                        *Click to find an interesting example
+                        by running the model on random pages
+                        until we get a confidence score that is not High,
+                        up to 20 tries*"""
+                        )
 
     # States to store boolean values
     heuristic_noteworthy = gr.State()
@@ -336,6 +348,28 @@ with gr.Blocks(title="Noteworthy Differences") as demo:
         api_name=False,
     )
 
+    # Special random button handler
+    special_random_btn.click(
+        fn=find_interesting_example,
+        inputs=[number_behind, units_behind],
+        outputs=[
+            page_title,
+            new_revision,
+            new_timestamp,
+            old_revision,
+            old_timestamp,
+            heuristic_noteworthy,
+            fewshot_noteworthy,
+            judge_noteworthy,
+            heuristic_rationale,
+            fewshot_rationale,
+            judge_reasoning,
+            noteworthy_text,
+            confidence_score,
+        ],
+        api_name=False,
+    )
+
 if __name__ == "__main__":
 
     # Setup theme without background image

app_functions.py

@@ -4,6 +4,7 @@ from wiki_data_fetcher import (
     get_wikipedia_introduction,
     extract_revision_info,
     get_revisions_behind,
+    get_random_wikipedia_title,
 )
 from models import classifier, judge
 import gradio as gr
@@ -259,3 +260,108 @@ def _run_judge(
         )
 
     return noteworthy, noteworthy_text, reasoning, confidence
+
+
+@logfire.instrument("🎲 Special Random")
+def find_interesting_example(number_behind: int, units_behind: str):
+    """
+    Find an interesting example by repeatedly getting random pages and running the model
+    until we find one with a confidence score that is not High, up to 20 tries.
+    """
+    max_tries = 20
+
+    for attempt in range(max_tries):
+        # Get random page title
+        page_title = get_random_wikipedia_title()
+        if not page_title:
+            continue
+
+        gr.Info(f"Page title {attempt + 1}: {page_title}", duration=20)
+
+        try:
+            # Initialize Logfire span
+            span_name = f"{page_title} - {number_behind} {units_behind}"
+            with logfire.span(span_name):
+
+                # Fetch current revision
+                new_revision, new_timestamp = _fetch_current_revision(page_title)
+                if not new_revision:
+                    continue
+
+                # Fetch previous revision
+                old_revision, old_timestamp = _fetch_previous_revision(
+                    page_title, number_behind, units_behind, new_revision
+                )
+                if not old_revision:
+                    continue
+
+                # Run heuristic classifier
+                heuristic_noteworthy, heuristic_rationale = _run_heuristic_classifier(
+                    old_revision, new_revision
+                )
+                if heuristic_rationale is None:
+                    continue
+
+                # Run few-shot classifier
+                fewshot_noteworthy, fewshot_rationale = _run_fewshot_classifier(
+                    old_revision, new_revision
+                )
+                if fewshot_rationale is None:
+                    continue
+
+                # Run judge
+                judge_noteworthy, noteworthy_text, judge_reasoning, confidence_score = (
+                    _run_judge(
+                        old_revision,
+                        new_revision,
+                        heuristic_noteworthy,
+                        fewshot_noteworthy,
+                        heuristic_rationale,
+                        fewshot_rationale,
+                    )
+                )
+
+            # Check if confidence score is not High
+            if confidence_score and confidence_score != "High":
+                # Found an interesting example
+                gr.Success(
+                    "Interesting example - ready for your feedback", duration=None
+                )
+                return (
+                    page_title,
+                    new_revision,
+                    new_timestamp,
+                    old_revision,
+                    old_timestamp,
+                    heuristic_noteworthy,
+                    fewshot_noteworthy,
+                    judge_noteworthy,
+                    heuristic_rationale,
+                    fewshot_rationale,
+                    judge_reasoning,
+                    noteworthy_text,
+                    confidence_score,
+                )
+
+        except Exception:
+            # If there's an error, continue to next attempt
+            continue
+
+    # If we get here, all 20 tries had High confidence
+    gr.Warning("No interesting examples found - try again", duration=None)
+    # Return empty values
+    return (
+        "",
+        "",
+        "",
+        "",
+        "",
+        None,
+        None,
+        None,
+        "",
+        "",
+        "",
+        "",
+        "",
+    )