Fix: filter on is_new_dataset when showing dataset papers

confidence_score stores the max class probability (always >= 0.5 for
binary classification), so filtering on confidence >= 0.5 returned all
212K papers instead of just the 49K new dataset papers. Now requires
is_new_dataset = True when confidence threshold >= 0.5.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app.py

@@ -268,7 +268,12 @@ def filter_papers(
     - Setting threshold to 0 shows all papers
     - Setting threshold >= 0.5 effectively shows only new_dataset papers
     """
-    if min_confidence > 0:
+    if min_confidence >= 0.5:
+        # Show only papers classified as new datasets, filtered by confidence
+        df = df.filter(
+            pl.col("is_new_dataset") & (pl.col("confidence_score") >= min_confidence)
+        )
+    elif min_confidence > 0:
         df = df.filter(pl.col("confidence_score") >= min_confidence)
 
     if category:
@@ -334,7 +339,10 @@ def semantic_search(
 
     # Build SQL filter (Lance supports SQL-like syntax)
     filters = []
-    if min_confidence > 0:
+    if min_confidence >= 0.5:
+        filters.append("is_new_dataset = true")
+        filters.append(f"confidence_score >= {min_confidence}")
+    elif min_confidence > 0:
         filters.append(f"confidence_score >= {min_confidence}")
     if category:
         # Escape single quotes in category name for SQL safety