Updated with real poster data support + poster pass-through gate

src/pubguard/data.py

@@ -283,8 +283,8 @@ def prepare_doc_type_dataset(
         except Exception:
             pass
 
-    # ── junk ─────────────────────────────────────────────────────
-    logger.info("Loading ag_news for junk class...")
+    # ── junk (100% real data — ag_news) ────────────────────────────
+    logger.info("Loading ag_news for junk class (full — no synthetic)...")
     try:
         ds = load_dataset(
             "ag_news",
@@ -292,7 +292,7 @@ def prepare_doc_type_dataset(
         )
         count = 0
         for row in ds:
-            if count >= n_per_class // 2:
+            if count >= n_per_class:
                 break
             text = row.get("text", "")
             if len(text) > 30:
@@ -302,16 +302,53 @@ def prepare_doc_type_dataset(
     except Exception as e:
         logger.warning(f"Could not load ag_news: {e}")
 
-    logger.info("Generating synthetic junk...")
-    synth_junk = generate_synthetic_junk(n_per_class // 2)
-    all_samples.extend(synth_junk)
-    logger.info(f"  junk (synthetic): {len(synth_junk)}")
-
-    # ── poster ───────────────────────────────────────────────────
-    logger.info("Generating synthetic poster data...")
-    synth_posters = generate_synthetic_posters(n_per_class)
+    # ── poster ────────────────────────────────────────────────────
+    # NOTE: Real poster text is nearly identical to paper text in
+    # embedding space (both are scientific). PubGuard uses text-only
+    # features, so we need SHORT, STRUCTURED poster-style texts that
+    # the embedding can distinguish from full papers.
+    #
+    # Strategy: synthetic poster templates (structured, short) +
+    # real poster texts TRUNCATED to first 500 chars (title/authors
+    # block, which has distinct formatting from paper introductions).
+    logger.info("Loading poster data (structured templates + real poster headers)...")
+    poster_count = 0
+
+    # (a) Synthetic templates — provide distinctive poster structure signal
+    synth_posters = generate_synthetic_posters(min(n_per_class // 2, 7500))
     all_samples.extend(synth_posters)
-    logger.info(f"  poster (synthetic): {len(synth_posters)}")
+    poster_count += len(synth_posters)
+    logger.info(f"  poster (synthetic templates): {len(synth_posters)}")
+
+    # (b) Real poster header text (first 500 chars only — title/authors block)
+    real_poster_count = 0
+    local_poster_data = Path("/home/joneill/pubverse_brett/poster_sentry/poster_texts_for_pubguard.ndjson")
+    if not local_poster_data.exists():
+        local_poster_data = Path.cwd().parent / "poster_sentry" / "poster_texts_for_pubguard.ndjson"
+
+    if local_poster_data.exists():
+        logger.info(f"  Adding real poster headers from: {local_poster_data}")
+        with open(local_poster_data) as f:
+            for line in f:
+                if real_poster_count >= n_per_class // 2:
+                    break
+                row = json.loads(line)
+                if row.get("label") == "poster":
+                    # Truncate to header region (title, authors, affiliations)
+                    text = row["text"][:500]
+                    if len(text) > 50:
+                        all_samples.append({"text": text, "label": "poster"})
+                        real_poster_count += 1
+        poster_count += real_poster_count
+        logger.info(f"  poster (real headers, ≤500 chars): {real_poster_count}")
+    else:
+        # Fill with more synthetic templates if no real data available
+        extra = generate_synthetic_posters(n_per_class // 2)
+        all_samples.extend(extra)
+        poster_count += len(extra)
+        logger.info(f"  poster (synthetic fallback): {len(extra)}")
+
+    logger.info(f"  poster total: {poster_count}")
 
     # ── Shuffle and save ─────────────────────────────────────────
     random.shuffle(all_samples)