Switch to synchronous dataset push (no CommitScheduler), add HF_TOKEN for private dataset access

app.py

@@ -6,13 +6,11 @@ import urllib.parse
 import urllib.request
 from datetime import datetime
 from html.parser import HTMLParser
-from pathlib import Path
 from uuid import uuid4
 
 import gradio as gr
 from PIL import Image, ImageDraw, ImageFont
 from adaptive_classifier import AdaptiveClassifier
-from huggingface_hub import CommitScheduler
 
 # ---------------------------------------------------------------------------
 # Model
@@ -26,24 +24,43 @@ print("Model loaded!")
 # ---------------------------------------------------------------------------
 # Persistent dataset via CommitScheduler
 # ---------------------------------------------------------------------------
-DATA_DIR = Path("prediction_data")
-DATA_DIR.mkdir(parents=True, exist_ok=True)
-DATA_FILE = DATA_DIR / f"predictions-{uuid4()}.jsonl"
-
-scheduler = CommitScheduler(
-    repo_id="adaptive-classifier/ai-detector-data",
-    repo_type="dataset",
-    folder_path=DATA_DIR,
-    path_in_repo="data",
-    private=True,
-)
+DATASET_REPO = "adaptive-classifier/ai-detector-data"
+_predictions = {}  # In-memory cache
+_hf_api = None
+
+
+def _get_api():
+    global _hf_api
+    if _hf_api is None:
+        from huggingface_hub import HfApi
+        _hf_api = HfApi()
+    return _hf_api
 
-# In-memory index for lookups by ID (populated from current session)
-_predictions = {}
+
+def _push_record(record: dict):
+    """Append a record to the dataset by uploading a JSONL file."""
+    import io
+    api = _get_api()
+    line = json.dumps(record) + "\n"
+    # Append to a single predictions.jsonl file
+    # Download existing, append, re-upload
+    try:
+        path = api.hf_hub_download(DATASET_REPO, "data/predictions.jsonl", repo_type="dataset")
+        existing = open(path).read()
+    except Exception:
+        existing = ""
+    content = existing + line
+    api.upload_file(
+        path_or_fileobj=io.BytesIO(content.encode()),
+        path_in_repo="data/predictions.jsonl",
+        repo_id=DATASET_REPO,
+        repo_type="dataset",
+        commit_message=f"Add prediction {record.get('id', '')}",
+    )
 
 
 def save_prediction(pred_id: str, text: str, url: str, label: str, confidence: float):
-    """Save a prediction to the dataset."""
+    """Save a prediction to memory and push to HF dataset."""
     record = {
         "id": pred_id,
         "text": text,
@@ -54,61 +71,45 @@ def save_prediction(pred_id: str, text: str, url: str, label: str, confidence: f
         "timestamp": datetime.now().isoformat(),
     }
     _predictions[pred_id] = record
-    with scheduler.lock:
-        with DATA_FILE.open("a") as f:
-            json.dump(record, f)
-            f.write("\n")
+    try:
+        _push_record(record)
+    except Exception as e:
+        print(f"Warning: failed to push prediction to dataset: {e}")
 
 
 def lookup_prediction(pred_id: str) -> dict | None:
-    """Look up a prediction by ID — check memory first, then dataset files."""
+    """Look up a prediction by ID — check memory, then HF dataset."""
     if pred_id in _predictions:
         return _predictions[pred_id]
-    # Search local data files (includes data synced from HF)
-    for f in sorted(DATA_DIR.glob("*.jsonl")):
-        try:
-            for line in f.read_text().strip().split("\n"):
-                if not line:
-                    continue
-                rec = json.loads(line)
-                if rec.get("id") == pred_id and "text" in rec:
-                    _predictions[pred_id] = rec  # cache it
-                    return rec
-        except Exception:
-            continue
-    # Try downloading from HF dataset
+    # Download from HF dataset
     try:
-        from huggingface_hub import HfApi
-        api = HfApi()
-        files = api.list_repo_files("adaptive-classifier/ai-detector-data", repo_type="dataset")
-        data_files = [f for f in files if f.startswith("data/") and f.endswith(".jsonl")]
-        for df in data_files:
-            path = api.hf_hub_download("adaptive-classifier/ai-detector-data", df, repo_type="dataset")
-            for line in open(path).read().strip().split("\n"):
-                if not line:
-                    continue
-                rec = json.loads(line)
-                if rec.get("id") == pred_id and "text" in rec:
-                    _predictions[pred_id] = rec
-                    return rec
+        api = _get_api()
+        path = api.hf_hub_download(DATASET_REPO, "data/predictions.jsonl", repo_type="dataset")
+        for line in open(path).read().strip().split("\n"):
+            if not line:
+                continue
+            rec = json.loads(line)
+            if rec.get("id") == pred_id and "text" in rec:
+                _predictions[pred_id] = rec
+                return rec
     except Exception:
         pass
     return None
 
 
 def save_feedback(pred_id: str, feedback: str):
-    """Save user feedback for a prediction."""
+    """Save user feedback to memory and push to HF dataset."""
     record = {
         "id": pred_id,
         "feedback": feedback,
         "timestamp": datetime.now().isoformat(),
     }
-    with scheduler.lock:
-        with DATA_FILE.open("a") as f:
-            json.dump(record, f)
-            f.write("\n")
     if pred_id in _predictions:
         _predictions[pred_id]["feedback"] = feedback
+    try:
+        _push_record(record)
+    except Exception as e:
+        print(f"Warning: failed to push feedback to dataset: {e}")
 
 
 # ---------------------------------------------------------------------------