updated data storage

backend/main.py

@@ -62,6 +62,7 @@ DATA_DIR = BASE_DIR / "storage"
 DATA_DIR.mkdir(parents=True, exist_ok=True)
 
 HISTORY_FILE = DATA_DIR / "chat_history.jsonl"
+USAGE_FILE = DATA_DIR / "usage_metrics.jsonl"
 
 DEFAULT_SESSION = "default"
 
@@ -153,6 +154,27 @@ def _get_env_value(name: Optional[str]) -> str:
     return os.getenv(f"HF_{name}", "").strip()
 
 
+def _maybe_download_existing_metrics() -> None:
+    """Download existing usage metrics from HF dataset on startup."""
+    repo_id = os.getenv("KIOSK_HF_DATASET_REPO", "").strip()
+    if not repo_id or hf_hub_download is None:
+        return
+    _load_env_once()
+    token = _get_env_value("KIOSK_HF_TOKEN") or os.getenv("HF_TOKEN", "").strip()
+    path_in_repo = os.getenv("KIOSK_HF_DATASET_PATH", "chat_history").strip()
+    filename = f"{path_in_repo}/{USAGE_FILE.name}" if path_in_repo else USAGE_FILE.name
+    try:
+        import shutil
+        downloaded = hf_hub_download(
+            repo_id=repo_id, repo_type="dataset", filename=filename, token=token or None,
+        )
+        USAGE_FILE.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy(downloaded, USAGE_FILE)
+        logger.info("Downloaded usage metrics from HF: repo=%s file=%s", repo_id, filename)
+    except Exception as exc:
+        logger.info("No existing metrics to download (starting fresh): %s", exc)
+
+
 def _maybe_download_existing_history() -> None:
     """Download existing chat history from HF dataset on startup."""
     repo_id = os.getenv("KIOSK_HF_DATASET_REPO", "").strip()
@@ -185,21 +207,17 @@ def _maybe_download_existing_history() -> None:
 
 
 def _maybe_start_hf_sync() -> None:
-    """Start optional HF dataset syncing for chat history."""
+    """Start optional HF dataset syncing for chat history and usage metrics."""
     global _hf_scheduler
-
     if _hf_scheduler is not None:
         return
-
     repo_id = os.getenv("KIOSK_HF_DATASET_REPO", "").strip()
     if not repo_id or CommitScheduler is None:
         return
-
     _load_env_once()
     token = _get_env_value("KIOSK_HF_TOKEN") or os.getenv("HF_TOKEN", "").strip()
     path_in_repo = os.getenv("KIOSK_HF_DATASET_PATH", "chat_history").strip()
     interval_minutes = float(os.getenv("KIOSK_HF_SYNC_INTERVAL_MINUTES", "10"))
-
     try:
         _hf_scheduler = CommitScheduler(
             repo_id=repo_id,
@@ -207,14 +225,12 @@ def _maybe_start_hf_sync() -> None:
             folder_path=str(DATA_DIR),
             path_in_repo=path_in_repo,
             token=token or None,
-            allow_patterns=[HISTORY_FILE.name],
+            allow_patterns=[HISTORY_FILE.name, USAGE_FILE.name],
             every=interval_minutes,
         )
         logger.info(
-            "Started HF CommitScheduler for chat history: repo=%s path=%s interval=%sm",
-            repo_id,
-            path_in_repo or ".",
-            interval_minutes,
+            "Started HF CommitScheduler for chat_history and usage_metrics: repo=%s path=%s interval=%s",
+            repo_id, path_in_repo or ".", interval_minutes,
         )
     except Exception as exc:
         warnings.warn(f"Unable to start HF sync: {exc}")
@@ -224,6 +240,7 @@ def _run_startup_tasks_in_background() -> None:
     """Run HF download and sync in a background thread so the server starts immediately."""
     def _run() -> None:
         try:
+            _maybe_download_existing_metrics()
             _maybe_download_existing_history()
             _maybe_start_hf_sync()
         except Exception as exc:
@@ -515,6 +532,14 @@ def record_history(
         "action": action,
     }
     _append_json_line(HISTORY_FILE, payload)
+    usage_entry = {
+        "timestamp": timestamp,
+        "session_id": session_id,
+        "blueprint": blueprint,
+        "question": question,
+    }
+    usage_entry.setdefault("action_type", action.get("type"))
+    _append_json_line(USAGE_FILE, usage_entry)
     return timestamp