Upload folder using huggingface_hub

.env.example

@@ -0,0 +1,4 @@
+HF_TOKEN=hf_...
+WITGYM_DATA_REPO=akshay4/witgym-data
+WITGYM_SKIP_HUB=1
+LLM_BACKEND=hf_api

.gitignore

@@ -1,68 +0,0 @@
-# Local index is generated; ship transcripts only (Space builds index on startup)
-data/*
-!data/transcripts/
-!data/transcripts/**
-
-# Virtual environments
-.venv/
-venv/
-env/
-
-# Environment and secrets
-.env
-.env.*
-!.env.example
-
-# Python bytecode and packaging
-__pycache__/
-*.py[cod]
-*$py.class
-*.so
-.Python
-*.egg
-*.egg-info/
-.eggs/
-dist/
-build/
-pip-wheel-metadata/
-
-# Test and type-check caches
-.pytest_cache/
-.mypy_cache/
-.ruff_cache/
-htmlcov/
-.coverage
-.coverage.*
-coverage.xml
-*.cover
-.hypothesis/
-
-# Jupyter
-.ipynb_checkpoints/
-
-# IDE and editors
-.idea/
-.vscode/
-*.swp
-*.swo
-*~
-
-# OS
-.DS_Store
-Thumbs.db
-
-# ML / Hugging Face caches and artifacts
-.cache/
-huggingface/
-checkpoints/
-runs/
-wandb/
-mlruns/
-*.pt
-*.pth
-*.ckpt
-*.safetensors
-
-# Logs
-*.log
-logs/

README.md

@@ -49,9 +49,34 @@ python app.py
 |--------|--------|
 | `HF_TOKEN` | Your Hugging Face access token |
 | `LLM_BACKEND` | `hf_api` (recommended on Spaces) |
+| `WITGYM_DATA_REPO` | Private dataset repo, e.g. `akshay4/witgym-data` |
 
 Optional: `HF_INFERENCE_PROVIDER` (defaults to `together` for Qwen3.5-9B — required for `enable_thinking: false`), `WITGYM_INDEX_PATH`.
 
+## Large data on Hugging Face Hub
+
+Files over 1 MB (`office_generated.txt`, `index.npz`) live in a **private dataset repo**, not in git. The app fetches them at startup via `hf_hub_download`.
+
+**One-time setup** (create private dataset `witgym-data`, then upload):
+
+```bash
+hf upload akshay4/witgym-data \
+  data/transcripts/office_generated.txt office_generated.txt \
+  --repo-type dataset --private
+
+hf upload akshay4/witgym-data \
+  data/index.npz index.npz \
+  --repo-type dataset
+```
+
+**When transcripts change:** re-run `witgym-index`, then re-upload `index.npz`.
+
+**Offline local dev:** set `WITGYM_SKIP_HUB=1` and keep local copies in `data/`.
+
+## Deploy (GitHub → Space)
+
+Pushes to `main` sync to [build-small-hackathon/WitGym](https://huggingface.co/spaces/build-small-hackathon/WitGym) via `.github/workflows/sync-to-hub.yml`. Add `HF_TOKEN` as a GitHub repository secret.
+
 ## Architecture
 
 - **Small talk** (hi, who are you) → polite identity reply, no pipeline

witgym/config.py

@@ -57,6 +57,7 @@ CLICHE_PENALTY_TOKENS = 6   # Penalise first N tokens of the obvious response
 
 # HuggingFace auth + inference backend
 HF_TOKEN = os.getenv("HF_TOKEN", "")
+WITGYM_DATA_REPO = os.getenv("WITGYM_DATA_REPO", "")
 # "local" = Transformers on device; "hf_api" = Inference Providers (Spaces default)
 LLM_BACKEND = os.getenv("LLM_BACKEND", "local")
 # Qwen3.5-9B thinking-mode toggle requires Together; "auto" often 400s on extra_body

witgym/hub_data.py

@@ -0,0 +1,64 @@
+"""Fetch >1MB artifacts from a private HF dataset repo."""
+import os
+from pathlib import Path
+from typing import Optional
+
+from loguru import logger
+
+from witgym import config
+
+HUB_ARTIFACTS = ("index.npz", "office_generated.txt")
+HUB_CACHE_DIR = "data/hub_cache"
+
+
+def _local_default_path(filename: str) -> Optional[Path]:
+    if filename == "index.npz":
+        return Path(config.INDEX_PATH)
+    if filename == "office_generated.txt":
+        return Path(config.TRANSCRIPT_DIR) / filename
+    return None
+
+
+def ensure_artifact(filename: str, local_dir: str = HUB_CACHE_DIR) -> Path:
+    """Return local path; use existing file, cache, or download from Hub."""
+    default = _local_default_path(filename)
+    if default is not None and default.exists():
+        return default
+
+    cache_dest = Path(local_dir) / filename
+    if cache_dest.exists():
+        return cache_dest
+
+    if os.getenv("WITGYM_SKIP_HUB") or not config.WITGYM_DATA_REPO or not config.HF_TOKEN:
+        raise FileNotFoundError(f"{filename} not found locally and Hub fetch disabled")
+
+    from huggingface_hub import hf_hub_download
+
+    logger.info(f"Downloading {filename} from {config.WITGYM_DATA_REPO}")
+    downloaded = hf_hub_download(
+        repo_id=config.WITGYM_DATA_REPO,
+        filename=filename,
+        repo_type="dataset",
+        token=config.HF_TOKEN,
+        local_dir=local_dir,
+    )
+    return Path(downloaded)
+
+
+def materialize_hub_transcripts() -> None:
+    """Copy hub transcript into data/transcripts/ so indexer glob stays unchanged."""
+    if os.getenv("WITGYM_SKIP_HUB") or not config.WITGYM_DATA_REPO:
+        return
+
+    dest = Path(config.TRANSCRIPT_DIR) / "office_generated.txt"
+    if dest.exists():
+        return
+
+    try:
+        hub_txt = ensure_artifact("office_generated.txt")
+    except FileNotFoundError:
+        return
+
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    dest.write_bytes(hub_txt.read_bytes())
+    logger.info(f"Materialized {dest} from Hub")

witgym/retriever.py

@@ -28,6 +28,18 @@ def load_index(index_path: str = config.INDEX_PATH) -> dict:
         if os.path.exists(npz_fallback):
             path = npz_fallback
 
+    from witgym.hub_data import materialize_hub_transcripts
+
+    materialize_hub_transcripts()
+
+    if not os.path.exists(path) and config.WITGYM_DATA_REPO:
+        from witgym.hub_data import ensure_artifact
+
+        try:
+            path = str(ensure_artifact("index.npz"))
+        except FileNotFoundError:
+            pass
+
     if not os.path.exists(path):
         transcript_dir = Path(config.TRANSCRIPT_DIR)
         if transcript_dir.is_dir() and any(transcript_dir.glob("*.txt")):