feat: Add support for remote pack loading with environment configuration and enhanced metadata handling

.hf_pack_cache/datasets--Bellok--warbler-cda-corpus/blobs/45a1df9f2f9e0221d282e0ba9a65b5ac8ac41354

@@ -0,0 +1,15 @@
+{
+  "name": "warbler-pack-hf-arxiv",
+  "version": "1.0.0",
+  "description": "Warbler pack generated from HuggingFace datasets (chunked)",
+  "created_at": "2026-04-08T12:07:25.549352",
+  "document_count": 250000,
+  "source": "HuggingFace",
+  "content_types": [
+    "scholarly_discussion"
+  ],
+  "chunked": true,
+  "chunk_count": 5,
+  "docs_per_chunk": 50000,
+  "chunk_pattern": "warbler-pack-hf-arxiv-chunk-*.jsonl"
+}

.hf_pack_cache/datasets--Bellok--warbler-cda-corpus/blobs/f7cb515e761ca01d38aa98b07149c0453840d68f

@@ -0,0 +1,56 @@
+{
+  "name": "warbler-pack-core",
+  "version": "0.1.0",
+  "description": "Core conversation pack for Warbler NPC system with essential dialogue templates",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.js"
+    },
+    "./templates": "./pack/templates.json"
+  },
+  "files": [
+    "dist/**/*",
+    "pack/templates.json",
+    "README.md",
+    "package.json"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "echo \"Info: Content pack - no tests required\"",
+    "validate": "node ../../scripts/validate-warbler-pack.mjs pack/templates.json",
+    "prepublishOnly": "npm run build && npm run validate"
+  },
+  "keywords": [
+    "warbler",
+    "npc",
+    "conversation",
+    "dialogue",
+    "templates",
+    "core"
+  ],
+  "author": "TWG Team",
+  "license": "MIT",
+  "dependencies": {
+    "warbler-core": "^0.1.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.3.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/jmeyer1980/TWG-TLDA.git",
+    "directory": "packs/warbler-pack-core"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "warbler": {
+    "packType": "core",
+    "templateCount": 8,
+    "compatibleEngine": "^0.1.0"
+  }
+}

.hf_pack_cache/datasets--Bellok--warbler-cda-corpus/refs/main

@@ -0,0 +1 @@
+1944661030a32fea59c540fa6957aca6e60b4ae0

.hf_pack_cache/datasets--Bellok--warbler-cda-corpus/snapshots/1944661030a32fea59c540fa6957aca6e60b4ae0/packs/warbler-pack-core/package.json

@@ -0,0 +1 @@
+../../../../blobs/f7cb515e761ca01d38aa98b07149c0453840d68f

.hf_pack_cache/datasets--Bellok--warbler-cda-corpus/snapshots/1944661030a32fea59c540fa6957aca6e60b4ae0/packs/warbler-pack-hf-arxiv/package.json

@@ -0,0 +1 @@
+../../../../blobs/45a1df9f2f9e0221d282e0ba9a65b5ac8ac41354

app.py

@@ -9,6 +9,7 @@ Provides a web UI for the FractalStat RAG system with GPU acceleration.
 import gradio as gr
 import os
 import time
+
 from warbler_cda.remote_pack_loader import RemotePackLoader
 
 # Import the HuggingFace Spaces GPU decorator
@@ -108,10 +109,10 @@ remote_loader = None
 documents = []
 try:
     print(f"📚 Attempting to fetch packs from remote dataset repo: {REMOTE_DATASET_REPO}")
-    remote_loader = RemotePackLoader(repo_id=REMOTE_DATASET_REPO)
+    remote_loader = RemotePackLoader.from_environment(repo_id=REMOTE_DATASET_REPO)
     remote_packs_dir = remote_loader.fetch_packs()
     print(f"✅ Remote packs downloaded to: {remote_packs_dir}")
-    pack_loader = PackLoader(packs_dir=remote_packs_dir)
+    pack_loader = PackLoader.from_environment(packs_dir=remote_packs_dir)
     documents = pack_loader.discover_documents()
     print(f"✅ Loaded {len(documents)} documents from remote dataset repo")
 except Exception as e:
@@ -308,7 +309,7 @@ def get_system_stats() -> str:
     output += "### 🕵️ Bob the Skeptic - Conflict Detection\n\n"
 
     # Access conflict detector if available
-    conflict_detector = getattr(api, 'conflict_detector', None) if hasattr(api, 'config') and api.config else None
+    conflict_detector = getattr(api, 'conflict_detector', None)
 
     if conflict_detector and hasattr(conflict_detector, 'get_global_conflict_summary'):
         try:
@@ -363,7 +364,11 @@ def get_system_stats() -> str:
 
     # Recent Activity
     output += "### 📈 Recent Activity\n\n"
-    output += f"**Retrieval Success Rate:** {metrics['system_health']['retrieval_success_rate']:.1% if 'retrieval_success_rate' in metrics['system_health'] else 'N/A'}\n\n"
+    retrieval_success_rate = metrics["system_health"].get("retrieval_success_rate")
+    if retrieval_success_rate is None:
+        output += "**Retrieval Success Rate:** N/A\n\n"
+    else:
+        output += f"**Retrieval Success Rate:** {retrieval_success_rate:.1%}\n\n"
 
     return output
 

tests/test_remote_pack_loader.py

@@ -0,0 +1,47 @@
+from unittest.mock import patch
+
+
+def test_build_allow_patterns_limits_chunked_remote_pack_downloads():
+    from warbler_cda.remote_pack_loader import RemotePackLoader
+
+    repo_files = [
+        "packs/warbler-pack-core/package.json",
+        "packs/warbler-pack-core/pack/templates.json",
+        "packs/warbler-pack-hf-arxiv/package.json",
+        "packs/warbler-pack-hf-arxiv/warbler-pack-hf-arxiv-chunk-001.jsonl",
+        "packs/warbler-pack-hf-arxiv/warbler-pack-hf-arxiv-chunk-002.jsonl",
+        "packs/warbler-pack-hf-arxiv/warbler-pack-hf-arxiv-chunk-003.jsonl",
+    ]
+
+    loader = RemotePackLoader(
+        repo_id="Bellok/warbler-cda-corpus",
+        max_documents_per_pack=5000,
+    )
+
+    with patch.object(loader, "_list_repo_files", return_value=repo_files), patch.object(
+        loader,
+        "_load_pack_metadata",
+        side_effect=lambda pack_name: {
+            "chunked": True,
+            "docs_per_chunk": 50000,
+        }
+        if pack_name == "warbler-pack-hf-arxiv"
+        else {"chunked": False},
+    ):
+        allow_patterns = loader.build_allow_patterns()
+
+    assert "packs/warbler-pack-core/package.json" in allow_patterns
+    assert "packs/warbler-pack-core/pack/templates.json" in allow_patterns
+    assert "packs/warbler-pack-hf-arxiv/package.json" in allow_patterns
+    assert "packs/warbler-pack-hf-arxiv/warbler-pack-hf-arxiv-chunk-001.jsonl" in allow_patterns
+    assert "packs/warbler-pack-hf-arxiv/warbler-pack-hf-arxiv-chunk-002.jsonl" not in allow_patterns
+
+
+def test_from_environment_applies_hosted_defaults_for_remote_loader():
+    from warbler_cda.remote_pack_loader import RemotePackLoader
+
+    with patch.dict("os.environ", {"SPACE_ID": "Bellok/warbler-cda"}, clear=False):
+        loader = RemotePackLoader.from_environment("Bellok/warbler-cda-corpus")
+
+    assert loader.max_documents_per_pack == 5000
+    assert "warbler-pack-hf-tinystories" in loader.exclude_packs

warbler_cda/remote_pack_loader.py

@@ -1,26 +1,175 @@
-"""RemotePackLoader: Download Warbler packs from a Hugging Face dataset repo."""
+"""Download hosted Warbler packs selectively from a Hugging Face dataset repo."""
+
+import fnmatch
+import json
+import logging
+import math
 import os
 from pathlib import Path
-from typing import List, Optional
-from huggingface_hub import snapshot_download
+from typing import Dict, List, Optional
+
+from huggingface_hub import HfApi, hf_hub_download, snapshot_download
+
+logger = logging.getLogger(__name__)
+
 
 class RemotePackLoader:
-    def __init__(self, repo_id: str, cache_dir: Optional[str] = None, allow_patterns: Optional[List[str]] = None):
+    """Download only the pack files needed for hosted-safe startup."""
+
+    def __init__(
+        self,
+        repo_id: str,
+        cache_dir: Optional[str] = None,
+        include_packs: Optional[List[str]] = None,
+        exclude_packs: Optional[List[str]] = None,
+        max_documents_per_pack: Optional[int] = None,
+        token: Optional[str] = None,
+    ):
         self.repo_id = repo_id
         self.cache_dir = cache_dir or os.getenv("HF_PACK_CACHE", ".hf_pack_cache")
-        self.allow_patterns = allow_patterns or ["packs/*"]
-        self.local_dir = None
+        self.include_packs = include_packs or []
+        self.exclude_packs = exclude_packs or []
+        self.max_documents_per_pack = max_documents_per_pack
+        self.local_dir: Optional[Path] = None
+        self.api = HfApi(token=token)
+
+    @classmethod
+    def from_environment(cls, repo_id: str):
+        """Create a remote loader configured with the same hosted-safe defaults as PackLoader."""
+        include_packs = cls._split_csv_env("WARBLER_INCLUDE_PACKS")
+        exclude_packs = cls._split_csv_env("WARBLER_EXCLUDE_PACKS")
+        max_documents_per_pack = cls._parse_int_env("WARBLER_MAX_DOCUMENTS_PER_PACK")
+
+        if cls._is_hosted_environment():
+            if not exclude_packs:
+                exclude_packs = ["warbler-pack-hf-tinystories"]
+            if max_documents_per_pack is None:
+                max_documents_per_pack = 5000
+
+        return cls(
+            repo_id=repo_id,
+            include_packs=include_packs,
+            exclude_packs=exclude_packs,
+            max_documents_per_pack=max_documents_per_pack,
+            token=os.getenv("HF_TOKEN"),
+        )
+
+    @staticmethod
+    def _is_hosted_environment() -> bool:
+        hosted_flag = os.getenv("WARBLER_HOSTED_MODE", "").lower()
+        return hosted_flag in {"1", "true", "yes", "on"} or bool(
+            os.getenv("SPACE_ID") or os.getenv("HF_SPACE_ID")
+        )
+
+    @staticmethod
+    def _split_csv_env(name: str) -> List[str]:
+        raw_value = os.getenv(name, "")
+        return [part.strip() for part in raw_value.split(",") if part.strip()]
+
+    @staticmethod
+    def _parse_int_env(name: str) -> Optional[int]:
+        raw_value = os.getenv(name)
+        if raw_value in (None, ""):
+            return None
+        try:
+            return int(raw_value)
+        except ValueError:
+            logger.warning("Ignoring invalid integer for %s: %s", name, raw_value)
+            return None
+
+    def _should_load_pack(self, pack_name: str) -> bool:
+        if self.include_packs:
+            included = any(fnmatch.fnmatch(pack_name, pattern) for pattern in self.include_packs)
+            if not included:
+                return False
+
+        if self.exclude_packs:
+            excluded = any(fnmatch.fnmatch(pack_name, pattern) for pattern in self.exclude_packs)
+            if excluded:
+                return False
+
+        return True
+
+    def _list_repo_files(self) -> List[str]:
+        return self.api.list_repo_files(repo_id=self.repo_id, repo_type="dataset")
+
+    def _load_pack_metadata(self, pack_name: str) -> Dict[str, object]:
+        metadata_path = f"packs/{pack_name}/package.json"
+        try:
+            downloaded_path = hf_hub_download(
+                repo_id=self.repo_id,
+                repo_type="dataset",
+                filename=metadata_path,
+                cache_dir=self.cache_dir,
+                token=os.getenv("HF_TOKEN"),
+            )
+        except Exception:
+            return {}
+
+        try:
+            return json.loads(Path(downloaded_path).read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            logger.warning("Failed to parse remote metadata for %s", pack_name)
+            return {}
+
+    def build_allow_patterns(self) -> List[str]:
+        """Build a minimal file set for the selected remote packs."""
+        repo_files = self._list_repo_files()
+        pack_files: Dict[str, List[str]] = {}
+
+        for repo_file in repo_files:
+            parts = Path(repo_file).parts
+            if len(parts) < 3 or parts[0] != "packs":
+                continue
+            pack_name = parts[1]
+            if not self._should_load_pack(pack_name):
+                continue
+            pack_files.setdefault(pack_name, []).append(repo_file)
+
+        allow_patterns: List[str] = []
+        for pack_name in sorted(pack_files):
+            files = sorted(pack_files[pack_name])
+            metadata = self._load_pack_metadata(pack_name)
+            package_json = f"packs/{pack_name}/package.json"
+            if package_json in files:
+                allow_patterns.append(package_json)
+
+            templates_path = f"packs/{pack_name}/pack/templates.json"
+            if templates_path in files:
+                allow_patterns.append(templates_path)
+
+            jsonl_files = [path for path in files if path.endswith(".jsonl")]
+            if metadata.get("chunked"):
+                docs_per_chunk = metadata.get("docs_per_chunk")
+                chunk_limit = None
+                if self.max_documents_per_pack and isinstance(docs_per_chunk, int) and docs_per_chunk > 0:
+                    chunk_limit = max(1, math.ceil(self.max_documents_per_pack / docs_per_chunk))
+                elif self.max_documents_per_pack:
+                    chunk_limit = 1
+
+                selected_jsonl = jsonl_files[:chunk_limit] if chunk_limit is not None else jsonl_files
+            else:
+                preferred_jsonl = f"packs/{pack_name}/{pack_name}.jsonl"
+                selected_jsonl = [preferred_jsonl] if preferred_jsonl in jsonl_files else jsonl_files[:1]
+
+            allow_patterns.extend(selected_jsonl)
+
+        return allow_patterns
 
     def fetch_packs(self) -> Path:
-        """Download the packs directory from the remote dataset repo."""
+        """Download only the selected pack files from the remote dataset repo."""
+        allow_patterns = self.build_allow_patterns()
+        if not allow_patterns:
+            raise RuntimeError(f"No remote pack files selected for repo {self.repo_id}")
+
         self.local_dir = Path(
             snapshot_download(
                 repo_id=self.repo_id,
                 repo_type="dataset",
                 cache_dir=self.cache_dir,
-                allow_patterns=self.allow_patterns,
+                allow_patterns=allow_patterns,
                 local_files_only=False,
-                resume_download=True,
+                token=os.getenv("HF_TOKEN"),
             )
         )
         return self.local_dir / "packs"