Add curated legal scraper tab

README.md

@@ -15,6 +15,7 @@ CPU-first Hugging Face Space for a GameMaster game design assistant. It provides
 - A Gradio UI at `/ui`
 - A plugin-friendly JSON endpoint at `/api/chat`
 - A local RAG ingestion pipeline driven by `sources.yaml`
+- A Legal Scraper tab for curated open game-design resources
 - A pluggable LLM layer with a CPU-safe fallback and optional local GGUF model support
 
 The project intentionally ships with no scraped corpus. Add only sources you own, public-domain sources, permissively licensed sources, or sources where you have explicit permission.
@@ -95,6 +96,29 @@ python -m gamemaster_copilot.ingest --embedding-backend sentence-transformers
 
 Index files are written to `data/index/`.
 
+## Legal Scraper Tab
+
+The UI includes a `Legal Scraper` tab that can scrape a curated allowlist and rebuild the RAG index. It does not crawl arbitrary sites.
+
+Current catalog:
+
+- Game Design Concepts course, `CC BY 3.0 US`
+- Wikipedia game design topics, `CC BY-SA 4.0 / GFDL`
+- MIT OCW CMS.608 Game Design, `CC BY-NC-SA`
+- Fate Core SRD, `CC BY 3.0 Unported`
+- Blades in the Dark SRD, `CC BY 3.0 Unported`
+- D&D Beyond SRD, `CC BY 4.0`
+
+Legal guardrails:
+
+- Only allowlisted catalog sources are scraped.
+- `robots.txt` is checked before web page fetches.
+- Binary assets, PDFs, images, scripts, and stylesheets are skipped.
+- Every chunk stores source URL, license, attribution, and tags.
+- Noncommercial/share-alike sources are flagged in the catalog metadata.
+
+On Hugging Face free CPU, index builds can be slow. Start with a low `Max documents per source`, then raise it when the Space has enough time to complete ingestion. Runtime-built indexes are container-local unless you configure persistent storage or commit generated `data/index/` artifacts.
+
 ## Local Run
 
 ```bash

app.py

@@ -6,6 +6,7 @@ import inspect
 
 from fastapi import FastAPI
 
+from gamemaster_copilot.catalog import build_catalog_index, catalog_summary_markdown, get_catalog
 from gamemaster_copilot.config import MODES, get_settings
 from gamemaster_copilot.schemas import ChatRequest, ChatResponse
 from gamemaster_copilot.service import CopilotService
@@ -53,6 +54,14 @@ def _build_gradio_ui():
     except Exception:
         return None
 
+    catalog_entries = get_catalog()
+    catalog_choice_labels = [
+        f"{entry.id} | {entry.label} | {entry.license}" for entry in catalog_entries
+    ]
+    catalog_label_to_id = {
+        f"{entry.id} | {entry.label} | {entry.license}": entry.id for entry in catalog_entries
+    }
+
     def submit(message: str, history: list[dict], project_context: str, mode: str, retrieval_k: int):
         history = history or []
         if not message or not message.strip():
@@ -71,6 +80,43 @@ def _build_gradio_ui():
         ]
         return "", history
 
+    def refresh_index_status():
+        return service.health()
+
+    def run_legal_scraper(selected_labels: list[str], max_docs_per_source: int):
+        selected_labels = selected_labels or []
+        selected_ids = [catalog_label_to_id[label] for label in selected_labels if label in catalog_label_to_id]
+        if not selected_ids:
+            return "Select at least one catalog source before scraping.", service.health()
+
+        try:
+            manifest = build_catalog_index(
+                selected_ids=selected_ids,
+                index_dir=settings.index_dir,
+                embedding_backend=settings.embedding_backend,
+                embedding_model=settings.embedding_model,
+                embedding_dimensions=settings.embedding_dimensions,
+                max_docs_per_source=int(max_docs_per_source),
+            )
+            health = service.reload_index()
+        except Exception as exc:
+            return f"Scrape failed: `{exc}`", service.health()
+
+        warning_text = ""
+        if manifest.get("warnings"):
+            warning_text = "\n\nWarnings:\n" + "\n".join(f"- {warning}" for warning in manifest["warnings"][:20])
+            if len(manifest["warnings"]) > 20:
+                warning_text += f"\n- ... {len(manifest['warnings']) - 20} more warnings"
+
+        summary = (
+            f"Built legal catalog index with **{manifest['scraped_document_count']} documents** "
+            f"and **{manifest['chunk_count']} chunks** from **{len(selected_ids)} catalog sources**.\n\n"
+            f"Embedding backend: `{manifest['embedding_backend']}`\n\n"
+            f"The chat retrieval layer has been reloaded. Current index chunks: `{health['chunk_count']}`."
+            f"{warning_text}"
+        )
+        return summary, manifest
+
     with gr.Blocks(title="GameMaster Design Copilot") as demo:
         gr.HTML(
             """
@@ -91,54 +137,89 @@ def _build_gradio_ui():
             </div>
             """
         )
-        with gr.Row():
-            with gr.Column(scale=3):
-                chatbot_kwargs = {"label": "Design conversation", "height": 520}
-                if "type" in inspect.signature(gr.Chatbot).parameters:
-                    chatbot_kwargs["type"] = "messages"
-                chatbot = gr.Chatbot(**chatbot_kwargs)
-                message = gr.Textbox(
-                    label="Design request",
-                    placeholder="Example: Critique this stamina system for dominant strategies...",
-                    lines=3,
-                )
+        with gr.Tabs():
+            with gr.Tab("Copilot"):
                 with gr.Row():
-                    send = gr.Button("Send", variant="primary")
-                    clear = gr.Button("Clear")
-            with gr.Column(scale=1):
-                mode = gr.Dropdown(
-                    choices=list(MODES.keys()),
-                    value="brainstorm",
-                    label="Mode",
-                )
-                retrieval_k = gr.Slider(
-                    minimum=0,
-                    maximum=10,
-                    value=4,
-                    step=1,
-                    label="Retrieved chunks",
+                    with gr.Column(scale=3):
+                        chatbot_kwargs = {"label": "Design conversation", "height": 520}
+                        if "type" in inspect.signature(gr.Chatbot).parameters:
+                            chatbot_kwargs["type"] = "messages"
+                        chatbot = gr.Chatbot(**chatbot_kwargs)
+                        message = gr.Textbox(
+                            label="Design request",
+                            placeholder="Example: Critique this stamina system for dominant strategies...",
+                            lines=3,
+                        )
+                        with gr.Row():
+                            send = gr.Button("Send", variant="primary")
+                            clear = gr.Button("Clear")
+                    with gr.Column(scale=1):
+                        mode = gr.Dropdown(
+                            choices=list(MODES.keys()),
+                            value="brainstorm",
+                            label="Mode",
+                        )
+                        retrieval_k = gr.Slider(
+                            minimum=0,
+                            maximum=10,
+                            value=4,
+                            step=1,
+                            label="Retrieved chunks",
+                        )
+                        project_context = gr.Textbox(
+                            label="Project context",
+                            placeholder="Genre, target audience, platform, rules constraints, design goals...",
+                            lines=14,
+                        )
+                        gr.Markdown(
+                            "Grounded answers require approved sources and a built RAG index. "
+                            "Use the Legal Scraper tab to build one from the curated catalog."
+                        )
+
+                send.click(
+                    submit,
+                    inputs=[message, chatbot, project_context, mode, retrieval_k],
+                    outputs=[message, chatbot],
                 )
-                project_context = gr.Textbox(
-                    label="Project context",
-                    placeholder="Genre, target audience, platform, rules constraints, design goals...",
-                    lines=14,
+                message.submit(
+                    submit,
+                    inputs=[message, chatbot, project_context, mode, retrieval_k],
+                    outputs=[message, chatbot],
                 )
+                clear.click(lambda: [], outputs=[chatbot])
+
+            with gr.Tab("Legal Scraper"):
                 gr.Markdown(
-                    "The default app can answer without an index, but grounded answers require approved sources "
-                    "and a built RAG index."
+                    "This scraper is intentionally allowlist-only. It does not crawl arbitrary sites, "
+                    "does not ingest unclear copyrighted material, skips binary assets, checks robots.txt, "
+                    "and preserves license/attribution metadata in every chunk."
                 )
-
-        send.click(
-            submit,
-            inputs=[message, chatbot, project_context, mode, retrieval_k],
-            outputs=[message, chatbot],
-        )
-        message.submit(
-            submit,
-            inputs=[message, chatbot, project_context, mode, retrieval_k],
-            outputs=[message, chatbot],
-        )
-        clear.click(lambda: [], outputs=[chatbot])
+                gr.Markdown(catalog_summary_markdown())
+                source_picker = gr.CheckboxGroup(
+                    choices=catalog_choice_labels,
+                    value=catalog_choice_labels,
+                    label="Catalog sources to scrape",
+                )
+                max_docs = gr.Slider(
+                    minimum=1,
+                    maximum=80,
+                    value=20,
+                    step=1,
+                    label="Max documents per source",
+                    info="Use a lower number for faster CPU Space runs; raise it for deeper crawling.",
+                )
+                with gr.Row():
+                    scrape_button = gr.Button("Scrape Selected Sources And Rebuild Index", variant="primary")
+                    status_button = gr.Button("Refresh Index Status")
+                scrape_status = gr.Markdown()
+                index_manifest = gr.JSON(label="Index manifest / status")
+
+                scrape_button.click(
+                    run_legal_scraper,
+                    inputs=[source_picker, max_docs],
+                    outputs=[scrape_status, index_manifest],
+                )
+                status_button.click(refresh_index_status, outputs=[index_manifest])
     return demo
 
 

gamemaster_copilot/catalog.py

@@ -0,0 +1,502 @@
+"""Curated legal source catalog and scraper-backed index builder."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import quote, urljoin, urlparse, urlunparse
+from urllib.robotparser import RobotFileParser
+
+import numpy as np
+import requests
+from bs4 import BeautifulSoup
+
+from .embeddings import create_embedding_backend
+from .index_store import save_index
+from .text import TextChunk, chunk_text, html_to_text, normalize_text
+
+
+DEFAULT_USER_AGENT = "GameMasterCopilot/0.1 (+https://huggingface.co/spaces/vivekchakraverty/gamemaster-design-copilot)"
+SKIP_EXTENSIONS = {
+    ".7z",
+    ".avi",
+    ".css",
+    ".gif",
+    ".gz",
+    ".ico",
+    ".jpeg",
+    ".jpg",
+    ".js",
+    ".json",
+    ".mp3",
+    ".mp4",
+    ".ogg",
+    ".pdf",
+    ".png",
+    ".svg",
+    ".tar",
+    ".webm",
+    ".zip",
+}
+
+
+@dataclass(frozen=True)
+class ScrapedDocument:
+    """One fetched document ready for chunking."""
+
+    source_id: str
+    title: str
+    text: str
+    url: str
+    license: str
+    attribution: str
+    tags: list[str]
+
+
+@dataclass(frozen=True)
+class CatalogEntry:
+    """One allowlisted source collection."""
+
+    id: str
+    label: str
+    description: str
+    license: str
+    permission: str
+    attribution: str
+    tags: list[str]
+    kind: str
+    source_url: str
+    start_urls: tuple[str, ...] = ()
+    allowed_prefixes: tuple[str, ...] = ()
+    api_url: str | None = None
+    pages: tuple[str, ...] = ()
+    notes: str = ""
+
+
+WIKIPEDIA_GAME_DESIGN_PAGES = (
+    "Game design",
+    "Video game design",
+    "Game mechanics",
+    "Gameplay",
+    "Game balance",
+    "Level design",
+    "Playtesting",
+    "Emergent gameplay",
+    "Nonlinear gameplay",
+    "Game studies",
+    "Ludology",
+    "Flow (psychology)",
+    "Tutorial (video games)",
+    "Boss (video games)",
+    "Game artificial intelligence",
+    "Serious game",
+)
+
+
+CATALOG: tuple[CatalogEntry, ...] = (
+    CatalogEntry(
+        id="game_design_concepts",
+        label="Game Design Concepts course",
+        description="Ian Schreiber's free game design course posts.",
+        license="CC BY 3.0 US",
+        permission="permissive",
+        attribution="Game Design Concepts by Ian Schreiber",
+        tags=["game-design", "course", "mechanics", "iteration", "balance"],
+        kind="wordpress_rest",
+        source_url="https://gamedesignconcepts.wordpress.com/about/",
+        api_url="https://gamedesignconcepts.wordpress.com/wp-json/wp/v2/posts",
+        notes="The course about page states that course content may be shared and adapted with attribution.",
+    ),
+    CatalogEntry(
+        id="wikipedia_game_design",
+        label="Wikipedia game design topics",
+        description="Selected game design, mechanics, level design, and playtesting encyclopedia topics.",
+        license="CC BY-SA 4.0 / GFDL",
+        permission="sharealike_open",
+        attribution="Wikipedia contributors",
+        tags=["game-design", "mechanics", "level-design", "encyclopedia"],
+        kind="mediawiki_pages",
+        source_url="https://en.wikipedia.org/wiki/Wikipedia:Copyrights",
+        api_url="https://api.wikimedia.org/core/v1/wikipedia/en/page",
+        pages=WIKIPEDIA_GAME_DESIGN_PAGES,
+        notes="Text reuse requires attribution and share-alike compliance.",
+    ),
+    CatalogEntry(
+        id="mit_ocw_cms608_2014",
+        label="MIT OCW CMS.608 Game Design",
+        description="MIT OpenCourseWare Game Design course pages from Spring 2014.",
+        license="CC BY-NC-SA",
+        permission="noncommercial_open",
+        attribution="MIT OpenCourseWare, CMS.608 Game Design, Spring 2014",
+        tags=["game-design", "course", "non-digital-games", "assignments"],
+        kind="crawl_prefix",
+        source_url="https://ocw.mit.edu/courses/cms-608-game-design-spring-2014/",
+        start_urls=("https://ocw.mit.edu/courses/cms-608-game-design-spring-2014/",),
+        allowed_prefixes=("https://ocw.mit.edu/courses/cms-608-game-design-spring-2014/",),
+        notes="MIT OCW materials are open, but this catalog entry is noncommercial/share-alike.",
+    ),
+    CatalogEntry(
+        id="fate_core_srd",
+        label="Fate Core SRD",
+        description="Fate Core SRD pages for tabletop RPG system design examples.",
+        license="CC BY 3.0 Unported",
+        permission="permissive",
+        attribution="Fate Core System by Evil Hat Productions and Fate SRD contributors",
+        tags=["ttrpg", "srd", "narrative-design", "resolution-mechanics"],
+        kind="crawl_prefix",
+        source_url="https://fate-srd.com/official-licensing-fate",
+        start_urls=("https://fate-srd.com/fate-core/", "https://fate-srd.com/official-licensing-fate"),
+        allowed_prefixes=("https://fate-srd.com/fate-core/", "https://fate-srd.com/official-licensing-fate"),
+        notes="The official licensing page identifies Fate SRD as the endorsed SRD source.",
+    ),
+    CatalogEntry(
+        id="blades_srd",
+        label="Blades in the Dark SRD",
+        description="Forged in the Dark SRD pages for position/effect, clocks, crews, and action resolution.",
+        license="CC BY 3.0 Unported",
+        permission="permissive",
+        attribution="Blades in the Dark by One Seven Design, developed and authored by John Harper",
+        tags=["ttrpg", "srd", "fiction-first", "clocks", "gm-tools"],
+        kind="crawl_prefix",
+        source_url="https://bladesinthedark.com/licensing",
+        start_urls=("https://bladesinthedark.com/basics/", "https://bladesinthedark.com/licensing"),
+        allowed_prefixes=("https://bladesinthedark.com/",),
+        notes="Catalog notes warn users not to use non-SRD setting, NPC, artwork, or map material.",
+    ),
+    CatalogEntry(
+        id="dnd_srd",
+        label="D&D Beyond SRD",
+        description="Official D&D SRD page for rules/mechanics examples under Creative Commons.",
+        license="CC BY 4.0",
+        permission="permissive",
+        attribution="System Reference Document by Wizards of the Coast LLC",
+        tags=["ttrpg", "srd", "rules", "encounters", "mechanics"],
+        kind="single_pages",
+        source_url="https://www.dndbeyond.com/srd",
+        start_urls=("https://www.dndbeyond.com/srd",),
+        notes="Only SRD content is included; trademarks and non-SRD material remain outside scope.",
+    ),
+)
+
+
+def get_catalog() -> list[CatalogEntry]:
+    """Return the curated legal source catalog."""
+
+    return list(CATALOG)
+
+
+def get_catalog_entry(entry_id: str) -> CatalogEntry:
+    for entry in CATALOG:
+        if entry.id == entry_id:
+            return entry
+    raise KeyError(f"unknown catalog source: {entry_id}")
+
+
+def catalog_summary_markdown() -> str:
+    lines = [
+        "### Curated Legal Source Catalog",
+        "",
+        "Only these allowlisted sources can be scraped by the UI. Robots.txt is checked for web pages, "
+        "binary assets are skipped, and each chunk keeps license and attribution metadata.",
+        "",
+    ]
+    for entry in CATALOG:
+        lines.append(f"- **{entry.label}** (`{entry.id}`): {entry.license}. {entry.description}")
+    return "\n".join(lines)
+
+
+def _safe_slug(value: str) -> str:
+    cleaned = "".join(char.lower() if char.isalnum() else "-" for char in value)
+    cleaned = "-".join(part for part in cleaned.split("-") if part)
+    return cleaned[:80] or hashlib.sha256(value.encode("utf-8")).hexdigest()[:16]
+
+
+def _canonical_url(url: str) -> str:
+    parsed = urlparse(url)
+    return urlunparse((parsed.scheme, parsed.netloc, parsed.path.rstrip("/") + "/", "", "", ""))
+
+
+def _skip_url(url: str) -> bool:
+    path = urlparse(url).path.lower()
+    return any(path.endswith(extension) for extension in SKIP_EXTENSIONS)
+
+
+def _robots_allowed(url: str, user_agent: str) -> bool:
+    parsed = urlparse(url)
+    robots_url = urljoin(f"{parsed.scheme}://{parsed.netloc}", "/robots.txt")
+    try:
+        response = requests.get(robots_url, headers={"User-Agent": user_agent}, timeout=10)
+    except Exception:
+        return False
+    if response.status_code == 404:
+        return True
+    if response.status_code >= 400:
+        return False
+    parser = RobotFileParser()
+    parser.set_url(robots_url)
+    parser.parse(response.text.splitlines())
+    return parser.can_fetch(user_agent, url)
+
+
+def _fetch(url: str, user_agent: str, timeout_seconds: int) -> requests.Response:
+    if not _robots_allowed(url, user_agent):
+        raise PermissionError(f"robots.txt disallows fetching {url}")
+    response = requests.get(url, headers={"User-Agent": user_agent}, timeout=timeout_seconds)
+    response.raise_for_status()
+    return response
+
+
+def _document_from_html(entry: CatalogEntry, url: str, html: str, fallback_title: str | None = None) -> ScrapedDocument:
+    soup = BeautifulSoup(html, "html.parser")
+    title_node = soup.find("h1") or soup.find("title")
+    title = normalize_text(title_node.get_text(" ")) if title_node else fallback_title or entry.label
+    text = html_to_text(html)
+    return ScrapedDocument(
+        source_id=f"{entry.id}.{_safe_slug(url)}",
+        title=title or entry.label,
+        text=text,
+        url=url,
+        license=entry.license,
+        attribution=entry.attribution,
+        tags=list(entry.tags),
+    )
+
+
+def _extract_links(html: str, base_url: str, allowed_prefixes: tuple[str, ...]) -> list[str]:
+    soup = BeautifulSoup(html, "html.parser")
+    links: list[str] = []
+    for anchor in soup.find_all("a", href=True):
+        url = urljoin(base_url, anchor["href"])
+        parsed = urlparse(url)
+        if parsed.scheme not in {"http", "https"}:
+            continue
+        candidate = urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", ""))
+        if _skip_url(candidate):
+            continue
+        if any(candidate.startswith(prefix) for prefix in allowed_prefixes):
+            links.append(candidate)
+    return links
+
+
+def _scrape_single_pages(entry: CatalogEntry, max_docs: int, user_agent: str, timeout_seconds: int) -> tuple[list[ScrapedDocument], list[str]]:
+    docs: list[ScrapedDocument] = []
+    warnings: list[str] = []
+    for url in entry.start_urls[:max_docs]:
+        try:
+            response = _fetch(url, user_agent, timeout_seconds)
+            docs.append(_document_from_html(entry, url, response.text))
+        except Exception as exc:
+            warnings.append(f"skipped {url}: {exc}")
+    return docs, warnings
+
+
+def _scrape_crawl_prefix(entry: CatalogEntry, max_docs: int, user_agent: str, timeout_seconds: int) -> tuple[list[ScrapedDocument], list[str]]:
+    docs: list[ScrapedDocument] = []
+    warnings: list[str] = []
+    queue = list(entry.start_urls)
+    seen: set[str] = set()
+
+    while queue and len(docs) < max_docs:
+        url = queue.pop(0)
+        canonical = _canonical_url(url)
+        if canonical in seen or _skip_url(url):
+            continue
+        seen.add(canonical)
+        if not any(url.startswith(prefix) for prefix in entry.allowed_prefixes):
+            continue
+
+        try:
+            response = _fetch(url, user_agent, timeout_seconds)
+            content_type = response.headers.get("content-type", "").lower()
+            if "text/html" not in content_type:
+                warnings.append(f"skipped non-html URL {url}: {content_type or 'unknown content type'}")
+                continue
+            docs.append(_document_from_html(entry, url, response.text))
+            for link in _extract_links(response.text, url, entry.allowed_prefixes):
+                if _canonical_url(link) not in seen and link not in queue:
+                    queue.append(link)
+            time.sleep(0.1)
+        except Exception as exc:
+            warnings.append(f"skipped {url}: {exc}")
+    return docs, warnings
+
+
+def _scrape_wordpress_rest(entry: CatalogEntry, max_docs: int, user_agent: str, timeout_seconds: int) -> tuple[list[ScrapedDocument], list[str]]:
+    if not entry.api_url:
+        return [], [f"{entry.id} is missing api_url"]
+
+    docs: list[ScrapedDocument] = []
+    warnings: list[str] = []
+    page = 1
+    while len(docs) < max_docs:
+        url = f"{entry.api_url}?per_page=20&page={page}&_fields=link,title,content"
+        try:
+            response = _fetch(url, user_agent, timeout_seconds)
+        except requests.HTTPError as exc:
+            if exc.response is not None and exc.response.status_code == 400:
+                break
+            warnings.append(f"stopped WordPress fetch at page {page}: {exc}")
+            break
+        except Exception as exc:
+            warnings.append(f"stopped WordPress fetch at page {page}: {exc}")
+            break
+
+        posts = response.json()
+        if not posts:
+            break
+        for post in posts:
+            title = html_to_text(post.get("title", {}).get("rendered", "")) or entry.label
+            link = post.get("link") or entry.source_url
+            content = post.get("content", {}).get("rendered", "")
+            text = html_to_text(content)
+            if not text:
+                continue
+            docs.append(
+                ScrapedDocument(
+                    source_id=f"{entry.id}.{_safe_slug(link)}",
+                    title=title,
+                    text=text,
+                    url=link,
+                    license=entry.license,
+                    attribution=entry.attribution,
+                    tags=list(entry.tags),
+                )
+            )
+            if len(docs) >= max_docs:
+                break
+        page += 1
+    return docs, warnings
+
+
+def _scrape_mediawiki_pages(entry: CatalogEntry, max_docs: int, user_agent: str, timeout_seconds: int) -> tuple[list[ScrapedDocument], list[str]]:
+    if not entry.api_url:
+        return [], [f"{entry.id} is missing api_url"]
+
+    docs: list[ScrapedDocument] = []
+    warnings: list[str] = []
+    for title in entry.pages[:max_docs]:
+        encoded_title = quote(title.replace(" ", "_"), safe="")
+        fetch_url = f"{entry.api_url.rstrip('/')}/{encoded_title}/html"
+        try:
+            response = _fetch(fetch_url, user_agent, timeout_seconds)
+            text = html_to_text(response.text)
+            if not text:
+                warnings.append(f"empty Wikipedia page extract: {title}")
+                continue
+            full_url = f"https://en.wikipedia.org/wiki/{encoded_title}"
+            docs.append(
+                ScrapedDocument(
+                    source_id=f"{entry.id}.{_safe_slug(title)}",
+                    title=title,
+                    text=text,
+                    url=full_url,
+                    license=entry.license,
+                    attribution=entry.attribution,
+                    tags=list(entry.tags),
+                )
+            )
+            time.sleep(0.1)
+        except Exception as exc:
+            warnings.append(f"skipped Wikipedia page {title}: {exc}")
+    return docs, warnings
+
+
+def scrape_catalog_entry(
+    entry: CatalogEntry,
+    *,
+    max_docs: int = 30,
+    user_agent: str = DEFAULT_USER_AGENT,
+    timeout_seconds: int = 20,
+) -> tuple[list[ScrapedDocument], list[str]]:
+    """Scrape one curated catalog entry."""
+
+    bounded_max_docs = max(1, min(max_docs, 200))
+    if entry.kind == "single_pages":
+        return _scrape_single_pages(entry, bounded_max_docs, user_agent, timeout_seconds)
+    if entry.kind == "crawl_prefix":
+        return _scrape_crawl_prefix(entry, bounded_max_docs, user_agent, timeout_seconds)
+    if entry.kind == "wordpress_rest":
+        return _scrape_wordpress_rest(entry, bounded_max_docs, user_agent, timeout_seconds)
+    if entry.kind == "mediawiki_pages":
+        return _scrape_mediawiki_pages(entry, bounded_max_docs, user_agent, timeout_seconds)
+    raise ValueError(f"unsupported catalog scraper kind: {entry.kind}")
+
+
+def build_catalog_index(
+    *,
+    selected_ids: list[str],
+    index_dir: str | Path,
+    embedding_backend: str,
+    embedding_model: str,
+    embedding_dimensions: int,
+    max_docs_per_source: int = 30,
+    chunk_words: int = 260,
+    overlap_words: int = 50,
+) -> dict:
+    """Scrape selected allowlisted catalog sources and persist a RAG index."""
+
+    if not selected_ids:
+        raise ValueError("select at least one catalog source")
+
+    entries = [get_catalog_entry(entry_id) for entry_id in selected_ids]
+    embedder = create_embedding_backend(
+        embedding_backend,
+        model_name=embedding_model,
+        dimensions=embedding_dimensions,
+    )
+
+    documents: list[ScrapedDocument] = []
+    chunks: list[TextChunk] = []
+    warnings: list[str] = []
+    seen_hashes: set[str] = set()
+
+    for entry in entries:
+        docs, entry_warnings = scrape_catalog_entry(entry, max_docs=max_docs_per_source)
+        warnings.extend(f"{entry.id}: {warning}" for warning in entry_warnings)
+        documents.extend(docs)
+        for doc in docs:
+            for chunk in chunk_text(
+                source_id=doc.source_id,
+                title=doc.title,
+                text=doc.text,
+                url=doc.url,
+                license=doc.license,
+                attribution=doc.attribution,
+                tags=doc.tags,
+                chunk_words=chunk_words,
+                overlap_words=overlap_words,
+            ):
+                digest = hashlib.sha256(chunk.text.encode("utf-8")).hexdigest()
+                if digest in seen_hashes:
+                    continue
+                seen_hashes.add(digest)
+                chunks.append(chunk)
+
+    if chunks:
+        vectors = embedder.encode([chunk.text for chunk in chunks])
+    else:
+        vectors = np.zeros((0, embedder.dimensions), dtype=np.float32)
+
+    manifest = {
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "source_registry": "curated_legal_catalog",
+        "legal_policy": "Curated allowlist only; robots.txt checked for web pages; binary assets skipped; citations retain license and attribution metadata.",
+        "selected_catalog_source_ids": selected_ids,
+        "registered_source_count": len(entries),
+        "scraped_document_count": len(documents),
+        "ingested_source_count": len({doc.source_id for doc in documents}),
+        "chunk_count": len(chunks),
+        "embedding_backend": embedder.name,
+        "embedding_dimensions": int(vectors.shape[1]) if vectors.ndim == 2 else embedder.dimensions,
+        "catalog": [asdict(entry) for entry in entries],
+        "warnings": warnings,
+    }
+    save_index(index_dir, chunks=chunks, vectors=vectors, manifest=manifest)
+
+    manifest_path = Path(index_dir) / "catalog-manifest.json"
+    manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
+    return manifest

gamemaster_copilot/service.py

@@ -35,6 +35,13 @@ class CopilotService:
             "index_dir": str(self.settings.index_dir),
         }
 
+    def reload_index(self) -> dict:
+        """Reload the on-disk vector index after ingestion."""
+
+        self.index.loaded = False
+        self.index.load()
+        return self.health()
+
     def chat(self, request: ChatRequest) -> ChatResponse:
         warnings: list[str] = []
         results: list[SearchResult] = []
@@ -70,4 +77,3 @@ class CopilotService:
             retrieved_chunks=[result.to_retrieved_chunk() for result in results],
             warnings=warnings,
         )
-

gamemaster_copilot/sources.py

@@ -9,7 +9,14 @@ import yaml
 from pydantic import BaseModel, Field, HttpUrl, model_validator
 
 
-Permission = Literal["public_domain", "permissive", "owned", "explicit_permission"]
+Permission = Literal[
+    "public_domain",
+    "permissive",
+    "sharealike_open",
+    "noncommercial_open",
+    "owned",
+    "explicit_permission",
+]
 
 
 class CrawlPolicy(BaseModel):

tests/test_catalog.py

@@ -0,0 +1,53 @@
+from pathlib import Path
+
+import gamemaster_copilot.catalog as catalog
+from gamemaster_copilot.catalog import ScrapedDocument, build_catalog_index, get_catalog
+
+
+def test_catalog_entries_have_required_legal_metadata() -> None:
+    entries = get_catalog()
+
+    assert entries
+    for entry in entries:
+        assert entry.id
+        assert entry.license
+        assert entry.permission in {"permissive", "sharealike_open", "noncommercial_open"}
+        assert entry.attribution
+        assert entry.source_url.startswith("https://")
+
+
+def test_build_catalog_index_with_mocked_scraper(tmp_path: Path, monkeypatch) -> None:
+    def fake_scrape(entry, *, max_docs=30, user_agent=catalog.DEFAULT_USER_AGENT, timeout_seconds=20):
+        return (
+            [
+                ScrapedDocument(
+                    source_id=f"{entry.id}.sample",
+                    title=f"{entry.label} Sample",
+                    text="Level design teaches mechanics by sequencing pressure, safety, discovery, and feedback.",
+                    url=entry.source_url,
+                    license=entry.license,
+                    attribution=entry.attribution,
+                    tags=entry.tags,
+                )
+            ],
+            [],
+        )
+
+    monkeypatch.setattr(catalog, "scrape_catalog_entry", fake_scrape)
+
+    manifest = build_catalog_index(
+        selected_ids=["wikipedia_game_design"],
+        index_dir=tmp_path / "index",
+        embedding_backend="hash",
+        embedding_model="unused",
+        embedding_dimensions=384,
+        max_docs_per_source=2,
+        chunk_words=12,
+        overlap_words=2,
+    )
+
+    assert manifest["scraped_document_count"] == 1
+    assert manifest["chunk_count"] > 0
+    assert manifest["selected_catalog_source_ids"] == ["wikipedia_game_design"]
+    assert (tmp_path / "index" / "catalog-manifest.json").exists()
+