test: add full site template matrix coverage

- add registry tests that validate every template domain maps correctly
- add API tests for /api/sites list/get/match endpoints
- add site template matrix report under docs/test
- validate 56 template catalog with comprehensive assertions

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

backend/tests/test_api/test_sites.py

@@ -0,0 +1,86 @@
+"""Tests for site template API endpoints."""
+
+from fastapi.testclient import TestClient
+
+
+class TestSitesAPI:
+    """Validate /api/sites template routes."""
+
+    def test_list_sites_returns_minimum_templates(self, client: TestClient) -> None:
+        """List endpoint should expose a rich inbuilt catalog."""
+
+        response = client.get("/api/sites")
+        assert response.status_code == 200
+
+        data = response.json()
+        assert "sites" in data
+        assert "count" in data
+        assert data["count"] >= 30
+        assert len(data["sites"]) >= 30
+
+        site_ids = {site["site_id"] for site in data["sites"]}
+        assert "reddit" in site_ids
+        assert "github" in site_ids
+        assert "youtube" in site_ids
+
+    def test_get_specific_site_template(self, client: TestClient) -> None:
+        """Fetch one known site template."""
+
+        response = client.get("/api/sites/reddit")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["site_id"] == "reddit"
+        assert "reddit.com" in data["domains"]
+        assert "navigation_steps" in data
+        assert len(data["navigation_steps"]) > 0
+
+    def test_get_unknown_site_template_404(self, client: TestClient) -> None:
+        """Unknown site IDs should return 404."""
+
+        response = client.get("/api/sites/not-a-real-site")
+        assert response.status_code == 404
+
+    def test_match_site_by_asset_domain(self, client: TestClient) -> None:
+        """Domain matching should pick correct template."""
+
+        response = client.post(
+            "/api/sites/match",
+            json={
+                "instructions": "get trending communities",
+                "assets": ["https://reddit.com"],
+            },
+        )
+        assert response.status_code == 200
+        payload = response.json()
+        assert payload["matched"] is True
+        assert payload["site"]["site_id"] == "reddit"
+
+    def test_match_site_by_instruction_alias(self, client: TestClient) -> None:
+        """Alias matching should work even when URL is missing."""
+
+        response = client.post(
+            "/api/sites/match",
+            json={
+                "instructions": "scrape latest youtube videos",
+                "assets": [],
+            },
+        )
+        assert response.status_code == 200
+        payload = response.json()
+        assert payload["matched"] is True
+        assert payload["site"]["site_id"] == "youtube"
+
+    def test_match_site_returns_false_for_unknown(self, client: TestClient) -> None:
+        """Matcher should return matched=false when no template fits."""
+
+        response = client.post(
+            "/api/sites/match",
+            json={
+                "instructions": "scrape intranet dashboard",
+                "assets": ["https://internal.local.example"],
+            },
+        )
+        assert response.status_code == 200
+        payload = response.json()
+        assert payload["matched"] is False
+        assert payload["site"] is None

backend/tests/test_sites/test_registry.py

@@ -0,0 +1,56 @@
+"""Comprehensive tests for all built-in site templates."""
+
+from fastapi.testclient import TestClient
+
+from app.api.routes.scrape import _create_intelligent_navigation_plan
+from app.sites.registry import list_site_templates, match_site_template
+from app.sites.templates import SITE_TEMPLATES
+
+
+def test_site_template_catalog_is_large_and_unique() -> None:
+    """Catalog should contain many unique templates for production coverage."""
+
+    assert len(SITE_TEMPLATES) >= 50
+    ids = [template.site_id for template in SITE_TEMPLATES]
+    assert len(ids) == len(set(ids))
+
+
+def test_every_template_matches_its_primary_domain() -> None:
+    """Every template must be discoverable from its primary domain."""
+
+    for template in SITE_TEMPLATES:
+        primary_domain = template.domains[0]
+        matched = match_site_template("extract structured data", [f"https://{primary_domain}"])
+        assert matched is not None, f"No match for {template.site_id} ({primary_domain})"
+        assert matched.site_id == template.site_id
+
+
+def test_every_template_generates_site_aware_navigation_plan() -> None:
+    """Planner should attach site template metadata for all known domains."""
+
+    for template in SITE_TEMPLATES:
+        primary_domain = template.domains[0]
+        plan = _create_intelligent_navigation_plan("collect entries", [f"https://{primary_domain}"])
+        assert plan.get("site_template_id") == template.site_id
+        assert plan.get("site_template_name") == template.name
+
+
+def test_every_template_available_through_api(client: TestClient) -> None:
+    """Each template should be retrievable via /api/sites/{site_id}."""
+
+    for template in SITE_TEMPLATES:
+        response = client.get(f"/api/sites/{template.site_id}")
+        assert response.status_code == 200, f"API retrieval failed for {template.site_id}"
+        payload = response.json()
+        assert payload["site_id"] == template.site_id
+        assert len(payload["domains"]) >= 1
+        assert "navigation_steps" in payload
+
+
+def test_registry_serialization_covers_all_templates() -> None:
+    """Serialized registry output should include every template exactly once."""
+
+    serialized = list_site_templates()
+    serialized_ids = {item["site_id"] for item in serialized}
+    template_ids = {template.site_id for template in SITE_TEMPLATES}
+    assert serialized_ids == template_ids

docs/test/site_template_matrix_report.md

@@ -0,0 +1,74 @@
+# Site Template Matrix Test Report
+
+**Date:** 2026-04-05  
+**Scope:** Backend site-template registry, agent integration, and full template coverage tests
+
+## Summary
+
+- Inbuilt templates expanded to **56 sites**
+- Agents now load template context during planning/navigation
+- New API surface added: `/api/sites`, `/api/sites/{site_id}`, `/api/sites/match`
+- Full template test suite added and passing
+
+## Automated Tests
+
+Command:
+
+```bash
+cd backend
+python -m pytest tests/test_sites/test_registry.py tests/test_api/test_sites.py -q
+```
+
+Result:
+
+- **11 passed**
+- Coverage includes:
+  - catalog size and uniqueness
+  - domain matching for every template
+  - navigation-plan site-template propagation for every template
+  - API retrieval for every template
+  - registry serialization completeness
+
+## Runtime Validation
+
+### 1. Template catalog endpoint
+
+- `GET /api/sites`
+- Result: `count = 56`
+
+### 2. Template match endpoint
+
+- `POST /api/sites/match` with `https://reddit.com`
+- Result: `matched = true`, `site_id = reddit`
+
+### 3. Agent template self-reference
+
+Reddit scrape stream validation confirmed:
+
+- `site_template` step emitted by navigator
+- `planner_python.extracted_data.site_template_id = reddit`
+- `navigator_python.extracted_data.site_template_id = reddit`
+
+### 4. Strategy integration checks
+
+- Reddit request → `navigation_strategy = reddit_trending`
+- GitHub trending request → `navigation_strategy = github_trending`
+- Generic known domains (e.g., YouTube) → `site_template_id` populated, strategy-aware exploration
+
+## Folder Structure Additions
+
+```text
+backend/app/sites/
+  __init__.py
+  models.py
+  templates.py
+  registry.py
+
+backend/tests/test_sites/
+  test_registry.py
+```
+
+## Notes
+
+- Reddit direct endpoints are network-blocked in this environment; scraper uses fallback strategy while still preserving template-aware agent flow.
+- Template-aware events are now visible in execution trace for debugging and orchestration transparency.