Sync ctx b1e7e1e (part 3)

GitHub commit: b1e7e1ee28ac2147594e3d1f59009fbd0dd74647

src/mcp_add.py

@@ -719,7 +719,7 @@ def main() -> None:
             print(f"Error: {json_path} does not exist.", file=sys.stderr)
             sys.exit(1)
         try:
-            raw_records = [json.loads(json_path.read_text(encoding="utf-8"))]
+            raw_records = [json.loads(json_path.read_text(encoding="utf-8-sig"))]
         except json.JSONDecodeError as exc:
             print(f"Error: failed to parse JSON: {exc}", file=sys.stderr)
             sys.exit(1)
@@ -730,9 +730,9 @@ def main() -> None:
             print(f"Error: {jsonl_path} does not exist.", file=sys.stderr)
             sys.exit(1)
         for lineno, line in enumerate(
-            jsonl_path.read_text(encoding="utf-8").splitlines(), 1
+            jsonl_path.read_text(encoding="utf-8-sig").splitlines(), 1
         ):
-            line = line.strip()
+            line = line.lstrip("\ufeff").strip()
             if not line:
                 continue
             try:
@@ -742,7 +742,7 @@ def main() -> None:
 
     elif args.from_stdin:
         for lineno, line in enumerate(sys.stdin, 1):
-            line = line.strip()
+            line = line.lstrip("\ufeff").strip()
             if not line:
                 continue
             try:
@@ -767,6 +767,8 @@ def main() -> None:
         f"\nDone{dry_label}: {added} added, {merged} updated, "
         f"{reviewed} reviewed, {rejected} rejected, {errors} errors"
     )
+    if rejected or errors:
+        sys.exit(1)
 
 
 if __name__ == "__main__":

src/scan_repo.py

@@ -132,12 +132,15 @@ def read_json_safe(path: str) -> dict | None:
         return None
 
 
-def read_toml_deps(path: str) -> list[str]:
+def read_toml_deps(path: str, *, include_optional: bool = True) -> list[str]:
     """Extract dependency names from pyproject.toml.
 
     Covers PEP 621 ``[project].dependencies`` / ``optional-dependencies`` and
     Poetry-style ``[tool.poetry].dependencies`` / ``dev-dependencies``. Version
-    specifiers and extras are stripped via PEP 508 splitting.
+    specifiers and extras are stripped via PEP 508 splitting. Callers can set
+    ``include_optional=False`` when deciding the primary project type; optional
+    extras should remain searchable signals but should not make a docs/tooling
+    repo look like an ML project just because it offers an embeddings extra.
     """
     try:
         data = tomllib.loads(Path(path).read_text(encoding="utf-8-sig"))
@@ -152,15 +155,17 @@ def read_toml_deps(path: str) -> list[str]:
         deps = project.get("dependencies", [])
         if isinstance(deps, list):
             raw.extend(d for d in deps if isinstance(d, str))
-        opt = project.get("optional-dependencies", {})
-        if isinstance(opt, dict):
-            for group in opt.values():
-                if isinstance(group, list):
-                    raw.extend(d for d in group if isinstance(d, str))
+        if include_optional:
+            opt = project.get("optional-dependencies", {})
+            if isinstance(opt, dict):
+                for group in opt.values():
+                    if isinstance(group, list):
+                        raw.extend(d for d in group if isinstance(d, str))
 
     poetry = data.get("tool", {}).get("poetry", {}) if isinstance(data.get("tool"), dict) else {}
     if isinstance(poetry, dict):
-        for key in ("dependencies", "dev-dependencies"):
+        keys = ("dependencies", "dev-dependencies") if include_optional else ("dependencies",)
+        for key in keys:
             deps = poetry.get(key, {})
             if isinstance(deps, dict):
                 raw.extend(k for k in deps.keys() if isinstance(k, str) and k.lower() != "python")
@@ -222,6 +227,7 @@ def detect_stack(repo_path: str, signals: dict) -> dict:
 
     # Collect all deps from Python and JS configs
     all_py_deps: list[str] = []
+    core_py_deps: list[str] = []
     all_js_deps: list[str] = []
     pkg_json = None
 
@@ -229,10 +235,15 @@ def detect_stack(repo_path: str, signals: dict) -> dict:
         base = os.path.basename(cfg)
         if base == "pyproject.toml":
             all_py_deps.extend(read_toml_deps(cfg))
+            core_py_deps.extend(read_toml_deps(cfg, include_optional=False))
         elif base == "requirements.txt":
-            all_py_deps.extend(read_requirements(cfg))
+            deps = read_requirements(cfg)
+            all_py_deps.extend(deps)
+            core_py_deps.extend(deps)
         elif base == "Pipfile":
-            all_py_deps.extend(read_requirements(cfg))
+            deps = read_requirements(cfg)
+            all_py_deps.extend(deps)
+            core_py_deps.extend(deps)
         elif base == "package.json":
             data = read_json_safe(cfg)
             if data:
@@ -242,6 +253,7 @@ def detect_stack(repo_path: str, signals: dict) -> dict:
                         all_js_deps.extend(k.lower() for k in data[section])
 
     py_dep_set = set(all_py_deps)
+    py_core_dep_set = set(core_py_deps)
     js_dep_set = set(all_js_deps)
 
     # --- LANGUAGES ---
@@ -540,6 +552,14 @@ def detect_stack(repo_path: str, signals: dict) -> dict:
 
     # --- PROJECT TYPE ---
     fw_names = {f["name"] for f in profile["frameworks"]}
+    core_ml_deps = py_core_dep_set & {"torch", "pytorch", "tensorflow", "transformers"}
+    core_ai_deps = py_core_dep_set & {
+        "langchain",
+        "langchain-core",
+        "llama-index",
+        "crewai",
+        "dspy-ai",
+    }
     if fw_names & {"react", "vue", "angular", "svelte", "nextjs", "nuxt"}:
         if fw_names & {"fastapi", "django", "flask", "express", "nestjs"}:
             profile["project_type"] = "fullstack"
@@ -547,9 +567,9 @@ def detect_stack(repo_path: str, signals: dict) -> dict:
             profile["project_type"] = "frontend"
     elif fw_names & {"fastapi", "django", "flask", "express", "nestjs", "gin", "actix"}:
         profile["project_type"] = "api-service"
-    elif fw_names & {"pytorch", "tensorflow", "huggingface"}:
+    elif (fw_names & {"pytorch", "tensorflow", "huggingface"}) and core_ml_deps:
         profile["project_type"] = "ml-project"
-    elif fw_names & {"langchain", "llamaindex", "crewai"}:
+    elif (fw_names & {"langchain", "llamaindex", "crewai"}) and core_ai_deps:
         profile["project_type"] = "ai-agent"
     elif profile["infrastructure"]:
         profile["project_type"] = "infrastructure"

src/skill_add.py

@@ -394,7 +394,7 @@ def add_skill(
             f"Split the skill or trim content before ingestion."
         )
 
-    content = source_path.read_text(encoding="utf-8", errors="replace")
+    content = source_path.read_text(encoding="utf-8-sig", errors="replace")
     line_count = len(content.splitlines())
 
     installed_path = skills_dir / name / "SKILL.md"
@@ -711,6 +711,8 @@ def main() -> None:
         f"\nDone: {added} added, {updated} updated, {converted} converted, "
         f"{skipped} skipped, {errors} errors"
     )
+    if errors:
+        sys.exit(1)
 
 
 if __name__ == "__main__":

src/tests/test_agent_add.py

@@ -206,6 +206,25 @@ def test_new_agent_add_writes_converted_agent_mirror(
     assert mirror.read_text(encoding="utf-8") == source_text
 
 
+def test_bom_prefixed_agent_frontmatter_is_accepted(
+    tmp_path: Path,
+    monkeypatch: Any,
+) -> None:
+    wiki, agents_dir, source = _setup_paths(tmp_path)
+    source.write_text("\ufeff" + _agent_text(), encoding="utf-8")
+    _patch_side_effects(monkeypatch)
+
+    result = agent_add.add_agent(
+        source_path=source,
+        name="reviewer-agent",
+        wiki_path=wiki,
+        agents_dir=agents_dir,
+    )
+
+    assert result["is_new_page"] is True
+    assert (wiki / "entities" / "agents" / "reviewer-agent.md").exists()
+
+
 def test_existing_agent_update_refreshes_converted_agent_mirror(
     tmp_path: Path,
     monkeypatch: Any,

src/tests/test_ctx_monitor.py

@@ -3918,6 +3918,43 @@ def test_render_home_defers_sidecar_grade_scan(
     assert "data-home-grade='A'" in html_out
 
 
+def test_api_grades_reuses_kpi_summary_without_sidecar_scan() -> None:
+    class Summary:
+        def to_dict(self) -> dict[str, object]:
+            return {"grade_counts": {"A": 2, "B": 1, "F": 4}}
+
+    def fail_scan() -> dict[str, object]:
+        raise AssertionError("grades API should reuse the cached KPI summary")
+
+    response = readonly_api.handle_readonly_route(
+        "api_grades",
+        {},
+        {},
+        readonly_api.ReadOnlyApiDeps(
+            summarize_sessions=lambda: {},
+            read_manifest=lambda: {},
+            status_payload=lambda: {},
+            kpi_summary=lambda: Summary(),
+            grade_distribution_payload=fail_scan,
+            sidecar_page_payload=lambda _qs: {},
+            runtime_lifecycle_summary=lambda: {},
+            skillspector_audit_payload=lambda _qs: {},
+            effective_config_payload=lambda: {},
+            search_wiki_entities=lambda _q, _type, _limit: [],
+            wiki_entity_detail=lambda _slug, _type: None,
+            load_sidecar=lambda _slug, _type: None,
+            graph_neighborhood=lambda _slug, _hops, _limit, _type: {},
+            normalize_dashboard_entity_type=lambda raw: raw,
+        ),
+    )
+
+    assert response is not None
+    assert response.payload == {
+        "grades": {"A": 2, "B": 1, "C": 0, "D": 0, "F": 4},
+        "total": 7,
+    }
+
+
 def test_render_skills_emits_sidebar_filters(fake_claude: Path) -> None:
     _write_sidecar(fake_claude, "a", {"slug": "a", "grade": "A", "raw_score": 0.9,
                                        "subject_type": "skill"})
@@ -4239,6 +4276,7 @@ def test_render_wiki_index_wrapper_matches_extracted_orchestration(
         write_html_disk_cache=mt.cache_service.write_html_disk_cache,
         wiki_render_disk_cache_path=mt.wiki_render_disk_cache_path,
         wiki_index_entries=mt.wiki_index_entries,
+        search_wiki_entities=mt.search_wiki_entities,
         wiki_stats=mt.wiki_stats,
         load_sidecar=mt.load_sidecar,
         dashboard_entity_types=mt.DASHBOARD_ENTITY_TYPES,
@@ -4354,6 +4392,33 @@ def test_wiki_index_entries_use_dashboard_index_without_markdown_pages(
     assert mt.render_wiki_index(query="python") == html_out
 
 
+def test_render_wiki_index_query_uses_bounded_search(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    def fail_catalog_rebuild(*args: object, **kwargs: object) -> object:
+        raise AssertionError("queried catalog should use bounded search results")
+
+    monkeypatch.setattr(mt, "wiki_index_entries", fail_catalog_rebuild)
+    monkeypatch.setattr(
+        mt,
+        "search_wiki_entities",
+        lambda query, entity_type=None, limit=80: [{
+            "slug": "code-reviewer",
+            "display_slug": "code-reviewer",
+            "type": entity_type or "agent",
+            "description": f"{query} result",
+            "tags": ["review"],
+            "search_tags": ["review"],
+        }],
+    )
+
+    html_out = mt.render_wiki_index(entity_type="agent", query="review")
+
+    assert "code-reviewer" in html_out
+    assert "review result" in html_out
+    assert "href='/wiki/code-reviewer?type=agent'" in html_out
+
+
 def test_render_wiki_index_supports_type_query_and_autocomplete(fake_claude: Path) -> None:
     skills_dir = fake_claude / "skill-wiki" / "entities" / "skills"
     agents_dir = fake_claude / "skill-wiki" / "entities" / "agents"

src/tests/test_harness_add.py

@@ -230,3 +230,27 @@ def test_cli_from_json_adds_harness(
 
     assert (wiki / "entities" / "harnesses" / "text-to-cad.md").exists()
     assert "added: text-to-cad" in capsys.readouterr().out
+
+
+def test_cli_from_bom_json_adds_harness(
+    tmp_path: Path,
+    capsys: Any,
+) -> None:
+    record_path = tmp_path / "harness.json"
+    record_path.write_text(
+        "\ufeff"
+        + json.dumps(
+            {
+                "repo_url": "https://github.com/example/bom-harness",
+                "description": "Harness loaded from a Windows UTF-8 BOM JSON file.",
+                "tags": ["llm", "testing"],
+            }
+        ),
+        encoding="utf-8",
+    )
+    wiki = tmp_path / "wiki"
+
+    harness_add.main(["--from-json", str(record_path), "--wiki", str(wiki)])
+
+    assert (wiki / "entities" / "harnesses" / "bom-harness.md").exists()
+    assert "added: bom-harness" in capsys.readouterr().out

src/tests/test_harness_recommendations.py

@@ -273,6 +273,21 @@ def test_ctx_init_keeps_domain_goal_strong_with_install_requirements(
     assert "gpt-5" not in results[0]["fit_signals"]
 
 
+def test_ctx_init_rejects_narrow_domain_harness_for_generic_coding_agent(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(ctx_init, "_load_recommendation_graph", _harness_graph)
+
+    results = ctx_init.recommend_harnesses(
+        "build an openai python coding agent with filesystem and test verification",
+        top_k=5,
+        model_provider="openai",
+        model="openai/gpt-4.1",
+    )
+
+    assert "text-to-cad" not in [row["name"] for row in results]
+
+
 def test_ctx_init_recommends_harness_from_wiki_frontmatter_only(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:

src/tests/test_mcp_add.py

@@ -832,3 +832,120 @@ class TestAddMcpFromFixtures:
         record = McpRecord.from_dict(data)
         result = patched_mcp_add.add_mcp(record=record, wiki_path=wiki_dir)
         assert result["is_new_page"] is True
+
+
+class TestAddMcpCliInput:
+    def test_from_json_accepts_utf8_bom(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        tmp_path: Path,
+    ) -> None:
+        import mcp_add  # noqa: PLC0415
+
+        record_path = tmp_path / "mcp.json"
+        record_path.write_text(
+            "\ufeff"
+            + json.dumps(
+                {
+                    "name": "bom-json-mcp",
+                    "description": "MCP loaded from a Windows UTF-8 BOM JSON file.",
+                    "sources": ["test"],
+                    "github_url": "https://github.com/example/bom-json-mcp",
+                    "tags": ["testing"],
+                }
+            ),
+            encoding="utf-8",
+        )
+        wiki = tmp_path / "wiki"
+        monkeypatch.setattr(sys, "argv", [
+            "mcp_add.py",
+            "--from-json", str(record_path),
+            "--wiki", str(wiki),
+        ])
+        monkeypatch.setattr("mcp_add.check_intake", _fake_allow)
+        monkeypatch.setattr("mcp_add.record_embedding", _fake_record_embedding)
+        monkeypatch.setattr("mcp_add.update_index", lambda *a, **k: None)
+        monkeypatch.setattr("mcp_add.append_log", lambda *a, **k: None)
+
+        mcp_add.main()
+
+        assert (
+            wiki / "entities" / "mcp-servers" / "b" / "bom-json-mcp.md"
+        ).exists()
+
+    def test_from_stdin_accepts_utf8_bom(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        tmp_path: Path,
+    ) -> None:
+        import io
+        import mcp_add  # noqa: PLC0415
+
+        wiki = tmp_path / "wiki"
+        monkeypatch.setattr(sys, "argv", [
+            "mcp_add.py",
+            "--from-stdin",
+            "--wiki", str(wiki),
+        ])
+        monkeypatch.setattr(
+            sys,
+            "stdin",
+            io.StringIO(
+                "\ufeff"
+                + json.dumps(
+                    {
+                        "name": "bom-stdin-mcp",
+                        "description": (
+                            "MCP loaded from a Windows UTF-8 BOM stdin stream."
+                        ),
+                        "sources": ["test"],
+                        "github_url": "https://github.com/example/bom-stdin-mcp",
+                        "tags": ["testing"],
+                    }
+                )
+                + "\n"
+            ),
+        )
+        monkeypatch.setattr("mcp_add.check_intake", _fake_allow)
+        monkeypatch.setattr("mcp_add.record_embedding", _fake_record_embedding)
+        monkeypatch.setattr("mcp_add.update_index", lambda *a, **k: None)
+        monkeypatch.setattr("mcp_add.append_log", lambda *a, **k: None)
+
+        mcp_add.main()
+
+        assert (
+            wiki / "entities" / "mcp-servers" / "b" / "bom-stdin-mcp.md"
+        ).exists()
+
+    def test_rejected_batch_exits_nonzero(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        tmp_path: Path,
+    ) -> None:
+        import mcp_add  # noqa: PLC0415
+
+        record_path = tmp_path / "mcp.json"
+        record_path.write_text(
+            json.dumps(
+                {
+                    "name": "rejected-cli-mcp",
+                    "description": "Rejected MCP candidate for exit-code regression.",
+                    "sources": ["test"],
+                    "github_url": "https://github.com/example/rejected-cli-mcp",
+                    "tags": ["testing"],
+                }
+            ),
+            encoding="utf-8",
+        )
+        monkeypatch.setattr(sys, "argv", [
+            "mcp_add.py",
+            "--from-json", str(record_path),
+            "--wiki", str(tmp_path / "wiki"),
+        ])
+        monkeypatch.setattr("mcp_add.check_intake", _fake_reject)
+        monkeypatch.setattr("mcp_add.record_embedding", _fake_record_embedding)
+
+        with pytest.raises(SystemExit) as exc:
+            mcp_add.main()
+
+        assert exc.value.code == 1

src/tests/test_scan_repo.py

@@ -303,6 +303,19 @@ dev = ["black"]
         assert "pytest-cov" in deps
         assert "black" in deps
 
+    def test_can_ignore_optional_deps_for_primary_stack(self, tmp_path: Path) -> None:
+        p = _write(tmp_path / "pyproject.toml", """
+[project]
+dependencies = ["core-dep"]
+
+[project.optional-dependencies]
+embeddings = ["torch>=2"]
+""")
+
+        deps = sr.read_toml_deps(str(p), include_optional=False)
+
+        assert deps == ["core-dep"]
+
     def test_extracts_poetry_deps(self, tmp_path: Path) -> None:
         p = _write(tmp_path / "pyproject.toml", """
 [tool.poetry]
@@ -671,6 +684,20 @@ class TestDetectStackProjectType:
         profile = sr.detect_stack(str(tmp_path), signals)
         assert profile["project_type"] == "ml-project"
 
+    def test_optional_torch_does_not_make_project_ml(self, tmp_path: Path) -> None:
+        py = _write(tmp_path / "pyproject.toml", """
+[project]
+dependencies = ["networkx"]
+
+[project.optional-dependencies]
+embeddings = ["torch>=2"]
+""")
+        signals = _make_signals(config_files=[str(py)])
+        profile = sr.detect_stack(str(tmp_path), signals)
+
+        assert any(f["name"] == "pytorch" for f in profile["frameworks"])
+        assert profile["project_type"] == "unknown"
+
     def test_ai_agent_when_langchain_present(self, tmp_path: Path) -> None:
         py = _write(tmp_path / "requirements.txt", "langchain\n")
         signals = _make_signals(config_files=[str(py)])

src/tests/test_skill_add.py

@@ -633,6 +633,27 @@ class TestAddSkill:
         assert result["converted"] is False
         assert result["is_new_page"] is True
 
+    def test_bom_prefixed_skill_frontmatter_is_accepted(self, tmp_path, monkeypatch):
+        wiki = self._setup_wiki(tmp_path)
+        skills_dir = tmp_path / "skills"
+        self._setup_intake_allow()
+        _fake_batch_convert.convert_skill.return_value = {"status": "error"}
+        monkeypatch.setattr(_sa, "update_index", MagicMock())
+        monkeypatch.setattr(_sa, "append_log", MagicMock())
+
+        source = tmp_path / "SKILL.md"
+        source.write_text("\ufeff" + self._skill_text(), encoding="utf-8")
+
+        result = add_skill(
+            source_path=source,
+            name="myskill",
+            wiki_path=wiki,
+            skills_dir=skills_dir,
+        )
+
+        assert result["name"] == "myskill"
+        assert result["is_new_page"] is True
+
     def test_required_security_scan_blocks_before_install(
         self, tmp_path, monkeypatch
     ):