fix(F-14b): add pickle/model integrity checking mechanism

Adds backend/core/model_integrity.py: a SHA-256 checksum gate in front
of pickle.load() calls, checked against data/reference/known_hashes.json
-- pinned in this repo, not fetched from the same untrusted channel the
model files themselves come from (main.py's startup step downloads them
from a Hugging Face Space named by the attacker-influenceable SPACE_ID
env var, with no verification before unpickling).

Honest about current scope: this session's uploaded snapshot only has
Git-LFS pointer stubs for data/reference/*.pkl (132-byte text files),
so there are no real files to hash yet -- known_hashes.json starts
empty and verify_integrity() logs a loud once-per-file warning rather
than hard-failing (refusing to start with zero pinned hashes would
break every deployment today). scripts/generate_model_hashes.py lets
you populate real hashes once you have real model files; from that
point on, any mismatch fail-closes automatically with no further code
changes. This commit adds the mechanism only -- wiring it into the
actual pickle.load() call sites is the next commits.

backend/core/model_integrity.py

@@ -0,0 +1,104 @@
+"""
+Model/reference-file integrity checking (F-14).
+
+pickle.load() on a file whose provenance is "whatever a remote channel
+currently contains" is a real code-execution surface if that channel is
+ever compromised -- and main.py's startup step downloads
+own_embedding_model.pt, clip_database.pkl, own_centroids.pkl, and
+ensemble_xgb.pkl from a Hugging Face Space named by the SPACE_ID
+environment variable, with no verification before those .pkl files are
+unpickled. SPACE_ID is attacker-influenceable in a misconfigured
+deployment.
+
+This module provides a SHA-256 checksum gate in front of every
+pickle.load() call site, checked against a value pinned in THIS repo
+(KNOWN_HASHES below / data/reference/known_hashes.json) -- not fetched
+from the same untrusted channel as the files themselves, which is the
+whole point.
+
+Current status (see REMAINING_FIXES.md): KNOWN_HASHES starts empty.
+This session's uploaded snapshot only contains Git-LFS pointer stubs
+for data/reference/*.pkl (132-byte text files, not real model data), so
+there is nothing real to hash yet. Once real model files exist in your
+environment, run:
+
+    python scripts/generate_model_hashes.py
+
+...to populate data/reference/known_hashes.json with real SHA-256
+values, then commit that file. Until it's populated, verify_integrity()
+logs a loud warning (once per file) instead of hard-failing -- refusing
+to start the app with no pinned hashes at all would break every
+deployment today, which is worse than the gap it would close. Once
+real hashes are committed, this becomes fail-closed automatically with
+no further code changes: any mismatch raises ModelIntegrityError.
+"""
+import hashlib
+import json
+from pathlib import Path
+from typing import Optional
+
+from backend.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+_HASHES_PATH = Path(__file__).parent.parent.parent / "data" / "reference" / "known_hashes.json"
+_warned_files: set = set()
+
+
+class ModelIntegrityError(Exception):
+    """Raised when a reference/model file's SHA-256 does not match the
+    value pinned in data/reference/known_hashes.json."""
+
+
+def _load_known_hashes() -> dict:
+    if not _HASHES_PATH.exists():
+        return {}
+    try:
+        with open(_HASHES_PATH, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception as e:
+        logger.warning(f"Could not parse {_HASHES_PATH}: {e}")
+        return {}
+
+
+def _sha256_of(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def verify_integrity(path: Path, known_hashes: Optional[dict] = None) -> None:
+    """Verify path's SHA-256 against the pinned value for its filename.
+
+    - No pinned hash on file for this filename -> log a warning once,
+      then allow the load (see module docstring for why this isn't a
+      hard failure yet).
+    - Pinned hash present and it matches -> silent success.
+    - Pinned hash present and it does NOT match -> raises
+      ModelIntegrityError. Callers should let this propagate (do not
+      swallow it into the generic "file missing/corrupt, fall back to
+      neutral" except-branches these call sites already have) since a
+      hash mismatch is a materially different, higher-severity signal
+      than "file absent" or "file is an LFS pointer stub".
+    """
+    known = known_hashes if known_hashes is not None else _load_known_hashes()
+    expected = known.get(path.name)
+    if not expected:
+        if path.name not in _warned_files:
+            _warned_files.add(path.name)
+            logger.warning(
+                "No pinned SHA-256 for %s in %s -- integrity NOT verified "
+                "before loading. Run scripts/generate_model_hashes.py once "
+                "you have real model files, then commit the result.",
+                path.name, _HASHES_PATH,
+            )
+        return
+    actual = _sha256_of(path)
+    if actual != expected:
+        raise ModelIntegrityError(
+            f"{path.name}: SHA-256 mismatch. Expected {expected}, got {actual}. "
+            f"Refusing to load -- this file's contents do not match what is "
+            f"pinned in {_HASHES_PATH}."
+        )

data/reference/known_hashes.json

@@ -0,0 +1 @@
+{}

scripts/generate_model_hashes.py

@@ -0,0 +1,88 @@
+"""
+Generate data/reference/known_hashes.json from the real reference/model
+files currently on disk (F-14).
+
+Run this once you have real (non-Git-LFS-stub) copies of:
+    data/reference/own_embedding_model.pt
+    data/reference/clip_database.pkl
+    data/reference/own_centroids.pkl
+    data/reference/ensemble_xgb.pkl
+
+...then commit the resulting known_hashes.json. From that point on,
+backend/core/model_integrity.py's verify_integrity() fail-closes on any
+mismatch instead of just logging a warning -- no code changes needed,
+it reads this file automatically.
+
+Usage:
+    python scripts/generate_model_hashes.py
+"""
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).parent.parent
+REF_DIR = REPO_ROOT / "data" / "reference"
+HASHES_PATH = REF_DIR / "known_hashes.json"
+
+# Keep in sync with backend/main.py's _model_files list.
+TRACKED_FILES = [
+    "own_embedding_model.pt",
+    "clip_database.pkl",
+    "own_centroids.pkl",
+    "ensemble_xgb.pkl",
+]
+
+_LFS_POINTER_PREFIX = b"version https://git-lfs.github.com/spec"
+
+
+def sha256_of(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def main() -> int:
+    hashes = {}
+    if HASHES_PATH.exists():
+        try:
+            hashes = json.loads(HASHES_PATH.read_text(encoding="utf-8"))
+        except Exception:
+            hashes = {}
+
+    skipped = []
+    updated = []
+    for fname in TRACKED_FILES:
+        path = REF_DIR / fname
+        if not path.exists():
+            skipped.append((fname, "not found"))
+            continue
+        with open(path, "rb") as f:
+            head = f.read(len(_LFS_POINTER_PREFIX))
+        if head == _LFS_POINTER_PREFIX:
+            skipped.append((fname, "still a Git-LFS pointer stub -- run `git lfs pull` first"))
+            continue
+        hashes[fname] = sha256_of(path)
+        updated.append(fname)
+
+    HASHES_PATH.parent.mkdir(parents=True, exist_ok=True)
+    HASHES_PATH.write_text(json.dumps(hashes, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+    for fname in updated:
+        print(f"  hashed: {fname}")
+    for fname, reason in skipped:
+        print(f"  skipped: {fname} ({reason})")
+    print(f"\nWrote {HASHES_PATH} with {len(hashes)} entries.")
+    if skipped:
+        print(
+            f"\n{len(skipped)} file(s) were skipped -- those files will still "
+            f"load with only a warning (no integrity check) until you re-run "
+            f"this script after obtaining the real files."
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())