feat: implement anomaly detection to filter suspicious word relations

Dockerfile

@@ -39,14 +39,25 @@ COPY --chown=appuser *.py ./
 # Copy pre-built frontend
 COPY --chown=appuser --from=frontend-build /app/frontend/dist ./frontend/dist
 
-# Data directories (HF cache, engine state, trained models)
-RUN mkdir -p /data/huggingface /data/engine_state /data/w2v_state /data/trained_model \
+# Data directories (HF cache, gensim/GloVe cache, engine state, trained models)
+RUN mkdir -p /data/huggingface /data/gensim-data /data/engine_state /data/w2v_state /data/trained_model \
     && chown -R appuser:appuser /app /data
 
 ENV HF_HOME=/data/huggingface
 ENV TRANSFORMERS_CACHE=/data/huggingface
 ENV ENGINE_STATE_DIR=/data/engine_state
 ENV W2V_STATE_DIR=/data/w2v_state
+# GloVe background model cache for anomaly detection — keep it under /data so it
+# sits on persistent storage and isn't re-downloaded (~128MB) on every cold start.
+ENV GENSIM_DATA_DIR=/data/gensim-data
+# Override the background model (e.g. glove-wiki-gigaword-50) without code changes.
+ENV BACKGROUND_MODEL=glove-wiki-gigaword-100
+
+# Bake the background model into the image so anomaly detection works offline,
+# with no runtime download or cold-start wait (adds ~128MB to the image).
+# return_path=True just ensures the download/cache without loading it into RAM.
+RUN uv run python -c "import os, gensim.downloader as d; d.load(os.environ['BACKGROUND_MODEL'], return_path=True)" \
+    && chown -R appuser:appuser /data/gensim-data
 
 # Switch to non-root user
 USER appuser

anomaly.py

@@ -0,0 +1,401 @@
+"""
+Anomalous-relation detection — finding "code word" candidates.
+
+A code word is a *common English word used uncommonly in this corpus*. We make
+that precise by contrasting two distributional word spaces:
+
+  - the corpus Word2Vec (how words associate IN these documents), and
+  - a pretrained general-English model (how they associate NORMALLY).
+
+The whole design avoids two traps:
+  1. Raw cosines from different embedding spaces are NOT comparable, so we never
+     subtract a corpus cosine from a background cosine. We compare neighbour
+     *sets* (scale-free) and, when we need per-neighbour scores, we standardise
+     within each space (z-scores relative to the same anchor word) before
+     combining — that subtraction *is* legitimate.
+  2. "Low similarity" is the default for almost all word pairs and is not a
+     signal. The signal is *surprise*: strong here, weak normally.
+
+Three stages:
+  A. sweep         — rank which words behave most differently here vs. normally
+                     (neighbour-set divergence, z-scored across the vocabulary).
+  B. relations     — for one flagged word, the specific neighbours that are
+                     strong in-corpus but weak/absent in general English.
+  C. incongruence  — uses the transformer to find the specific occurrences
+                     (chunks/docs) where a keyword is used unlike its norm.
+
+Stages A/B need the corpus Word2Vec + background model. Stage C needs the
+transformer engine (contextual embeddings), which is the only one of the three
+models that can judge a single *occurrence* in context.
+"""
+
+import logging
+import re
+import threading
+from typing import Optional
+
+import numpy as np
+
+from contextual_similarity import ContextualSimilarityEngine
+from word2vec_baseline import Word2VecEngine
+from background_model import BackgroundModel
+
+logger = logging.getLogger(__name__)
+
+# Reuse the engine's stopword list so the three tools agree on what to ignore.
+_STOPWORDS = ContextualSimilarityEngine._STOPWORDS
+_ALPHA = re.compile(r"^[a-z]+$")
+
+# Capitalised spans (names/orgs) and dates, for the Stage-C investigation view.
+_ENTITY_RE = re.compile(r"\b[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+){0,3}\b")
+_DATE_RE = re.compile(
+    r"\b(?:\d{1,2}[/-]\d{1,2}[/-]\d{2,4}"
+    r"|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2}"
+    r"|\d{4})\b"
+)
+
+
+def _is_candidate_word(word: str) -> bool:
+    """A gating word must be a plain lowercase English word, not a stopword/short token."""
+    return len(word) >= 3 and bool(_ALPHA.match(word)) and word not in _STOPWORDS
+
+
+def _normalize_rows(mat: np.ndarray) -> np.ndarray:
+    norms = np.linalg.norm(mat, axis=1, keepdims=True)
+    norms[norms == 0] = 1e-12
+    return (mat / norms).astype(np.float32)
+
+
+class SharedSpace:
+    """
+    The shared vocabulary of words present in BOTH the corpus Word2Vec and the
+    background model, with their normalised vectors in each space.
+
+    Restricting to shared words keeps neighbour-set overlap fair (both spaces
+    rank the *same* candidate set) and excludes names/jargon (absent from the
+    background) — which are domain vocabulary, not code words.
+    """
+
+    def __init__(self, words: list[str], corpus_vecs: np.ndarray,
+                 bg_vecs: np.ndarray, corpus_counts: np.ndarray):
+        self.words = words
+        self.index = {w: i for i, w in enumerate(words)}
+        self.C = _normalize_rows(corpus_vecs)   # (n, d_corpus)
+        self.B = _normalize_rows(bg_vecs)        # (n, d_bg)
+        self.counts = corpus_counts              # corpus frequency per word
+
+    def __len__(self) -> int:
+        return len(self.words)
+
+
+def build_shared_space(
+    w2v: Word2VecEngine,
+    background: BackgroundModel,
+    min_count: int = 5,
+    max_vocab: int = 3000,
+) -> SharedSpace:
+    """
+    Build the shared-vocabulary space.
+
+    Args:
+        min_count: ignore corpus words rarer than this (their vectors are noise).
+        max_vocab: cap to the N most frequent shared words for tractability.
+    """
+    wv = w2v.model.wv
+    # Collect candidate words: frequent enough in the corpus, common in English.
+    candidates: list[tuple[str, int]] = []
+    for word in wv.index_to_key:
+        if not _is_candidate_word(word):
+            continue
+        count = wv.get_vecattr(word, "count")
+        if count < min_count:
+            continue
+        if not background.has(word):
+            continue
+        candidates.append((word, count))
+
+    # Keep the most frequent ones (their corpus vectors are the most reliable).
+    candidates.sort(key=lambda x: -x[1])
+    if len(candidates) > max_vocab:
+        logger.info("Shared vocab capped: %d candidates -> top %d by corpus frequency",
+                    len(candidates), max_vocab)
+        candidates = candidates[:max_vocab]
+
+    words = [w for w, _ in candidates]
+    counts = np.array([c for _, c in candidates], dtype=np.int64)
+    corpus_vecs = np.array([wv[w] for w in words], dtype=np.float32)
+    bg_vecs = np.array([background.kv[w] for w in words], dtype=np.float32)
+    logger.info("Shared space built: %d words (corpus∩background, count>=%d)", len(words), min_count)
+    return SharedSpace(words, corpus_vecs, bg_vecs, counts)
+
+
+# Module-level cache so the (expensive) shared space is reused across requests
+# until the underlying corpus Word2Vec changes.
+_cache_lock = threading.Lock()
+_cached: dict = {"key": None, "space": None}
+
+
+def get_shared_space(w2v: Word2VecEngine, background: BackgroundModel,
+                     min_count: int, max_vocab: int) -> SharedSpace:
+    key = (id(w2v.model), len(w2v.model.wv), background.model_name, min_count, max_vocab)
+    with _cache_lock:
+        if _cached["key"] == key and _cached["space"] is not None:
+            return _cached["space"]
+        space = build_shared_space(w2v, background, min_count, max_vocab)
+        _cached["key"] = key
+        _cached["space"] = space
+        return space
+
+
+def _topk_neighbours(sims: np.ndarray, self_idx: int, k: int) -> list[int]:
+    """Indices of the top-k most similar rows, excluding the anchor itself."""
+    sims = sims.copy()
+    sims[self_idx] = -np.inf
+    if k >= len(sims):
+        order = np.argsort(sims)[::-1]
+    else:
+        part = np.argpartition(sims, -k)[-k:]
+        order = part[np.argsort(sims[part])[::-1]]
+    return [int(i) for i in order]
+
+
+# ------------------------------------------------------------------ #
+#  Stage A — corpus-wide sweep: which words behave most anomalously?
+# ------------------------------------------------------------------ #
+
+def sweep_anomalous_words(
+    w2v: Word2VecEngine,
+    background: BackgroundModel,
+    min_count: int = 5,
+    max_vocab: int = 3000,
+    neighbours: int = 25,
+    top_n: int = 30,
+    preview: int = 6,
+) -> dict:
+    """
+    Rank words by how differently they associate here vs. in general English.
+
+    For each shared word W we compute its top-`neighbours` in each space and
+    score `shift = 1 - overlap@k` (Jaccard-style overlap of the two neighbour
+    sets). Overlap is scale-free — no cross-space cosine arithmetic. We then
+    z-score `shift` across the whole vocabulary so flagging adapts to the corpus
+    instead of using a magic threshold.
+    """
+    space = get_shared_space(w2v, background, min_count, max_vocab)
+    n = len(space)
+    if n < neighbours + 2:
+        return {"ready": True, "vocab_size": n, "results": [],
+                "note": "Shared vocabulary too small to compute neighbourhoods."}
+
+    C, B = space.C, space.B
+    k = min(neighbours, n - 1)
+
+    shifts = np.zeros(n, dtype=np.float32)
+    corpus_nbrs: list[list[int]] = [None] * n
+    bg_nbrs: list[list[int]] = [None] * n
+    for i in range(n):
+        c_sims = C @ C[i]
+        b_sims = B @ B[i]
+        cn = _topk_neighbours(c_sims, i, k)
+        bn = _topk_neighbours(b_sims, i, k)
+        corpus_nbrs[i] = cn
+        bg_nbrs[i] = bn
+        overlap = len(set(cn) & set(bn)) / k
+        shifts[i] = 1.0 - overlap
+
+    mean, std = float(shifts.mean()), float(shifts.std())
+    std = std if std > 1e-9 else 1e-9
+
+    order = np.argsort(shifts)[::-1][:top_n]
+    results = []
+    for i in order:
+        i = int(i)
+        bg_set = set(bg_nbrs[i])
+        # Corpus neighbours that are NOT normal neighbours = the surprising ties.
+        surprising = [space.words[j] for j in corpus_nbrs[i] if j not in bg_set][:preview]
+        normal = [space.words[j] for j in bg_nbrs[i]][:preview]
+        results.append({
+            "word": space.words[i],
+            "corpus_frequency": int(space.counts[i]),
+            "shift": round(float(shifts[i]), 4),
+            "z_score": round((float(shifts[i]) - mean) / std, 3),
+            "surprising_neighbors": surprising,   # strong here, absent normally
+            "normal_neighbors": normal,           # what's normal in English
+        })
+
+    return {
+        "ready": True,
+        "vocab_size": n,
+        "neighbours": k,
+        "shift_mean": round(mean, 4),
+        "shift_std": round(std, 4),
+        "results": results,
+    }
+
+
+# ------------------------------------------------------------------ #
+#  Stage B — per-relation surprise for a single flagged word.
+# ------------------------------------------------------------------ #
+
+def relation_surprise(
+    word: str,
+    w2v: Word2VecEngine,
+    background: BackgroundModel,
+    min_count: int = 5,
+    max_vocab: int = 3000,
+    top_k: int = 15,
+) -> dict:
+    """
+    For a single word, the neighbours that are strong in-corpus but weak/absent
+    in general English — the concrete "pizza -> fitness" rows.
+
+    We standardise each space's similarities to the anchor word into z-scores
+    (dimensionless, relative to the same anchor), then `surprise = corpus_z -
+    background_z`. Subtracting two within-space z-scores IS valid, unlike
+    subtracting raw cross-space cosines.
+    """
+    word = word.lower().strip()
+    space = get_shared_space(w2v, background, min_count, max_vocab)
+
+    if word not in space.index:
+        in_corpus = word in w2v.model.wv
+        in_bg = background.has(word)
+        if in_corpus and not in_bg:
+            reason = ("not a common English word (absent from the background model), "
+                      "so it's treated as domain vocabulary — a name/jargon, not a code-word candidate")
+        elif not in_corpus:
+            reason = "not in the corpus vocabulary (or too rare)"
+        else:
+            reason = "below the minimum corpus frequency"
+        return {"word": word, "ready": True, "found": False, "reason": reason, "relations": []}
+
+    i = space.index[word]
+    c_sims = space.C @ space.C[i]
+    b_sims = space.B @ space.B[i]
+
+    def zscore(sims: np.ndarray) -> np.ndarray:
+        m, s = sims.mean(), sims.std()
+        return (sims - m) / (s if s > 1e-9 else 1e-9)
+
+    c_z, b_z = zscore(c_sims), zscore(b_sims)
+    surprise = c_z - b_z
+    surprise[i] = -np.inf  # exclude self
+
+    order = np.argsort(surprise)[::-1][:top_k]
+    relations = [{
+        "neighbor": space.words[j],
+        "corpus_sim": round(float(c_sims[j]), 4),       # raw cosines: display only
+        "background_sim": round(float(b_sims[j]), 4),
+        "corpus_z": round(float(c_z[j]), 3),
+        "background_z": round(float(b_z[j]), 3),
+        "surprise": round(float(surprise[j]), 3),
+    } for j in (int(x) for x in order)]
+
+    # Contrast: what this word's NORMAL neighbours are, in general English.
+    normal_order = _topk_neighbours(b_sims, i, min(top_k, len(space) - 1))
+    normal = [{"neighbor": space.words[j], "background_sim": round(float(b_sims[j]), 4)}
+              for j in normal_order]
+
+    return {
+        "word": word,
+        "ready": True,
+        "found": True,
+        "corpus_frequency": int(space.counts[i]),
+        "relations": relations,
+        "normal_neighbors": normal,
+    }
+
+
+# ------------------------------------------------------------------ #
+#  Stage C — contextual incongruence (the "zoom in"), via the transformer.
+# ------------------------------------------------------------------ #
+
+def _extract_entities(text: str, limit: int = 8) -> list[str]:
+    """Capitalised name-like spans and dates, for the investigation view."""
+    found: list[str] = []
+    seen: set[str] = set()
+    for m in _ENTITY_RE.finditer(text):
+        token = m.group().strip()
+        # Skip single sentence-initial capitalised stopwords ("The", "And", ...)
+        if " " not in token and token.lower() in _STOPWORDS:
+            continue
+        if token.lower() not in seen:
+            seen.add(token.lower())
+            found.append(token)
+    for m in _DATE_RE.finditer(text):
+        d = m.group()
+        if d.lower() not in seen:
+            seen.add(d.lower())
+            found.append(d)
+    return found[:limit]
+
+
+def contextual_incongruence(
+    engine: ContextualSimilarityEngine,
+    keyword: str,
+    canonical_meaning: Optional[str] = None,
+    top_k: int = 10,
+) -> dict:
+    """
+    Find the occurrences where `keyword` is used most unlike its norm.
+
+    Reference meaning is either:
+      - `canonical_meaning` (a gloss you supply, e.g. "pizza, an Italian food"), or
+      - the centroid of all the keyword's occurrence embeddings (its typical
+        usage in THIS corpus) when no gloss is given.
+
+    Per occurrence: incongruence = 1 - cos(chunk_embedding, reference). The
+    highest-incongruence chunks are the candidate coded usages — returned with
+    doc/snippet and co-occurring entities so you can read them directly.
+    """
+    engine._ensure_index()
+    contexts = engine.find_keyword_contexts(keyword)
+    if not contexts:
+        return {"keyword": keyword, "total_occurrences": 0, "occurrences": []}
+
+    chunk_indices = [engine.chunks.index(ctx.chunk) for ctx in contexts]
+    embeds = engine.embeddings[chunk_indices]  # rows are L2-normalised (build_index)
+
+    if canonical_meaning and canonical_meaning.strip():
+        ref = engine.model.encode([canonical_meaning], normalize_embeddings=True,
+                                  convert_to_numpy=True)[0].astype(np.float32)
+        ref_label = canonical_meaning.strip()
+        ref_kind = "gloss"
+    else:
+        centroid = embeds.mean(axis=0)
+        norm = np.linalg.norm(centroid)
+        ref = (centroid / (norm if norm > 0 else 1e-12)).astype(np.float32)
+        ref_label = "corpus-typical usage (centroid of all occurrences)"
+        ref_kind = "centroid"
+
+    sims = embeds @ ref               # cosine (both sides unit-norm)
+    incong = 1.0 - sims
+    median_incong = float(np.median(incong))
+
+    order = np.argsort(incong)[::-1][:top_k]
+    occurrences = []
+    for j in (int(x) for x in order):
+        ctx = contexts[j]
+        text = ctx.chunk.text
+        if ctx.highlight_positions:
+            start, end = ctx.highlight_positions[0]
+            s, e = max(0, start - 120), min(len(text), end + 120)
+            snippet = ("..." if s > 0 else "") + text[s:e].strip() + ("..." if e < len(text) else "")
+        else:
+            snippet = text[:240]
+        occurrences.append({
+            "doc_id": ctx.chunk.doc_id,
+            "chunk_index": ctx.chunk.chunk_index,
+            "incongruence": round(float(incong[j]), 4),
+            "snippet": snippet,
+            "entities": _extract_entities(text),
+        })
+
+    return {
+        "keyword": keyword,
+        "total_occurrences": len(contexts),
+        "reference": ref_label,
+        "reference_kind": ref_kind,
+        "median_incongruence": round(median_incong, 4),
+        "occurrences": occurrences,
+    }

background_model.py

@@ -0,0 +1,93 @@
+"""
+Background language model — pretrained general-English word vectors.
+
+This is the "what's normal in English" reference used by anomaly detection.
+It is deliberately a *distributional* word-embedding model (GloVe), the same
+*kind* of object as the corpus Word2Vec: comparing a word's neighbourhood in
+the corpus against its neighbourhood here is only meaningful when both spaces
+are word-co-occurrence embeddings. A sentence-transformer (or an OpenAI text
+embedding) is the wrong type for this role — it models text similarity, not
+word co-occurrence, and lives in an unrelated vector space.
+
+The model is loaded lazily on first use (gensim downloads & caches it to
+~/gensim-data, or $GENSIM_DATA_DIR). A failed download degrades gracefully:
+callers see `ready == False` instead of a crash.
+"""
+
+import logging
+import os
+import threading
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# glove-wiki-gigaword-100: ~128MB, 400k lowercase words, dim=100.
+# Override with BACKGROUND_MODEL env var (e.g. glove-wiki-gigaword-50).
+DEFAULT_MODEL = os.environ.get("BACKGROUND_MODEL", "glove-wiki-gigaword-100")
+
+
+class BackgroundModel:
+    """Lazily-loaded pretrained general-English word vectors (gensim KeyedVectors)."""
+
+    def __init__(self, model_name: str = DEFAULT_MODEL):
+        self.model_name = model_name
+        self._kv = None  # gensim KeyedVectors once loaded
+        self._lock = threading.Lock()
+        self._load_failed = False
+
+    @property
+    def ready(self) -> bool:
+        return self._kv is not None
+
+    @property
+    def kv(self):
+        """The underlying gensim KeyedVectors, or None if not loaded."""
+        return self._kv
+
+    def load(self) -> bool:
+        """
+        Load the model if needed. Returns True on success, False on failure.
+
+        Thread-safe and idempotent. The first call may download ~128MB; later
+        calls (and restarts, if the cache survives) are instant. On failure the
+        model is marked failed so we don't retry a doomed download every request.
+        """
+        if self._kv is not None:
+            return True
+        with self._lock:
+            if self._kv is not None:
+                return True
+            if self._load_failed:
+                return False
+            try:
+                import gensim.downloader as gd
+                logger.info("Loading background model '%s' (first use may download ~100MB)...",
+                            self.model_name)
+                self._kv = gd.load(self.model_name)
+                logger.info("Background model ready: %d words, dim=%d",
+                            len(self._kv), self._kv.vector_size)
+                return True
+            except Exception:
+                logger.exception("Failed to load background model '%s' — "
+                                 "anomaly detection will be unavailable.", self.model_name)
+                self._load_failed = True
+                return False
+
+    def has(self, word: str) -> bool:
+        """True if the word exists in the background vocabulary (i.e. common English)."""
+        return self._kv is not None and word in self._kv
+
+    def similarity(self, a: str, b: str) -> Optional[float]:
+        """Cosine similarity between two words in general English, or None if either is OOV."""
+        if self._kv is None or a not in self._kv or b not in self._kv:
+            return None
+        return float(self._kv.similarity(a, b))
+
+    def status(self) -> dict:
+        return {
+            "model_name": self.model_name,
+            "ready": self.ready,
+            "load_failed": self._load_failed,
+            "vocab_size": len(self._kv) if self._kv is not None else 0,
+            "vector_size": int(self._kv.vector_size) if self._kv is not None else 0,
+        }

docker-compose.yml

@@ -6,8 +6,11 @@ services:
     volumes:
       # Persist HuggingFace model cache between restarts
       - hf-cache:/data/huggingface
-      # Persist engine state and trained models
+      # Persist the GloVe background model (anomaly detection) so it's not re-downloaded
+      - gensim-cache:/data/gensim-data
+      # Persist engine state, Word2Vec state, and trained models
       - engine-state:/data/engine_state
+      - w2v-state:/data/w2v_state
       - ./trained_model:/data/trained_model
     environment:
       - HOST=0.0.0.0
@@ -15,4 +18,6 @@ services:
 
 volumes:
   hf-cache:
+  gensim-cache:
   engine-state:
+  w2v-state:

frontend/src/App.tsx

@@ -5,25 +5,21 @@ import TrainingPanel from "./components/TrainingPanel";
 import EngineSetup from "./components/EngineSetup";
 import SemanticSearch from "./components/SemanticSearch";
 import TextCompare from "./components/TextCompare";
-import KeywordAnalysis from "./components/KeywordAnalysis";
-import KeywordMatcher from "./components/KeywordMatcher";
-import BatchAnalysis from "./components/BatchAnalysis";
 import SimilarWords from "./components/SimilarWords";
 import ContextAnalysis from "./components/ContextAnalysis";
+import AnomalyPanel from "./components/AnomalyPanel";
 import Word2VecPanel from "./components/Word2VecPanel";
-import Word2VecTools from "./components/Word2VecTools";
 import DatasetPanel from "./components/DatasetPanel";
 import MetricCard from "./components/MetricCard";
 import "./styles.css";
 
-type NavGroup = "data" | "training" | "analysis";
+type NavGroup = "data" | "training";
 type TrainingTab = "model" | "w2v";
-type AnalysisTab = "context" | "words" | "search" | "compare" | "keyword" | "match" | "batch";
+type AnalysisTab = "context" | "anomalies";
 
-const STEPS: { id: NavGroup; label: string; needsIndex?: boolean }[] = [
+const STEPS: { id: NavGroup; label: string }[] = [
   { id: "data", label: "Data & Setup" },
   { id: "training", label: "Training" },
-  { id: "analysis", label: "Analysis", needsIndex: true },
 ];
 
 const TRAINING_TABS: { id: TrainingTab; label: string }[] = [
@@ -33,25 +29,18 @@ const TRAINING_TABS: { id: TrainingTab; label: string }[] = [
 
 const ANALYSIS_TABS: { id: AnalysisTab; label: string }[] = [
   { id: "context", label: "Context" },
-  { id: "words", label: "Similar Words" },
-  { id: "search", label: "Search" },
-  { id: "compare", label: "Compare" },
-  { id: "keyword", label: "Keywords" },
-  { id: "match", label: "Matcher" },
-  { id: "batch", label: "Batch" },
+  { id: "anomalies", label: "Anomalies" },
 ];
 
 export default function App() {
   const [group, setGroup] = useState<NavGroup>("data");
-  const [trainingTab, setTrainingTab] = useState<TrainingTab>("model");
+  const [trainingTab, setTrainingTab] = useState<TrainingTab>("w2v");
   const [analysisTab, setAnalysisTab] = useState<AnalysisTab>("context");
   const [stats, setStats] = useState<CorpusStats | null>(null);
   const [showManualSetup, setShowManualSetup] = useState(false);
   const [serverError, setServerError] = useState<string | null>(null);
   const [w2vReady, setW2vReady] = useState(false);
   const [w2vInfo, setW2vInfo] = useState<{ vocab_size: number; sentences: number; vector_size: number } | null>(null);
-  const [resetLoading, setResetLoading] = useState(false);
-  const ready = stats !== null && stats.index_built;
 
   useEffect(() => {
     checkConnection().then((err) => {
@@ -62,6 +51,7 @@ export default function App() {
           if (res.ready) {
             setW2vReady(true);
             setW2vInfo({ vocab_size: res.vocab_size!, sentences: res.sentences!, vector_size: res.vector_size! });
+            setGroup("training");
           }
         }).catch(() => {});
       }
@@ -77,94 +67,6 @@ export default function App() {
     setW2vInfo(ready && info ? info : null);
   }
 
-  async function handleReset() {
-    setResetLoading(true);
-    try {
-      await api.w2vReset();
-      setW2vReady(false);
-      setW2vInfo(null);
-    } catch {
-      // ignore
-    } finally {
-      setResetLoading(false);
-    }
-  }
-
-  function handleStepClick(id: NavGroup, needsIndex?: boolean) {
-    if (needsIndex && !ready) return;
-    setGroup(id);
-  }
-
-  // ── W2V trained: stats bar + analysis tabs, no stepper ──
-  if (w2vReady && w2vInfo) {
-    return (
-      <div className="app">
-        <header className="app-header">
-          <h1>Contextual Similarity Engine</h1>
-          {stats && (
-            <div className="header-stats">
-              <span className="badge">{stats.model_name}</span>
-              <span className="badge">{stats.total_documents} docs</span>
-              <span className="badge">{stats.total_chunks} chunks</span>
-            </div>
-          )}
-        </header>
-
-        {serverError && (
-          <div className="server-error-banner">
-            <strong>Server unavailable:</strong> {serverError}
-          </div>
-        )}
-
-        {/* W2V stats bar */}
-        <div className="content">
-          <div className="panel">
-            <div style={{ display: "flex", alignItems: "center", justifyContent: "space-between", flexWrap: "wrap", gap: 12 }}>
-              <h2 style={{ margin: 0 }}>Word2Vec Baseline</h2>
-              <button className="btn btn-secondary" onClick={handleReset} disabled={resetLoading}
-                style={{ fontSize: "0.85em" }}>
-                {resetLoading ? "Resetting..." : "Reset & Retrain"}
-              </button>
-            </div>
-            <div className="metric-grid" style={{ marginTop: 12 }}>
-              <MetricCard value={w2vInfo.vocab_size} label="Vocabulary" />
-              <MetricCard value={w2vInfo.sentences} label="Sentences" />
-              <MetricCard value={w2vInfo.vector_size} label="Dimensions" />
-            </div>
-          </div>
-
-          {/* W2V-specific tools: Similar Words, Compare, Semantic Search */}
-          <Word2VecTools />
-        </div>
-
-        {/* Transformer Analysis sub-tabs */}
-        <nav className="subtabs">
-          {ANALYSIS_TABS.map((t) => (
-            <button
-              key={t.id}
-              className={`subtab ${analysisTab === t.id ? "subtab-active" : ""}`}
-              onClick={() => setAnalysisTab(t.id)}
-            >
-              {t.label}
-            </button>
-          ))}
-        </nav>
-
-        {/* Analysis content */}
-        <main className="content">
-          {analysisTab === "context" && <ContextAnalysis />}
-          {analysisTab === "words" && <SimilarWords />}
-          {analysisTab === "search" && <SemanticSearch />}
-          {analysisTab === "compare" && <TextCompare />}
-          {analysisTab === "keyword" && <KeywordAnalysis />}
-          {analysisTab === "match" && <KeywordMatcher />}
-          {analysisTab === "batch" && <BatchAnalysis />}
-        </main>
-      </div>
-    );
-  }
-
-  // ── Normal stepper flow ──
   return (
     <div className="app">
       <header className="app-header">
@@ -174,9 +76,6 @@ export default function App() {
             <span className="badge">{stats.model_name}</span>
             <span className="badge">{stats.total_documents} docs</span>
             <span className="badge">{stats.total_chunks} chunks</span>
-            <span className={`badge ${stats.index_built ? "badge-ok" : "badge-warn"}`}>
-              {stats.index_built ? "Index ready" : "Index not built"}
-            </span>
           </div>
         )}
       </header>
@@ -187,24 +86,22 @@ export default function App() {
         </div>
       )}
 
-      {/* Progress Stepper */}
+      {/* Progress Stepper — hidden once training is complete */}
+      {!w2vReady && (
       <nav className="stepper">
         {STEPS.map((step, i) => {
-          const disabled = step.needsIndex && !ready;
           const active = group === step.id;
-          const done = step.id === "data" && ready;
+          const done = (step.id === "data" && stats !== null && stats.index_built)
+            || (step.id === "training" && w2vReady);
           return (
             <Fragment key={step.id}>
-              {i > 0 && (
-                <div className={`stepper-line ${!disabled ? "stepper-line-active" : ""}`} />
-              )}
+              {i > 0 && <div className="stepper-line stepper-line-active" />}
               <div className="stepper-item">
                 <button
                   className={`stepper-circle ${active ? "stepper-active" : ""} ${done && !active ? "stepper-done" : ""}`}
-                  onClick={() => handleStepClick(step.id, step.needsIndex)}
-                  disabled={disabled}
+                  onClick={() => setGroup(step.id)}
                 >
-                  {done && !active ? "\u2713" : i + 1}
+                  {done && !active ? "✓" : i + 1}
                 </button>
                 <span className={`stepper-label ${active ? "stepper-label-active" : ""}`}>
                   {step.label}
@@ -214,62 +111,79 @@ export default function App() {
           );
         })}
       </nav>
-
-      {/* Sub-tabs */}
-      {group === "training" && (
-        <nav className="subtabs">
-          {TRAINING_TABS.map((t) => (
-            <button
-              key={t.id}
-              className={`subtab ${trainingTab === t.id ? "subtab-active" : ""}`}
-              onClick={() => setTrainingTab(t.id)}
-            >
-              {t.label}
-            </button>
-          ))}
-        </nav>
-      )}
-
-      {group === "analysis" && (
-        <nav className="subtabs">
-          {ANALYSIS_TABS.map((t) => (
-            <button
-              key={t.id}
-              className={`subtab ${analysisTab === t.id ? "subtab-active" : ""}`}
-              onClick={() => setAnalysisTab(t.id)}
-            >
-              {t.label}
-            </button>
-          ))}
-        </nav>
       )}
 
       {/* Content */}
       <main className="content">
         {group === "data" && (
           <>
-            <DatasetPanel onStatsUpdate={setStats} />
+            <DatasetPanel onStatsUpdate={setStats} onLoaded={() => setGroup("training")} />
             <button
               className="collapsible-toggle"
               onClick={() => setShowManualSetup(!showManualSetup)}
             >
-              <span className="collapsible-arrow">{showManualSetup ? "\u25be" : "\u25b8"}</span>
+              <span className="collapsible-arrow">{showManualSetup ? "▾" : "▸"}</span>
               Or add documents manually
             </button>
             {showManualSetup && <EngineSetup onStatsUpdate={setStats} />}
           </>
         )}
 
-        {group === "training" && trainingTab === "model" && <TrainingPanel />}
-        {group === "training" && trainingTab === "w2v" && <Word2VecPanel onReady={handleW2vReady} />}
+        {group === "training" && !w2vReady && (
+          <>
+            <nav className="subtabs">
+              {TRAINING_TABS.map((t) => (
+                <button
+                  key={t.id}
+                  className={`subtab ${trainingTab === t.id ? "subtab-active" : ""}`}
+                  onClick={() => setTrainingTab(t.id)}
+                >
+                  {t.label}
+                </button>
+              ))}
+            </nav>
+            {trainingTab === "model" && <TrainingPanel />}
+            {trainingTab === "w2v" && <Word2VecPanel onReady={handleW2vReady} />}
+          </>
+        )}
+
+        {group === "training" && w2vReady && w2vInfo && (
+          <>
+            <div className="panel">
+              <h2 style={{ marginTop: 0 }}>Trained Corpus</h2>
+              <p className="panel-desc">
+                Word2Vec model is trained and persisted. Use the tools below to explore similarity.
+              </p>
+              <div className="metric-grid">
+                <MetricCard value={w2vInfo.vocab_size} label="Vocabulary" />
+                <MetricCard value={w2vInfo.sentences} label="Sentences" />
+                <MetricCard value={w2vInfo.vector_size} label="Dimensions" />
+              </div>
+            </div>
 
-        {group === "analysis" && analysisTab === "context" && <ContextAnalysis />}
-        {group === "analysis" && analysisTab === "words" && <SimilarWords />}
-        {group === "analysis" && analysisTab === "search" && <SemanticSearch />}
-        {group === "analysis" && analysisTab === "compare" && <TextCompare />}
-        {group === "analysis" && analysisTab === "keyword" && <KeywordAnalysis />}
-        {group === "analysis" && analysisTab === "match" && <KeywordMatcher />}
-        {group === "analysis" && analysisTab === "batch" && <BatchAnalysis />}
+            <nav className="subtabs">
+              {ANALYSIS_TABS.map((t) => (
+                <button
+                  key={t.id}
+                  className={`subtab ${analysisTab === t.id ? "subtab-active" : ""}`}
+                  onClick={() => setAnalysisTab(t.id)}
+                >
+                  {t.label}
+                </button>
+              ))}
+            </nav>
+
+            {analysisTab === "context" && (
+              <>
+                <SimilarWords />
+                <TextCompare />
+                <SemanticSearch />
+                <ContextAnalysis />
+              </>
+            )}
+            {analysisTab === "anomalies" && <AnomalyPanel />}
+          </>
+        )}
       </main>
     </div>
   );

frontend/src/api.ts

@@ -9,6 +9,7 @@ import type {
   W2VInitResponse, W2VQueryResult, W2VSimilarWord,
   DatasetInfo, DatasetLoadRequest, DatasetLoadResponse, DatasetPreviewResponse,
   ContextAnalysisResponse,
+  BackgroundStatus, AnomalySweepResponse, AnomalyRelationResponse, IncongruenceResponse,
 } from "./types";
 
 // HuggingFace Spaces proxy requires the __sign token on every request.
@@ -146,9 +147,6 @@ export const api = {
   w2vStatus: () =>
     client.get<{ ready: boolean; vocab_size?: number; sentences?: number; vector_size?: number; has_saved_state?: boolean }>("/w2v/status").then(r => r.data),
 
-  w2vReset: () =>
-    client.post<{ status: string; message: string }>("/w2v/reset").then(r => r.data),
-
   w2vCompare: (data: { text_a: string; text_b: string }) =>
     client.post<CompareResponse>("/w2v/compare", data).then(r => r.data),
 
@@ -158,6 +156,22 @@ export const api = {
   w2vSimilarWords: (data: { word: string; top_k: number }) =>
     client.post<{ word: string; similar: W2VSimilarWord[] }>("/w2v/similar-words", data).then(r => r.data),
 
+  // ---- Anomalous-relation detection ----
+  backgroundStatus: () =>
+    client.get<BackgroundStatus>("/background/status").then(r => r.data),
+
+  backgroundLoad: () =>
+    client.post<BackgroundStatus>("/background/load", null, long).then(r => r.data),
+
+  analyzeAnomalies: (data?: { min_count?: number; max_vocab?: number; neighbours?: number; top_n?: number }) =>
+    client.post<AnomalySweepResponse>("/analyze/anomalies", data ?? {}, long).then(r => r.data),
+
+  analyzeAnomalyRelations: (data: { word: string; top_k?: number }) =>
+    client.post<AnomalyRelationResponse>("/analyze/anomaly-relations", data).then(r => r.data),
+
+  analyzeIncongruence: (data: { keyword: string; canonical_meaning?: string; top_k?: number }) =>
+    client.post<IncongruenceResponse>("/analyze/incongruence", data).then(r => r.data),
+
   // ---- Dataset (HuggingFace) ----
   datasetInfo: () =>
     client.get<DatasetInfo>("/dataset/info").then(r => r.data),

frontend/src/components/AnomalyPanel.tsx

@@ -0,0 +1,289 @@
+import { useState, useEffect } from "react";
+import { api } from "../api";
+import type {
+  BackgroundStatus, AnomalySweepResponse, AnomalyRelationResponse, IncongruenceResponse,
+} from "../types";
+import { useApiCall } from "../hooks/useApiCall";
+import ScoreBar from "./ScoreBar";
+import StatusMessage from "./StatusMessage";
+
+export default function AnomalyPanel() {
+  const [bg, setBg] = useState<BackgroundStatus | null>(null);
+  const [bgLoading, setBgLoading] = useState(false);
+  const [bgError, setBgError] = useState("");
+
+  // Stage A — corpus sweep
+  const [showAdvanced, setShowAdvanced] = useState(false);
+  const [minCount, setMinCount] = useState(5);
+  const [neighbours, setNeighbours] = useState(25);
+  const [topN, setTopN] = useState(30);
+  const sweep = useApiCall<AnomalySweepResponse>();
+
+  // Stage B — per-word relations
+  const [selectedWord, setSelectedWord] = useState<string | null>(null);
+  const relations = useApiCall<AnomalyRelationResponse>();
+
+  // Stage C — contextual incongruence (zoom in)
+  const [keyword, setKeyword] = useState("");
+  const [canonical, setCanonical] = useState("");
+  const incong = useApiCall<IncongruenceResponse>();
+
+  useEffect(() => {
+    api.backgroundStatus().then(setBg).catch(() => {});
+  }, []);
+
+  async function loadBackground() {
+    setBgLoading(true); setBgError("");
+    try {
+      setBg(await api.backgroundLoad());
+    } catch {
+      setBgError("Background model failed to load (network/disk). Anomaly detection needs it.");
+    } finally {
+      setBgLoading(false);
+    }
+  }
+
+  async function runSweep() {
+    setSelectedWord(null);
+    relations.clear();
+    const res = await sweep.run(() =>
+      api.analyzeAnomalies({ min_count: minCount, neighbours, top_n: topN }));
+    if (res && !bg?.ready) api.backgroundStatus().then(setBg).catch(() => {});
+  }
+
+  async function drillInto(word: string) {
+    setSelectedWord(word);
+    await relations.run(() => api.analyzeAnomalyRelations({ word, top_k: 15 }));
+  }
+
+  async function zoomIn(word: string, gloss?: string) {
+    setKeyword(word);
+    if (gloss !== undefined) setCanonical(gloss);
+    await incong.run(() =>
+      api.analyzeIncongruence({ keyword: word, canonical_meaning: gloss || undefined, top_k: 10 }));
+    document.getElementById("zoom-section")?.scrollIntoView({ behavior: "smooth" });
+  }
+
+  const bgReady = bg?.ready ?? false;
+
+  return (
+    <div>
+      {/* Background model status */}
+      <div className="panel">
+        <h2>Anomalous Relations</h2>
+        <p className="panel-desc">
+          Find <strong>code-word candidates</strong>: common English words that behave uncommonly
+          in this corpus. We contrast each word's neighbours in the corpus Word2Vec against a
+          pretrained general-English model (GloVe). A relation is flagged when it is{" "}
+          <em>strong here but weak/absent in normal English</em> — not merely "low similarity".
+        </p>
+        {bg && (
+          <div className="flex-row" style={{ alignItems: "center", gap: 8 }}>
+            <span
+              className="badge"
+              style={{
+                background: `rgba(${bgReady ? "74, 222, 128" : "255, 170, 0"}, 0.15)`,
+                color: bgReady ? "var(--ok)" : "var(--accent)",
+              }}
+            >
+              {bg.model_name}: {bgReady ? `ready (${bg.vocab_size.toLocaleString()} words)` : "not loaded"}
+            </span>
+            {!bgReady && (
+              <button className="btn" onClick={loadBackground} disabled={bgLoading}>
+                {bgLoading ? <><span className="spinner" /> Downloading…</> : "Load background model"}
+              </button>
+            )}
+          </div>
+        )}
+        {bgError && <div className="mt-2"><StatusMessage type="err" message={bgError} /></div>}
+      </div>
+
+      {/* Stage A — corpus sweep */}
+      <div className="panel">
+        <h3 style={{ marginTop: 0 }}>1 · Scan corpus for anomalous words</h3>
+        <p className="panel-desc">
+          Ranks words by neighbour-set divergence (z-scored across the vocabulary). Higher z = the
+          word's corpus associations look more unlike general English.
+        </p>
+
+        <button className="advanced-toggle" onClick={() => setShowAdvanced(!showAdvanced)}>
+          {showAdvanced ? "▾" : "▸"} Advanced Settings
+        </button>
+        {showAdvanced && (
+          <div className="advanced-section">
+            <div className="form-row">
+              <div className="form-group" style={{ maxWidth: 130 }}>
+                <label>Min corpus freq</label>
+                <input type="number" value={minCount} onChange={e => setMinCount(+e.target.value)} min={1} max={1000} />
+              </div>
+              <div className="form-group" style={{ maxWidth: 130 }}>
+                <label>Neighbours (k)</label>
+                <input type="number" value={neighbours} onChange={e => setNeighbours(+e.target.value)} min={5} max={100} />
+              </div>
+              <div className="form-group" style={{ maxWidth: 130 }}>
+                <label>Top N results</label>
+                <input type="number" value={topN} onChange={e => setTopN(+e.target.value)} min={1} max={200} />
+              </div>
+            </div>
+          </div>
+        )}
+
+        <button className="btn btn-primary" onClick={runSweep} disabled={sweep.loading} style={{ marginTop: 8 }}>
+          {sweep.loading ? <><span className="spinner" /> Scanning…</> : "Scan corpus"}
+        </button>
+
+        {sweep.error && <div className="mt-2"><StatusMessage type="err" message={sweep.error} /></div>}
+        {sweep.data?.note && <div className="mt-2"><StatusMessage type="err" message={sweep.data.note} /></div>}
+
+        {sweep.data && sweep.data.results.length > 0 && (
+          <div className="mt-2">
+            <div className="section-label">
+              {sweep.data.results.length} flagged · shared vocab {sweep.data.vocab_size.toLocaleString()} ·
+              mean shift {sweep.data.shift_mean}
+            </div>
+            <table className="data-table">
+              <thead>
+                <tr>
+                  <th>Word</th><th>Freq</th><th>z</th>
+                  <th>Surprising neighbours (here, not normal)</th><th></th>
+                </tr>
+              </thead>
+              <tbody>
+                {sweep.data.results.map((r) => (
+                  <tr
+                    key={r.word}
+                    onClick={() => drillInto(r.word)}
+                    style={{ cursor: "pointer", background: selectedWord === r.word ? "rgba(108,140,255,0.08)" : undefined }}
+                  >
+                    <td style={{ fontWeight: 600 }}>{r.word}</td>
+                    <td>{r.corpus_frequency}</td>
+                    <td>
+                      <span className="badge" style={{
+                        background: `rgba(${r.z_score >= 2 ? "255,107,107" : "108,140,255"},0.15)`,
+                        color: r.z_score >= 2 ? "var(--err)" : "var(--accent)",
+                      }}>{r.z_score.toFixed(2)}</span>
+                    </td>
+                    <td style={{ fontSize: "0.85rem" }}>{r.surprising_neighbors.join(", ") || "—"}</td>
+                    <td style={{ color: "var(--accent)", fontSize: "0.8rem" }}>inspect →</td>
+                  </tr>
+                ))}
+              </tbody>
+            </table>
+          </div>
+        )}
+      </div>
+
+      {/* Stage B — per-word relations drilldown */}
+      {selectedWord && (
+        <div className="panel">
+          <h3 style={{ marginTop: 0 }}>2 · Relations for "{selectedWord}"</h3>
+          {relations.loading && <StatusMessage type="loading" message="Computing relations…" />}
+          {relations.error && <StatusMessage type="err" message={relations.error} />}
+          {relations.data && !relations.data.found && (
+            <StatusMessage type="err" message={`"${selectedWord}" — ${relations.data.reason}.`} />
+          )}
+          {relations.data?.found && (
+            <>
+              <p className="panel-desc">
+                Surprise = (how strongly tied here) − (how strongly tied in general English), each
+                standardised within its own space. High surprise = the suspicious pairing.
+              </p>
+              <table className="data-table">
+                <thead>
+                  <tr><th>Neighbour</th><th>Surprise</th><th>Corpus sim</th><th>Normal-English sim</th></tr>
+                </thead>
+                <tbody>
+                  {relations.data.relations.map((rel) => (
+                    <tr key={rel.neighbor}>
+                      <td style={{ fontWeight: 600 }}>{rel.neighbor}</td>
+                      <td><ScoreBar score={rel.surprise} max={4} /></td>
+                      <td>{rel.corpus_sim.toFixed(3)}</td>
+                      <td>{rel.background_sim.toFixed(3)}</td>
+                    </tr>
+                  ))}
+                </tbody>
+              </table>
+              {relations.data.normal_neighbors && (
+                <div className="mt-2">
+                  <div className="section-label">For contrast — "{selectedWord}" normally relates to:</div>
+                  <div style={{ fontSize: "0.85rem", color: "var(--muted)" }}>
+                    {relations.data.normal_neighbors.map(n => n.neighbor).join(", ")}
+                  </div>
+                </div>
+              )}
+              <button className="btn btn-primary mt-2" onClick={() => zoomIn(selectedWord, "")}>
+                Zoom in on occurrences →
+              </button>
+            </>
+          )}
+        </div>
+      )}
+
+      {/* Stage C — contextual incongruence */}
+      <div className="panel" id="zoom-section">
+        <h3 style={{ marginTop: 0 }}>3 · Zoom in — incongruent occurrences</h3>
+        <p className="panel-desc">
+          Uses the transformer to rank each occurrence of a keyword by how unlike its norm it is.
+          Leave the meaning blank to compare against the keyword's <em>typical</em> usage in this
+          corpus, or supply a dictionary meaning (e.g. "pizza, an Italian food") to flag usages that
+          drift from it. Highest-incongruence chunks are the candidate coded usages.
+        </p>
+        <div className="form-row">
+          <div className="form-group">
+            <label>Keyword</label>
+            <input value={keyword} onChange={e => setKeyword(e.target.value)}
+              onKeyDown={e => e.key === "Enter" && keyword.trim() && zoomIn(keyword.trim(), canonical)}
+              placeholder="e.g. pizza" />
+          </div>
+          <div className="form-group" style={{ flex: 2 }}>
+            <label>Canonical meaning (optional)</label>
+            <input value={canonical} onChange={e => setCanonical(e.target.value)}
+              onKeyDown={e => e.key === "Enter" && keyword.trim() && zoomIn(keyword.trim(), canonical)}
+              placeholder="leave blank to use corpus-typical usage" />
+          </div>
+          <div className="form-group form-group-sm">
+            <label>&nbsp;</label>
+            <button className="btn btn-primary" disabled={incong.loading || !keyword.trim()}
+              onClick={() => zoomIn(keyword.trim(), canonical)}>
+              {incong.loading ? "…" : "Zoom"}
+            </button>
+          </div>
+        </div>
+
+        {incong.error && <StatusMessage type="err" message={incong.error} />}
+        {incong.data && incong.data.total_occurrences === 0 && (
+          <StatusMessage type="err" message={`No occurrences of "${incong.data.keyword}" found.`} />
+        )}
+        {incong.data && incong.data.occurrences.length > 0 && (
+          <div className="mt-2">
+            <div className="section-label">
+              {incong.data.total_occurrences} occurrences · reference: {incong.data.reference} ·
+              median incongruence {incong.data.median_incongruence}
+            </div>
+            <div className="flex-col gap-3">
+              {incong.data.occurrences.map((occ, i) => (
+                <div key={i} className="result-card">
+                  <div className="result-header">
+                    <span className="context-snippet-source">{occ.doc_id} · chunk {occ.chunk_index}</span>
+                    <span className="badge" style={{
+                      background: "rgba(255,107,107,0.15)", color: "var(--err)",
+                    }}>incongruence {occ.incongruence.toFixed(3)}</span>
+                  </div>
+                  <div className="context-snippet mt-2">{occ.snippet}</div>
+                  {occ.entities.length > 0 && (
+                    <div className="mt-2">
+                      <span className="section-label">Co-occurring: </span>
+                      {occ.entities.map((e, j) => (
+                        <span key={j} className="badge" style={{ marginRight: 4 }}>{e}</span>
+                      ))}
+                    </div>
+                  )}
+                </div>
+              ))}
+            </div>
+          </div>
+        )}
+      </div>
+    </div>
+  );
+}

frontend/src/components/DatasetPanel.tsx

@@ -10,9 +10,10 @@ import LogViewer from "./LogViewer";
 
 interface Props {
   onStatsUpdate?: (stats: any) => void;
+  onLoaded?: () => void;
 }
 
-export default function DatasetPanel({ onStatsUpdate }: Props) {
+export default function DatasetPanel({ onStatsUpdate, onLoaded }: Props) {
   const [info, setInfo] = useState<DatasetInfo | null>(null);
   const [error, setError] = useState("");
 
@@ -65,6 +66,7 @@ export default function DatasetPanel({ onStatsUpdate }: Props) {
           console.warn("Failed to refresh stats after load:", e);
         }
       }
+      onLoaded?.();  // advance to the training step
     } catch (err) {
       setError(getErrorMessage(err));
     } finally {

frontend/src/components/Word2VecPanel.tsx

@@ -1,9 +1,7 @@
 import { useState, useEffect } from "react";
 import { api, getErrorMessage } from "../api";
-import type { W2VInitResponse } from "../types";
 import StatusMessage from "./StatusMessage";
 import LogViewer from "./LogViewer";
-import MetricCard from "./MetricCard";
 
 interface Props {
   onReady: (ready: boolean, info?: { vocab_size: number; sentences: number; vector_size: number }) => void;
@@ -11,8 +9,6 @@ interface Props {
 
 export default function Word2VecPanel({ onReady }: Props) {
   const [statusChecked, setStatusChecked] = useState(false);
-  const [trainResult, setTrainResult] = useState<W2VInitResponse | null>(null);
-
   const [vectorSize, setVectorSize] = useState(100);
   const [windowSize, setWindowSize] = useState(5);
   const [w2vEpochs, setW2vEpochs] = useState(50);
@@ -30,14 +26,14 @@ export default function Word2VecPanel({ onReady }: Props) {
   }, []);
 
   async function handleTrainFromEngine() {
-    setInitLoading(true); setError(""); setTrainResult(null);
+    setInitLoading(true); setError("");
     try {
       const res = await api.w2vInitFromEngine({
         vector_size: vectorSize,
         window: windowSize,
         epochs: w2vEpochs,
       });
-      setTrainResult(res);
+      onReady(true, { vocab_size: res.vocab_size, sentences: res.sentences, vector_size: res.vector_size });
     } catch (err) {
       setError(getErrorMessage(err));
     } finally {
@@ -49,42 +45,16 @@ export default function Word2VecPanel({ onReady }: Props) {
     return <div className="panel"><p>Checking Word2Vec status...</p></div>;
   }
 
-  // Training complete — show results + continue button
-  if (trainResult) {
-    return (
-      <div>
-        <div className="panel">
-          <h2>Training Complete</h2>
-          <div className="metric-grid">
-            <MetricCard value={trainResult.vocab_size} label="Vocabulary" />
-            <MetricCard value={trainResult.sentences} label="Sentences" />
-            <MetricCard value={trainResult.vector_size} label="Dimensions" />
-            <MetricCard value={`${trainResult.seconds}s`} label="Train Time" />
-          </div>
-          <StatusMessage type="ok" message="Word2Vec model trained and saved. It will persist across restarts." />
-          <button className="btn btn-primary" style={{ marginTop: 12 }}
-            onClick={() => onReady(true, { vocab_size: trainResult.vocab_size, sentences: trainResult.sentences, vector_size: trainResult.vector_size })}>
-            Continue to Analysis
-          </button>
-        </div>
-
-        <LogViewer active={false} />
-      </div>
-    );
-  }
-
-  // Training form
   return (
     <div>
       <div className="panel">
         <h2>Word2Vec Baseline (gensim)</h2>
         <p className="panel-desc">
-          Static embeddings — one vector per word, no context awareness.
-          Train on all documents loaded in the engine to use as a baseline comparison.
+          Static embeddings trained on all documents loaded in the engine. Training runs once and persists across restarts.
         </p>
 
         <button className="advanced-toggle" onClick={() => setShowAdvanced(!showAdvanced)}>
-          {showAdvanced ? "\u25be" : "\u25b8"} Advanced Settings
+          {showAdvanced ? "▾" : "▸"} Advanced Settings
         </button>
 
         {showAdvanced && (
@@ -100,7 +70,7 @@ export default function Word2VecPanel({ onReady }: Props) {
               </div>
               <div className="form-group" style={{ maxWidth: 120 }}>
                 <label>Epochs</label>
-                <input type="number" value={w2vEpochs} onChange={e => setW2vEpochs(+e.target.value)} min={5} max={200} />
+                <input type="number" value={w2vEpochs} onChange={e => setW2vEpochs(+e.target.value)} min={5} max={1000} />
               </div>
             </div>
           </div>

frontend/src/types.ts

@@ -297,6 +297,71 @@ export interface ContextAnalysisResponse {
   meanings: ContextMeaning[];
 }
 
+// ---- Anomalous-relation detection ----
+
+export interface BackgroundStatus {
+  model_name: string;
+  ready: boolean;
+  load_failed: boolean;
+  vocab_size: number;
+  vector_size: number;
+}
+
+export interface AnomalyWord {
+  word: string;
+  corpus_frequency: number;
+  shift: number;
+  z_score: number;
+  surprising_neighbors: string[];
+  normal_neighbors: string[];
+}
+
+export interface AnomalySweepResponse {
+  ready: boolean;
+  vocab_size: number;
+  neighbours?: number;
+  shift_mean?: number;
+  shift_std?: number;
+  note?: string;
+  results: AnomalyWord[];
+}
+
+export interface AnomalyRelation {
+  neighbor: string;
+  corpus_sim: number;
+  background_sim: number;
+  corpus_z: number;
+  background_z: number;
+  surprise: number;
+}
+
+export interface AnomalyRelationResponse {
+  word: string;
+  ready: boolean;
+  found: boolean;
+  reason?: string;
+  corpus_frequency?: number;
+  relations: AnomalyRelation[];
+  normal_neighbors?: { neighbor: string; background_sim: number }[];
+}
+
+export interface IncongruentOccurrence {
+  doc_id: string;
+  chunk_index: number;
+  incongruence: number;
+  snippet: string;
+  entities: string[];
+}
+
+export interface IncongruenceResponse {
+  keyword: string;
+  total_occurrences: number;
+  reference?: string;
+  reference_kind?: "gloss" | "centroid";
+  median_incongruence?: number;
+  occurrences: IncongruentOccurrence[];
+}
+
 // ---- UI State ----
 
 export type EvalSection = "distribution" | "disambiguation" | "retrieval";

frontend/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/app.tsx","./src/api.ts","./src/main.tsx","./src/types.ts","./src/vite-env.d.ts","./src/components/batchanalysis.tsx","./src/components/contextanalysis.tsx","./src/components/datasetpanel.tsx","./src/components/documentviewer.tsx","./src/components/enginesetup.tsx","./src/components/evaluationdashboard.tsx","./src/components/keywordanalysis.tsx","./src/components/keywordmatcher.tsx","./src/components/logviewer.tsx","./src/components/metriccard.tsx","./src/components/scorebar.tsx","./src/components/select.tsx","./src/components/semanticsearch.tsx","./src/components/similarwords.tsx","./src/components/statusmessage.tsx","./src/components/switch.tsx","./src/components/textcompare.tsx","./src/components/toggle.tsx","./src/components/trainingpanel.tsx","./src/components/word2vecpanel.tsx","./src/components/word2vectools.tsx","./src/hooks/useapicall.ts","./src/hooks/usecorpusloader.ts","./src/utils/colors.ts"],"version":"5.9.3"}
+{"root":["./src/app.tsx","./src/api.ts","./src/main.tsx","./src/types.ts","./src/vite-env.d.ts","./src/components/anomalypanel.tsx","./src/components/batchanalysis.tsx","./src/components/contextanalysis.tsx","./src/components/datasetpanel.tsx","./src/components/documentviewer.tsx","./src/components/enginesetup.tsx","./src/components/evaluationdashboard.tsx","./src/components/keywordanalysis.tsx","./src/components/keywordmatcher.tsx","./src/components/logviewer.tsx","./src/components/metriccard.tsx","./src/components/scorebar.tsx","./src/components/select.tsx","./src/components/semanticsearch.tsx","./src/components/similarwords.tsx","./src/components/statusmessage.tsx","./src/components/switch.tsx","./src/components/textcompare.tsx","./src/components/toggle.tsx","./src/components/trainingpanel.tsx","./src/components/word2vecpanel.tsx","./src/components/word2vectools.tsx","./src/hooks/useapicall.ts","./src/hooks/usecorpusloader.ts","./src/utils/colors.ts"],"version":"5.9.3"}

server.py

@@ -34,6 +34,8 @@ from contextual_similarity import ContextualSimilarityEngine
 from evaluation import Evaluator, GroundTruthEntry
 from training import CorpusTrainer
 from word2vec_baseline import Word2VecEngine
+from background_model import BackgroundModel
+from anomaly import sweep_anomalous_words, relation_surprise, contextual_incongruence
 from data_loader import load_raw_dataset, load_raw_to_engine, import_chromadb_to_engine, get_dataset_info
 
 logging.basicConfig(level=logging.INFO)
@@ -145,6 +147,7 @@ app.add_middleware(
 engine: Optional[ContextualSimilarityEngine] = None
 evaluator: Optional[Evaluator] = None
 w2v_engine: Optional[Word2VecEngine] = None
+background_model = BackgroundModel()  # general-English reference (lazy-loaded)
 
 ENGINE_SAVE_DIR = Path(os.environ.get("ENGINE_STATE_DIR", str(BASE_DIR / "engine_state")))
 W2V_SAVE_DIR = Path(os.environ.get("W2V_STATE_DIR", str(BASE_DIR / "w2v_state")))
@@ -280,7 +283,7 @@ class W2VInitRequest(BaseModel):
     corpus_texts: list[str] = Field(max_length=10_000)
     vector_size: int = Field(default=100, ge=50, le=500)
     window: int = Field(default=5, ge=1, le=20)
-    epochs: int = Field(default=50, ge=1, le=200)
+    epochs: int = Field(default=50, ge=1, le=1000)
 
 
 class W2VCompareRequest(BaseModel):
@@ -304,6 +307,26 @@ class ContextAnalysisRequest(BaseModel):
     top_words: int = Field(default=8, ge=1, le=30)
 
 
+class AnomalySweepRequest(BaseModel):
+    min_count: int = Field(default=5, ge=1, le=1000)
+    max_vocab: int = Field(default=3000, ge=100, le=20_000)
+    neighbours: int = Field(default=25, ge=5, le=100)
+    top_n: int = Field(default=30, ge=1, le=200)
+
+
+class AnomalyRelationRequest(BaseModel):
+    word: str = Field(max_length=200)
+    min_count: int = Field(default=5, ge=1, le=1000)
+    max_vocab: int = Field(default=3000, ge=100, le=20_000)
+    top_k: int = Field(default=15, ge=1, le=100)
+
+
+class IncongruenceRequest(BaseModel):
+    keyword: str = Field(max_length=200)
+    canonical_meaning: Optional[str] = Field(default=None, max_length=500)
+    top_k: int = Field(default=10, ge=1, le=100)
+
+
 class DatasetLoadRequest(BaseModel):
     source: Literal["raw", "embeddings"] = "raw"
     max_docs: int = Field(default=500, ge=1, le=100_000)
@@ -521,6 +544,69 @@ def match_keyword(req: KeywordMatchRequest):
     ]}
 
 
+# ------------------------------------------------------------------ #
+#  Anomalous-relation detection (code-word candidates)
+# ------------------------------------------------------------------ #
+
+@app.get("/api/background/status")
+def background_status():
+    """Status of the general-English background model used for anomaly detection."""
+    return background_model.status()
+
+
+@app.post("/api/background/load")
+def background_load():
+    """Eagerly load the background model (otherwise it loads on first anomaly query)."""
+    ok = background_model.load()
+    if not ok:
+        raise HTTPException(503, "Background model failed to load. Check server logs (network/disk).")
+    return background_model.status()
+
+
+@app.post("/api/analyze/anomalies")
+def analyze_anomalies(req: AnomalySweepRequest):
+    """Stage A: rank corpus words by how differently they associate here vs. normal English."""
+    _ensure_w2v()
+    _ensure_background()
+    logger.info("Anomaly sweep: min_count=%d, max_vocab=%d, neighbours=%d, top_n=%d",
+                req.min_count, req.max_vocab, req.neighbours, req.top_n)
+    t0 = time.time()
+    result = sweep_anomalous_words(
+        w2v_engine, background_model,
+        min_count=req.min_count, max_vocab=req.max_vocab,
+        neighbours=req.neighbours, top_n=req.top_n,
+    )
+    logger.info("Anomaly sweep complete: %d words ranked in %.1fs",
+                len(result.get("results", [])), time.time() - t0)
+    return _to_native(result)
+
+
+@app.post("/api/analyze/anomaly-relations")
+def analyze_anomaly_relations(req: AnomalyRelationRequest):
+    """Stage B: for one word, the neighbours strong here but weak/absent in general English."""
+    _ensure_w2v()
+    _ensure_background()
+    logger.info("Anomaly relations: word='%s', top_k=%d", req.word, req.top_k)
+    result = relation_surprise(
+        req.word, w2v_engine, background_model,
+        min_count=req.min_count, max_vocab=req.max_vocab, top_k=req.top_k,
+    )
+    return _to_native(result)
+
+
+@app.post("/api/analyze/incongruence")
+def analyze_incongruence(req: IncongruenceRequest):
+    """Stage C: occurrences where a keyword is used most unlike its norm (transformer-based)."""
+    _ensure_engine(); _ensure_index()
+    logger.info("Incongruence: keyword='%s', canonical=%s, top_k=%d",
+                req.keyword, bool(req.canonical_meaning), req.top_k)
+    result = contextual_incongruence(
+        engine, req.keyword,
+        canonical_meaning=req.canonical_meaning, top_k=req.top_k,
+    )
+    return _to_native(result)
+
+
 # ------------------------------------------------------------------ #
 #  Evaluation endpoints
 # ------------------------------------------------------------------ #
@@ -656,7 +742,7 @@ def w2v_init(req: W2VInitRequest):
 def w2v_init_from_engine(
     vector_size: int = Query(default=100, ge=50, le=500),
     window: int = Query(default=5, ge=1, le=20),
-    epochs: int = Query(default=50, ge=1, le=200),
+    epochs: int = Query(default=50, ge=1, le=1000),
 ):
     """Train Word2Vec directly from all documents already loaded in the engine.
 
@@ -729,18 +815,6 @@ def w2v_status():
     return {"ready": False, "has_saved_state": has_saved}
 
 
-@app.post("/api/w2v/reset")
-def w2v_reset():
-    """Delete saved Word2Vec state and clear the in-memory model."""
-    global w2v_engine
-    w2v_engine = None
-    import shutil
-    if W2V_SAVE_DIR.is_dir():
-        shutil.rmtree(str(W2V_SAVE_DIR))
-        logger.info("Word2Vec state deleted from %s", W2V_SAVE_DIR)
-    return {"status": "ok", "message": "Word2Vec state cleared. You can retrain now."}
-
-
 # ------------------------------------------------------------------ #
 #  Dataset endpoints (HuggingFace Epstein Files)
 # ------------------------------------------------------------------ #
@@ -843,6 +917,10 @@ def _ensure_w2v():
     if w2v_engine is None:
         raise HTTPException(400, "Word2Vec not initialized. POST /api/w2v/init first.")
 
+def _ensure_background():
+    if not background_model.load():
+        raise HTTPException(503, "Background model unavailable (download/load failed). Check server logs.")
+
 def _serialize_analysis(analysis):
     return {
         "keyword": analysis.keyword,