Fix build: bundle code in Space repo

Dockerfile

@@ -1,22 +1,22 @@
-# Hugging Face Space — clones GitHub repo at build time.
-# Push this file + README.md to: uzzam24/complexity-levels-api
+# Hugging Face Space — all code copied into Space repo (no GitHub clone needed).
+# Run sync_to_space.ps1 from project root, then git push the Space repo.
 FROM python:3.11-slim
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential git \
+    build-essential \
     && rm -rf /var/lib/apt/lists/*
 
 RUN useradd -m -u 1000 user
 
-WORKDIR /app
+WORKDIR /app/repo
 
-RUN git clone --depth 1 https://github.com/muhammad245/complexityLevels.git /app/repo
-
-RUN pip install --no-cache-dir -r /app/repo/deploy/backend/requirements.txt huggingface_hub \
+COPY src/deploy/backend/requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt huggingface_hub \
     && python -m spacy download en_core_web_sm
 
-RUN chmod +x /app/repo/deploy/huggingface/entrypoint.sh \
-    && chown -R user:user /app
+COPY --chown=user:user src/ /app/repo/
+
+RUN chmod +x /app/repo/deploy/huggingface/entrypoint.sh
 
 ENV MODEL_PATH=/app/model
 ENV HF_MODEL_REPO=uzzam24/deberta-complexity-lcp

src/deploy/backend/app/config.py

@@ -0,0 +1,39 @@
+"""Backend configuration — model path and CORS."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+# Repo root (Complexity_level/) when running locally or in Docker /app
+REPO_ROOT = Path(__file__).resolve().parents[3]
+BACKEND_ROOT = Path(__file__).resolve().parents[1]
+
+
+def resolve_model_path() -> Path:
+    """MODEL_PATH env, then overnight_bundle, then exported_models."""
+    if env := os.environ.get("MODEL_PATH"):
+        return Path(env).expanduser().resolve()
+
+    candidates = [
+        REPO_ROOT / "model",
+        Path("/app/model"),
+        REPO_ROOT / "overnight_bundle" / "exported_models" / "deberta_best",
+        REPO_ROOT / "exported_models" / "deberta_best",
+        Path("/app/overnight_bundle/exported_models/deberta_best"),
+        Path("/app/exported_models/deberta_best"),
+    ]
+    for path in candidates:
+        if (path / "model_weights.pt").exists():
+            return path
+    # Return first candidate for error messaging even if weights missing
+    return candidates[0]
+
+
+MODEL_PATH = resolve_model_path()
+CORS_ORIGINS = [
+    o.strip()
+    for o in os.environ.get("CORS_ORIGINS", "http://localhost:3000,http://127.0.0.1:3000").split(",")
+    if o.strip()
+]
+PORT = int(os.environ.get("PORT", "8000"))

src/deploy/backend/app/main.py

@@ -0,0 +1,135 @@
+"""FastAPI backend for Railway — word complexity inference."""
+
+from __future__ import annotations
+
+import os
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+
+from app.config import CORS_ORIGINS, MODEL_PATH
+from app.schemas import (
+    EfficiencyRequest,
+    EfficiencyResponse,
+    HealthResponse,
+    InterveneAllRequest,
+    InterveneAllResponse,
+    InterveneRequest,
+    InterveneResponse,
+    PredictRequest,
+    PredictResponse,
+)
+from app.services.efficiency import compare_efficiency
+from app.services.interventions import run_intervention
+from app.services.model import get_predictor, model_status, predict
+
+
+@asynccontextmanager
+async def lifespan(_app: FastAPI):
+    if os.environ.get("EAGER_LOAD_MODEL", "1") == "1":
+        try:
+            get_predictor()
+        except RuntimeError as exc:
+            print(f"WARN: model not loaded at startup: {exc}")
+    yield
+
+
+app = FastAPI(title="Word Complexity API", version="1.0.0", lifespan=lifespan)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"] if os.environ.get("CORS_ALLOW_ALL") == "1" else CORS_ORIGINS,
+    allow_credentials=os.environ.get("CORS_ALLOW_ALL") != "1",
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/")
+def root():
+    ready, err = model_status()
+    return {
+        "service": "Word Complexity API",
+        "health": "/health",
+        "docs": "/docs",
+        "model_ready": ready,
+        "model_error": err,
+    }
+
+
+@app.get("/health", response_model=HealthResponse)
+def health():
+    ready, err = model_status()
+    return HealthResponse(
+        status="ok" if ready else "degraded",
+        model_path=str(MODEL_PATH),
+        model_ready=ready,
+        model_error=err,
+    )
+
+
+@app.post("/api/predict", response_model=PredictResponse)
+def api_predict(body: PredictRequest):
+    try:
+        data, latency_ms, target_in = predict(body.sentence.strip(), body.target_word.strip())
+    except RuntimeError as exc:
+        raise HTTPException(status_code=503, detail=str(exc)) from exc
+    return PredictResponse(**data, latency_ms=latency_ms, target_in_sentence=target_in)
+
+
+@app.post("/api/intervene", response_model=InterveneResponse)
+def api_intervene(body: InterveneRequest):
+    try:
+        predictor = get_predictor()
+        detail = run_intervention(
+            predictor,
+            body.sentence.strip(),
+            body.target_word.strip(),
+            body.reason,
+            body.use_llm,
+        )
+    except RuntimeError as exc:
+        raise HTTPException(status_code=503, detail=str(exc)) from exc
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    return InterveneResponse(intervention=detail)
+
+
+@app.post("/api/intervene/all", response_model=InterveneAllResponse)
+def api_intervene_all(body: InterveneAllRequest):
+    try:
+        predictor = get_predictor()
+        base = predictor.predict(body.sentence.strip(), body.target_word.strip())
+        from utils import REASON_ORDER
+
+        results = [
+            run_intervention(
+                predictor,
+                body.sentence.strip(),
+                body.target_word.strip(),
+                reason,
+                body.use_llm,
+                predicted_reason=body.predicted_reason,
+            )
+            for reason in REASON_ORDER
+        ]
+    except RuntimeError as exc:
+        raise HTTPException(status_code=503, detail=str(exc)) from exc
+
+    best = max(results, key=lambda r: r.hardness_drop)
+    faithful = None
+    if body.predicted_reason:
+        faithful = best.reason == body.predicted_reason
+
+    return InterveneAllResponse(
+        base_level=base.complexity_level,
+        results=results,
+        best_reason=best.reason,
+        faithful=faithful,
+    )
+
+
+@app.post("/api/efficiency", response_model=EfficiencyResponse)
+def api_efficiency(body: EfficiencyRequest):
+    return compare_efficiency(body.sentence.strip(), body.target_word.strip(), body.local_latency_ms)

src/deploy/backend/app/schemas.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+
+class PredictRequest(BaseModel):
+    sentence: str = Field(..., min_length=1)
+    target_word: str = Field(..., min_length=1)
+
+
+class PredictResponse(BaseModel):
+    complexity_level: str
+    level_id: int
+    level_probs: dict[str, float]
+    difficult_class_prob: float
+    reason: str | None
+    reason_id: int | None
+    reason_probs: dict[str, float] | None
+    latency_ms: float
+    target_in_sentence: bool
+
+
+class InterveneRequest(BaseModel):
+    sentence: str
+    target_word: str
+    reason: str
+    use_llm: bool = False
+
+
+class InterventionDetail(BaseModel):
+    reason: str
+    original_sentence: str
+    edited_sentence: str
+    edit_method: str
+    before_level: str
+    after_level: str
+    hardness_drop: float
+    matches_predicted: bool | None = None
+
+
+class InterveneResponse(BaseModel):
+    intervention: InterventionDetail
+
+
+class InterveneAllRequest(BaseModel):
+    sentence: str
+    target_word: str
+    predicted_reason: str | None = None
+    use_llm: bool = False
+
+
+class InterveneAllResponse(BaseModel):
+    base_level: str
+    results: list[InterventionDetail]
+    best_reason: str
+    faithful: bool | None
+
+
+class EfficiencyRequest(BaseModel):
+    sentence: str
+    target_word: str
+    local_latency_ms: float = Field(..., ge=0)
+
+
+class EfficiencyResponse(BaseModel):
+    local_latency_ms: float
+    ai_latency_ms: float
+    ai_provider: str
+    ai_cost_usd: float
+    ai_cost_per_1k_usd: float
+    local_cost_usd: float
+    speedup_factor: float
+
+
+class HealthResponse(BaseModel):
+    status: str
+    model_path: str
+    model_ready: bool
+    model_error: str | None = None

src/deploy/backend/app/services/efficiency.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import sys
+
+from app.config import REPO_ROOT
+from app.schemas import EfficiencyResponse
+
+
+def _ensure_path() -> None:
+    if str(REPO_ROOT) not in sys.path:
+        sys.path.insert(0, str(REPO_ROOT))
+
+
+def compare_efficiency(sentence: str, target_word: str, local_latency_ms: float) -> EfficiencyResponse:
+    _ensure_path()
+    from ui.efficiency_panel import compare_query
+
+    cmp = compare_query(sentence, target_word, local_latency_ms)
+    return EfficiencyResponse(
+        local_latency_ms=cmp.local_latency_ms,
+        ai_latency_ms=cmp.ai_latency_ms,
+        ai_provider=cmp.ai_provider,
+        ai_cost_usd=cmp.ai_cost_usd,
+        ai_cost_per_1k_usd=cmp.ai_cost_per_1k_usd,
+        local_cost_usd=cmp.local_cost_usd,
+        speedup_factor=cmp.speedup_factor,
+    )

src/deploy/backend/app/services/interventions.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import sys
+
+from app.config import REPO_ROOT
+from app.schemas import InterventionDetail
+
+
+def _ensure_path() -> None:
+    if str(REPO_ROOT) not in sys.path:
+        sys.path.insert(0, str(REPO_ROOT))
+
+
+def run_intervention(
+    predictor,
+    sentence: str,
+    target_word: str,
+    reason: str,
+    use_llm: bool,
+    predicted_reason: str | None = None,
+) -> InterventionDetail:
+    _ensure_path()
+    from reason_interventions import apply_intervention
+
+    base = predictor.predict(sentence, target_word)
+    intervention = apply_intervention(reason, sentence, target_word, use_llm=use_llm)
+    after = predictor.predict(intervention.edited_sentence, target_word)
+    drop = base.difficult_class_prob - after.difficult_class_prob
+    return InterventionDetail(
+        reason=reason,
+        original_sentence=intervention.original_sentence,
+        edited_sentence=intervention.edited_sentence,
+        edit_method=intervention.edit_method,
+        before_level=base.complexity_level,
+        after_level=after.complexity_level,
+        hardness_drop=round(drop, 4),
+        matches_predicted=reason == predicted_reason if predicted_reason else None,
+    )

src/deploy/backend/app/services/model.py

@@ -0,0 +1,58 @@
+"""Lazy-load ComplexityPredictor from repo inference code."""
+
+from __future__ import annotations
+
+import sys
+import time
+from pathlib import Path
+
+from app.config import MODEL_PATH, REPO_ROOT
+
+_predictor = None
+_load_error: str | None = None
+
+
+def _ensure_inference_path() -> None:
+    if str(REPO_ROOT) not in sys.path:
+        sys.path.insert(0, str(REPO_ROOT))
+
+
+def get_predictor():
+    global _predictor, _load_error
+    if _predictor is not None:
+        return _predictor
+    if _load_error is not None:
+        raise RuntimeError(_load_error)
+
+    _ensure_inference_path()
+    weights = MODEL_PATH / "model_weights.pt"
+    if not weights.exists():
+        _load_error = (
+            f"Model weights not found at {weights}. "
+            "Set MODEL_PATH or place deberta_best under overnight_bundle/exported_models/."
+        )
+        raise RuntimeError(_load_error)
+
+    from predictor import ComplexityPredictor
+
+    t0 = time.perf_counter()
+    _predictor = ComplexityPredictor.load(MODEL_PATH)
+    print(f"Model loaded from {MODEL_PATH} in {(time.perf_counter() - t0):.1f}s")
+    return _predictor
+
+
+def model_status() -> tuple[bool, str | None]:
+    try:
+        get_predictor()
+        return True, None
+    except Exception as exc:
+        return False, str(exc)
+
+
+def predict(sentence: str, target_word: str) -> tuple[dict, float, bool]:
+    predictor = get_predictor()
+    t0 = time.perf_counter()
+    result = predictor.predict(sentence, target_word)
+    latency_ms = (time.perf_counter() - t0) * 1000
+    target_in = target_word in sentence
+    return result.to_dict(), latency_ms, target_in

src/deploy/backend/requirements.txt

@@ -0,0 +1,11 @@
+fastapi>=0.110.0
+uvicorn[standard]>=0.27.0
+python-multipart>=0.0.9
+torch>=2.0.0
+transformers>=4.36.0
+numpy>=1.24.0
+sentencepiece>=0.1.99
+spacy>=3.7.0
+wordfreq>=3.0.0
+nltk>=3.8.0
+openai>=1.0.0

src/deploy/huggingface/download_model.py

@@ -0,0 +1,61 @@
+"""
+Download exported model from Hugging Face Hub before API startup.
+
+Set Space secret / env:
+  HF_MODEL_REPO=your-username/deberta-complexity-lcp
+  MODEL_PATH=/app/model
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+
+def main() -> None:
+    repo = os.environ.get("HF_MODEL_REPO", "").strip()
+    target = Path(os.environ.get("MODEL_PATH", "/app/model"))
+
+    if not repo:
+        print("HF_MODEL_REPO not set — expecting model_weights.pt already on disk.")
+        weights = target / "model_weights.pt"
+        if weights.exists():
+            print(f"Found {weights}")
+        else:
+            print(f"WARN: no weights at {weights}", file=sys.stderr)
+        return
+
+    weights = target / "model_weights.pt"
+    if weights.exists():
+        print(f"Model already present at {weights}")
+        return
+
+    print(f"Downloading {repo} → {target}")
+    from huggingface_hub import snapshot_download
+
+    token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
+
+    target.mkdir(parents=True, exist_ok=True)
+    snapshot_download(
+        repo_id=repo,
+        local_dir=str(target),
+        token=token,
+        allow_patterns=[
+            "model_weights.pt",
+            "config.json",
+            "tokenizer/*",
+            "*.json",
+        ],
+    )
+
+    if not weights.exists():
+        raise FileNotFoundError(
+            f"Download finished but {weights} missing. "
+            f"Upload model with: python deploy/huggingface/upload_model_to_hub.py"
+        )
+    print(f"Ready: {weights} ({weights.stat().st_size / 1e6:.1f} MB)")
+
+
+if __name__ == "__main__":
+    main()

src/deploy/huggingface/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+ROOT="${APP_ROOT:-/app}"
+python "$ROOT/deploy/huggingface/download_model.py"
+exec uvicorn app.main:app --host 0.0.0.0 --port "${PORT:-7860}" --app-dir "$ROOT/deploy/backend"

src/linguistic_features.py

@@ -0,0 +1,83 @@
+"""
+Handcrafted linguistic features for hybrid LCP models.
+
+Literature: word frequency is the strongest single predictor; WordNet senses
+capture ambiguity; syntax score captures structural difficulty.
+"""
+
+from __future__ import annotations
+
+import math
+from functools import lru_cache
+
+import numpy as np
+import wordfreq
+
+from syntax_complexity import analyze_syntax
+
+FEATURE_NAMES = [
+    "log_word_frequency",
+    "word_length",
+    "syllable_estimate",
+    "wordnet_senses",
+    "syntax_complexity",
+]
+
+N_LINGUISTIC_FEATURES = len(FEATURE_NAMES)
+
+
+@lru_cache(maxsize=1)
+def _wordnet_ready() -> bool:
+    try:
+        from nltk.corpus import wordnet as wn
+
+        wn.synsets("test")
+        return True
+    except LookupError:
+        import nltk
+
+        nltk.download("wordnet", quiet=True)
+        nltk.download("omw-1.4", quiet=True)
+        return True
+
+
+def _syllable_estimate(word: str) -> float:
+    word = word.lower()
+    if not word:
+        return 0.0
+    vowels = "aeiouy"
+    count = 0
+    prev_vowel = False
+    for ch in word:
+        is_vowel = ch in vowels
+        if is_vowel and not prev_vowel:
+            count += 1
+        prev_vowel = is_vowel
+    return float(max(1, count))
+
+
+def _wordnet_sense_count(word: str) -> float:
+    _wordnet_ready()
+    from nltk.corpus import wordnet as wn
+
+    return float(len(wn.synsets(word.lower())))
+
+
+def extract_linguistic_features(sentence: str, target_word: str) -> np.ndarray:
+    """Return a 5-dim feature vector for one (sentence, target_word) pair."""
+    word = str(target_word).strip()
+    freq = wordfreq.word_frequency(word.lower(), "en")
+    log_freq = math.log10(freq + 1e-12)
+    word_len = len(word)
+    syllables = _syllable_estimate(word)
+    senses = _wordnet_sense_count(word)
+    syntax = analyze_syntax(str(sentence)).complexity_score
+    return np.array([log_freq, word_len, syllables, senses, syntax], dtype=np.float32)
+
+
+def normalize_features(matrix: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+    """Z-score normalize feature matrix; return normalized, mean, std."""
+    mean = matrix.mean(axis=0)
+    std = matrix.std(axis=0)
+    std = np.where(std < 1e-6, 1.0, std)
+    return (matrix - mean) / std, mean, std

src/model_loading.py

@@ -0,0 +1,54 @@
+"""Load trained LCP checkpoints with backward compatibility."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+import torch
+from transformers import AutoTokenizer
+
+from two_head_model import TwoHeadModel, add_tgt_tokens
+from utils import ENCODING_SPAN_MARK, MODELS, POOLING_CLS_CONCAT, POOLING_SPAN
+
+
+def load_model_from_checkpoint(ckpt_path: str | Path, device: torch.device | None = None):
+    ckpt_path = Path(ckpt_path)
+    device = device or torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    ckpt = torch.load(ckpt_path, map_location="cpu", weights_only=False)
+
+    model_key = ckpt.get("model_key", "deberta")
+    encoding = ckpt.get("encoding", ENCODING_SPAN_MARK)
+    pooling = ckpt.get("pooling", POOLING_CLS_CONCAT)  # legacy checkpoints
+    use_linguistic = ckpt.get("use_linguistic_features", False)
+    level_only = ckpt.get("level_only", False)
+
+    tokenizer = AutoTokenizer.from_pretrained(MODELS[model_key])
+    tgt_id, tgt_end_id = add_tgt_tokens(tokenizer)
+
+    model = TwoHeadModel(
+        model_name=MODELS[model_key],
+        pooling_mode=pooling,
+        use_linguistic_features=use_linguistic,
+    )
+    model.encoder.resize_token_embeddings(len(tokenizer))
+    model.load_state_dict(ckpt["model_state"])
+    model.set_tgt_token_ids(ckpt.get("tgt_id", tgt_id), ckpt.get("tgt_end_id", tgt_end_id))
+    model.to(device)
+    model.eval()
+
+    feat_mean = np.array(ckpt["feat_mean"]) if ckpt.get("feat_mean") is not None else None
+    feat_std = np.array(ckpt["feat_std"]) if ckpt.get("feat_std") is not None else None
+
+    meta = {
+        "model_key": model_key,
+        "encoding": encoding,
+        "pooling": pooling,
+        "use_linguistic_features": use_linguistic,
+        "level_only": level_only,
+        "tgt_id": ckpt.get("tgt_id", tgt_id),
+        "tgt_end_id": ckpt.get("tgt_end_id", tgt_end_id),
+        "feat_mean": feat_mean,
+        "feat_std": feat_std,
+    }
+    return model, tokenizer, meta

src/predictor.py

@@ -0,0 +1,200 @@
+"""
+UI-ready inference wrapper for the two-head complexity model.
+"""
+
+from __future__ import annotations
+
+import json
+import pickle
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import torch
+
+from linguistic_features import extract_linguistic_features
+from model_loading import load_model_from_checkpoint
+from utils import (
+    EXPORT_DIR,
+    HARD_LEVELS,
+    ID_TO_LEVEL,
+    ID_TO_REASON,
+    LEVEL_ORDER,
+    MODELS,
+    REASON_ORDER,
+    difficult_class_probability,
+    tokenize_lcp_input,
+)
+
+
+@dataclass
+class PredictionResult:
+    complexity_level: str
+    level_id: int
+    level_probs: dict[str, float]
+    difficult_class_prob: float  # P(Hard)+P(Very Hard) from 5-class softmax (auxiliary)
+    reason: str | None
+    reason_id: int | None
+    reason_probs: dict[str, float] | None
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+class ComplexityPredictor:
+    """Load once, call predict() from your UI or API."""
+
+    def __init__(
+        self,
+        model,
+        tokenizer,
+        meta: dict,
+        max_length: int = 192,
+        device: str | None = None,
+    ):
+        self.model = model
+        self.tokenizer = tokenizer
+        self.meta = meta
+        self.model_key = meta["model_key"]
+        self.encoding = meta.get("encoding", "span_mark")
+        self.max_length = max_length
+        self.device = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu"))
+        self.model.to(self.device)
+        self.model.eval()
+
+    @classmethod
+    def from_checkpoint(cls, checkpoint_path: str | Path, device: str | None = None) -> "ComplexityPredictor":
+        dev = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu"))
+        model, tokenizer, meta = load_model_from_checkpoint(checkpoint_path, dev)
+        return cls(model, tokenizer, meta, device=str(dev))
+
+    @classmethod
+    def load(cls, export_dir: str | Path, device: str | None = None) -> "ComplexityPredictor":
+        export_dir = Path(export_dir)
+        with open(export_dir / "config.json") as f:
+            config = json.load(f)
+
+        from transformers import AutoTokenizer
+        from two_head_model import TwoHeadModel
+
+        model_key = config["model_key"]
+        tokenizer = AutoTokenizer.from_pretrained(export_dir / "tokenizer")
+        model = TwoHeadModel(
+            model_name=MODELS[model_key],
+            pooling_mode=config.get("pooling", "span"),
+            use_linguistic_features=config.get("use_linguistic_features", False),
+        )
+        model.encoder.resize_token_embeddings(len(tokenizer))
+        weights = torch.load(export_dir / "model_weights.pt", map_location="cpu", weights_only=True)
+        model.load_state_dict(weights)
+        model.set_tgt_token_ids(config["tgt_id"], config.get("tgt_end_id"))
+
+        meta = {
+            "model_key": model_key,
+            "encoding": config.get("encoding", "span_mark"),
+            "pooling": config.get("pooling", "span"),
+            "use_linguistic_features": config.get("use_linguistic_features", False),
+            "tgt_id": config["tgt_id"],
+            "tgt_end_id": config.get("tgt_end_id"),
+            "feat_mean": np.array(config["feat_mean"]) if config.get("feat_mean") else None,
+            "feat_std": np.array(config["feat_std"]) if config.get("feat_std") else None,
+        }
+        return cls(model, tokenizer, meta, max_length=config.get("max_length", 192), device=device)
+
+    def save(self, export_dir: str | Path) -> Path:
+        export_dir = Path(export_dir)
+        export_dir.mkdir(parents=True, exist_ok=True)
+
+        config = {
+            "model_key": self.model_key,
+            "model_name": MODELS[self.model_key],
+            "tgt_id": self.meta.get("tgt_id"),
+            "tgt_end_id": self.meta.get("tgt_end_id"),
+            "encoding": self.encoding,
+            "pooling": self.meta.get("pooling", "span"),
+            "use_linguistic_features": self.meta.get("use_linguistic_features", False),
+            "feat_mean": self.meta.get("feat_mean").tolist() if self.meta.get("feat_mean") is not None else None,
+            "feat_std": self.meta.get("feat_std").tolist() if self.meta.get("feat_std") is not None else None,
+            "max_length": self.max_length,
+            "level_labels": LEVEL_ORDER,
+            "reason_labels": REASON_ORDER,
+            "hard_levels": list(HARD_LEVELS),
+            "inference_rule": "Reason is returned only when predicted level is Hard or Very Hard.",
+            "task": "5_class_complexity_prediction",
+            "outputs": "discrete_level_and_optional_reason_class",
+        }
+        with open(export_dir / "config.json", "w") as f:
+            json.dump(config, f, indent=2)
+
+        torch.save(self.model.state_dict(), export_dir / "model_weights.pt")
+        self.tokenizer.save_pretrained(export_dir / "tokenizer")
+        return export_dir
+
+    def save_pickle(self, pickle_path: str | Path) -> Path:
+        pickle_path = Path(pickle_path)
+        pickle_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(pickle_path, "wb") as f:
+            pickle.dump(self, f)
+        return pickle_path
+
+    @classmethod
+    def load_pickle(cls, pickle_path: str | Path, device: str | None = None) -> "ComplexityPredictor":
+        with open(pickle_path, "rb") as f:
+            obj = pickle.load(f)
+        if not isinstance(obj, cls):
+            raise TypeError(f"Expected ComplexityPredictor, got {type(obj)}")
+        if device:
+            obj.device = torch.device(device)
+            obj.model.to(obj.device)
+        obj.model.eval()
+        return obj
+
+    @torch.no_grad()
+    def predict(self, sentence: str, target_word: str, corpus: str | None = None) -> PredictionResult:
+        enc = tokenize_lcp_input(
+            self.tokenizer,
+            sentence,
+            target_word,
+            encoding=self.encoding,
+            max_length=self.max_length,
+            corpus=corpus,
+        )
+        enc = {k: v.to(self.device) for k, v in enc.items()}
+
+        kwargs = {"input_ids": enc["input_ids"], "attention_mask": enc["attention_mask"]}
+        if self.meta.get("use_linguistic_features"):
+            feats = extract_linguistic_features(sentence, target_word)
+            if self.meta.get("feat_mean") is not None:
+                feats = (feats - self.meta["feat_mean"]) / self.meta["feat_std"]
+            kwargs["linguistic_features"] = torch.tensor(feats, dtype=torch.float32).unsqueeze(0).to(self.device)
+
+        out = self.model(**kwargs)
+        level_probs_t = torch.softmax(out.level_logits, dim=-1)[0].cpu()
+        reason_probs_t = torch.softmax(out.reason_logits, dim=-1)[0].cpu()
+
+        level_id = int(level_probs_t.argmax())
+        level_label = ID_TO_LEVEL[level_id]
+        level_probs = {ID_TO_LEVEL[i]: float(level_probs_t[i]) for i in range(5)}
+        diff_prob = difficult_class_probability(level_probs_t)
+
+        reason_label = None
+        reason_id = None
+        reason_probs = None
+        if level_label in HARD_LEVELS:
+            reason_id = int(reason_probs_t.argmax())
+            reason_label = ID_TO_REASON[reason_id]
+            reason_probs = {ID_TO_REASON[i]: float(reason_probs_t[i]) for i in range(3)}
+
+        return PredictionResult(
+            complexity_level=level_label,
+            level_id=level_id,
+            level_probs=level_probs,
+            difficult_class_prob=diff_prob,
+            reason=reason_label,
+            reason_id=reason_id,
+            reason_probs=reason_probs,
+        )
+
+    def predict_batch(self, items: list[dict[str, str]]) -> list[PredictionResult]:
+        return [self.predict(item["sentence"], item["target_word"], item.get("corpus")) for item in items]

src/reason_interventions.py

@@ -0,0 +1,132 @@
+"""
+Sentence edits for reason faithfulness tests (UI + 07_faithfulness.py).
+
+Each reason type has one targeted edit; re-predict after edit to see if hardness drops.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+from syntax_complexity import find_hardest_span
+from utils import REASON_ORDER
+
+
+@dataclass
+class InterventionResult:
+    reason: str
+    original_sentence: str
+    edited_sentence: str
+    edit_method: str  # "openai", "gemini", or "rule"
+
+
+def _call_llm(prompt: str) -> tuple[str | None, str]:
+    if os.environ.get("OPENAI_API_KEY"):
+        try:
+            from openai import OpenAI
+
+            client = OpenAI()
+            resp = client.chat.completions.create(
+                model=os.environ.get("FAITHFULNESS_LLM", "gpt-4o-mini"),
+                messages=[{"role": "user", "content": prompt}],
+                temperature=0,
+                max_tokens=256,
+            )
+            return resp.choices[0].message.content.strip(), "openai"
+        except Exception:
+            pass
+    if os.environ.get("GOOGLE_API_KEY"):
+        try:
+            import google.generativeai as genai
+
+            genai.configure(api_key=os.environ["GOOGLE_API_KEY"])
+            model = genai.GenerativeModel(os.environ.get("FAITHFULNESS_GEMINI", "gemini-1.5-flash"))
+            return model.generate_content(prompt).text.strip(), "gemini"
+        except Exception:
+            pass
+    return None, "rule"
+
+
+def _rule_synonym_swap(sentence: str, target_word: str) -> str:
+    simple = {
+        "frankincense": "incense",
+        "astonishment": "surprise",
+        "partiality": "bias",
+        "dominion": "rule",
+        "assemblies": "meetings",
+        "ewe": "sheep",
+        "scribe": "writer",
+        "inflammation": "swelling",
+        "treaty": "deal",
+    }
+    rep = simple.get(target_word.lower(), target_word)
+    if target_word in sentence:
+        return sentence.replace(target_word, rep, 1)
+    return sentence
+
+
+def edit_lexical_rarity(sentence: str, target_word: str, use_llm: bool = True) -> InterventionResult:
+    prompt = (
+        f"Replace ONLY the word '{target_word}' in this sentence with a common, simple synonym "
+        f"of the same meaning and part of speech. Change nothing else.\n"
+        f"Sentence: {sentence}\n"
+        f"Output only the edited sentence."
+    )
+    method = "rule"
+    edited = _rule_synonym_swap(sentence, target_word)
+    if use_llm:
+        llm_out, method = _call_llm(prompt)
+        if llm_out:
+            edited = llm_out
+    return InterventionResult("Lexical Rarity", sentence, edited, method)
+
+
+def edit_contextual_ambiguity(sentence: str, target_word: str, use_llm: bool = True) -> InterventionResult:
+    prompt = (
+        f"Add at most 3 words to this sentence to make the meaning of '{target_word}' clear. "
+        f"Do NOT change or remove '{target_word}'.\n"
+        f"Sentence: {sentence}\n"
+        f"Output only the edited sentence."
+    )
+    method = "rule"
+    edited = f"{sentence} (meaning: {target_word})"
+    if use_llm:
+        llm_out, method = _call_llm(prompt)
+        if llm_out:
+            edited = llm_out
+    return InterventionResult("Contextual Ambiguity", sentence, edited, method)
+
+
+def edit_syntactic_complexity(sentence: str, target_word: str, use_llm: bool = True) -> InterventionResult:
+    span, start, end = find_hardest_span(sentence)
+    method = "rule"
+    simplified = span
+    if use_llm and span.strip() != sentence.strip():
+        prompt = (
+            f"Simplify ONLY this phrase to plain English. Keep all key words including '{target_word}'.\n"
+            f"Phrase: {span}\n"
+            f"Output only the simplified phrase."
+        )
+        llm_out, method = _call_llm(prompt)
+        if llm_out:
+            simplified = llm_out
+    edited = sentence[:start] + simplified + sentence[end:]
+    return InterventionResult("Syntactic Complexity", sentence, edited, method)
+
+
+EDIT_FNS = {
+    "Lexical Rarity": edit_lexical_rarity,
+    "Contextual Ambiguity": edit_contextual_ambiguity,
+    "Syntactic Complexity": edit_syntactic_complexity,
+}
+
+
+def apply_intervention(reason: str, sentence: str, target_word: str, use_llm: bool = True) -> InterventionResult:
+    if reason not in EDIT_FNS:
+        raise ValueError(f"Unknown reason: {reason}. Choose from {REASON_ORDER}")
+    return EDIT_FNS[reason](sentence, target_word, use_llm=use_llm)
+
+
+def apply_all_interventions(sentence: str, target_word: str, use_llm: bool = True) -> list[InterventionResult]:
+    return [fn(sentence, target_word, use_llm=use_llm) for fn in EDIT_FNS.values()]

src/syntax_complexity.py

@@ -0,0 +1,163 @@
+"""
+Structural syntax analysis for word complexity project.
+
+Provides:
+  - analyze_syntax(sentence): structural metrics for a sentence
+  - find_hardest_span(sentence): the most syntactically tangled span (for faithfulness edits)
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any
+
+import spacy
+from spacy.tokens import Doc, Span, Token
+
+
+@lru_cache(maxsize=1)
+def _load_nlp():
+    try:
+        return spacy.load("en_core_web_sm")
+    except OSError as exc:
+        raise RuntimeError(
+            "spaCy model 'en_core_web_sm' not found. Run: python -m spacy download en_core_web_sm"
+        ) from exc
+
+
+@dataclass
+class SyntaxMetrics:
+    token_count: int
+    max_tree_depth: int
+    avg_dependency_length: float
+    subordinate_clause_count: int
+    passive_voice_count: int
+    punctuation_density: float
+    complexity_score: float
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "token_count": self.token_count,
+            "max_tree_depth": self.max_tree_depth,
+            "avg_dependency_length": self.avg_dependency_length,
+            "subordinate_clause_count": self.subordinate_clause_count,
+            "passive_voice_count": self.passive_voice_count,
+            "punctuation_density": self.punctuation_density,
+            "complexity_score": self.complexity_score,
+        }
+
+
+def _token_depth(token: Token) -> int:
+    depth = 0
+    while token.head != token:
+        depth += 1
+        token = token.head
+    return depth
+
+
+def _subtree_depth(token: Token) -> int:
+    children = list(token.subtree)
+    if not children:
+        return 0
+    return 1 + max(_subtree_depth(child) for child in token.children) if token.children else 1
+
+
+def _dependency_length(token: Token) -> int:
+    if token.head == token:
+        return 0
+    return abs(token.i - token.head.i)
+
+
+def _count_subordinate_clauses(doc: Doc) -> int:
+    markers = {"ccomp", "xcomp", "advcl", "relcl", "acl"}
+    return sum(1 for tok in doc if tok.dep_ in markers)
+
+
+def _count_passive(doc: Doc) -> int:
+    return sum(
+        1
+        for tok in doc
+        if tok.dep_ in {"nsubjpass", "auxpass"} or (tok.tag_ == "VBN" and tok.dep_ == "ROOT")
+    )
+
+
+def analyze_syntax(sentence: str) -> SyntaxMetrics:
+    """Return structural complexity metrics for a sentence."""
+    nlp = _load_nlp()
+    doc = nlp(sentence or "")
+    tokens = [t for t in doc if not t.is_space]
+    if not tokens:
+        return SyntaxMetrics(0, 0, 0.0, 0, 0, 0.0, 0.0)
+
+    depths = [_token_depth(t) for t in tokens]
+    dep_lengths = [_dependency_length(t) for t in tokens]
+    max_depth = max(depths)
+    avg_dep = sum(dep_lengths) / len(dep_lengths)
+    sub_clauses = _count_subordinate_clauses(doc)
+    passive = _count_passive(doc)
+    punct = len(re.findall(r"[,;:()\[\]{}\"']", sentence))
+    punct_density = punct / max(len(sentence), 1)
+
+    complexity_score = (
+        0.35 * max_depth
+        + 0.25 * avg_dep
+        + 0.20 * sub_clauses
+        + 0.10 * passive
+        + 0.10 * punct_density * 20
+        + 0.05 * (len(tokens) / 30.0)
+    )
+
+    return SyntaxMetrics(
+        token_count=len(tokens),
+        max_tree_depth=max_depth,
+        avg_dependency_length=avg_dep,
+        subordinate_clause_count=sub_clauses,
+        passive_voice_count=passive,
+        punctuation_density=punct_density,
+        complexity_score=complexity_score,
+    )
+
+
+def _span_complexity(span: Span) -> float:
+    text = span.text.strip()
+    if not text:
+        return 0.0
+    metrics = analyze_syntax(text)
+    return metrics.complexity_score
+
+
+def find_hardest_span(sentence: str, min_tokens: int = 3) -> tuple[str, int, int]:
+    """
+    Find the most syntactically tangled contiguous span in the sentence.
+
+    Returns (span_text, start_char, end_char).
+    """
+    nlp = _load_nlp()
+    doc = nlp(sentence or "")
+    tokens = [t for t in doc if not t.is_space]
+    if len(tokens) < min_tokens:
+        return sentence, 0, len(sentence)
+
+    best_score = -1.0
+    best_span: Span | None = None
+
+    for start in range(len(tokens)):
+        for end in range(start + min_tokens, min(start + 25, len(tokens)) + 1):
+            span = doc[tokens[start].i : tokens[end - 1].i + 1]
+            score = _span_complexity(span)
+            if score > best_score:
+                best_score = score
+                best_span = span
+
+    if best_span is None:
+        return sentence, 0, len(sentence)
+
+    return best_span.text, best_span.start_char, best_span.end_char
+
+
+def syntax_signal_high(sentence: str, percentile_threshold: float = 0.6) -> bool:
+    """Heuristic: sentence is structurally complex relative to typical text."""
+    score = analyze_syntax(sentence).complexity_score
+    return score >= percentile_threshold * 8.0

src/two_head_model.py

@@ -0,0 +1,231 @@
+"""
+Stage E — Two-head model for 5-class word complexity + 3-class reason.
+
+No regression head. Outputs are discrete levels (Very Easy → Very Hard) and,
+when Hard/Very Hard, one of three explainable difficulty causes.
+
+Novel architecture (publish contribution): one encoder, dual heads, masked
+reason loss, target-span pooling for word-in-context LCP.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import AutoModel, AutoTokenizer, PreTrainedTokenizer
+
+from linguistic_features import N_LINGUISTIC_FEATURES
+from utils import (
+    MODELS,
+    POOLING_CLS_CONCAT,
+    POOLING_CLS_ONLY,
+    POOLING_SPAN,
+    POOLING_TGT_MARKER,
+    TGT_END_TOKEN,
+    TGT_TOKEN,
+)
+
+
+@dataclass
+class ModelOutput:
+    level_logits: torch.Tensor
+    reason_logits: torch.Tensor
+    pooled: torch.Tensor
+
+
+class TwoHeadModel(nn.Module):
+    """Shared encoder with level (5-class) and reason (3-class) heads."""
+
+    def __init__(
+        self,
+        model_name: str = MODELS["deberta"],
+        hidden_dropout: float = 0.1,
+        pooling_mode: str = POOLING_SPAN,
+        use_linguistic_features: bool = False,
+        # Legacy alias: cls_concat maps to cls_concat pooling
+        use_cls_concat: bool | None = None,
+    ):
+        super().__init__()
+        if use_cls_concat is not None:
+            pooling_mode = POOLING_CLS_CONCAT if use_cls_concat else POOLING_SPAN
+
+        self.encoder = AutoModel.from_pretrained(model_name, torch_dtype=torch.float32)
+        hidden = self.encoder.config.hidden_size
+        self.pooling_mode = pooling_mode
+        self.use_linguistic_features = use_linguistic_features
+        self.model_name = model_name
+
+        extra = N_LINGUISTIC_FEATURES if use_linguistic_features else 0
+        if pooling_mode == POOLING_CLS_CONCAT:
+            head_in = hidden * 2 + extra
+        else:
+            head_in = hidden + extra
+
+        self.dropout = nn.Dropout(hidden_dropout)
+        self.level_head = nn.Linear(head_in, 5)
+        self.reason_head = nn.Linear(head_in, 3)
+
+        self._tgt_id: Optional[int] = None
+        self._tgt_end_id: Optional[int] = None
+
+    def set_tgt_token_ids(self, tgt_id: int, tgt_end_id: int | None = None) -> None:
+        self._tgt_id = tgt_id
+        self._tgt_end_id = tgt_end_id
+
+    def set_tgt_token_id(self, tgt_id: int) -> None:
+        self.set_tgt_token_ids(tgt_id, self._tgt_end_id)
+
+    def _span_pool(self, last_hidden: torch.Tensor, input_ids: torch.Tensor) -> torch.Tensor:
+        """Mean-pool token hidden states inside [TGT] ... [/TGT] (target word subwords)."""
+        batch_size = input_ids.size(0)
+        pooled = []
+        cls_vec = last_hidden[:, 0, :]
+
+        for i in range(batch_size):
+            ids = input_ids[i]
+            open_pos = (ids == self._tgt_id).nonzero(as_tuple=True)[0] if self._tgt_id is not None else None
+            if open_pos is None or len(open_pos) == 0:
+                pooled.append(cls_vec[i])
+                continue
+
+            start = int(open_pos[0].item()) + 1
+            end = len(ids)
+            if self._tgt_end_id is not None:
+                close_pos = (ids == self._tgt_end_id).nonzero(as_tuple=True)[0]
+                close_after = close_pos[close_pos > open_pos[0]]
+                if len(close_after) > 0:
+                    end = int(close_after[0].item())
+
+            span_idx = [j for j in range(start, end) if ids[j].item() != 0]  # skip pad
+            if not span_idx:
+                pooled.append(last_hidden[i, int(open_pos[0].item()), :])
+            else:
+                vecs = last_hidden[i, span_idx, :]
+                pooled.append(vecs.mean(dim=0))
+
+        return torch.stack(pooled, dim=0)
+
+    def _first_marker_pool(self, last_hidden: torch.Tensor, input_ids: torch.Tensor) -> torch.Tensor:
+        cls_vec = last_hidden[:, 0, :]
+        if self._tgt_id is None:
+            return cls_vec
+
+        batch_size = input_ids.size(0)
+        vecs = []
+        for i in range(batch_size):
+            positions = (input_ids[i] == self._tgt_id).nonzero(as_tuple=True)[0]
+            if len(positions) > 0:
+                vecs.append(last_hidden[i, positions[0], :])
+            else:
+                vecs.append(cls_vec[i])
+        return torch.stack(vecs, dim=0)
+
+    def _pool(self, last_hidden: torch.Tensor, input_ids: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor:
+        cls_vec = last_hidden[:, 0, :]
+
+        if self.pooling_mode == POOLING_CLS_ONLY:
+            return cls_vec
+        if self.pooling_mode == POOLING_SPAN:
+            return self._span_pool(last_hidden, input_ids)
+        if self.pooling_mode == POOLING_TGT_MARKER:
+            return self._first_marker_pool(last_hidden, input_ids)
+        if self.pooling_mode == POOLING_CLS_CONCAT:
+            tgt_vec = self._first_marker_pool(last_hidden, input_ids)
+            return torch.cat([cls_vec, tgt_vec], dim=-1)
+
+        return self._span_pool(last_hidden, input_ids)
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        attention_mask: torch.Tensor,
+        linguistic_features: Optional[torch.Tensor] = None,
+    ) -> ModelOutput:
+        outputs = self.encoder(input_ids=input_ids, attention_mask=attention_mask)
+        pooled = self._pool(outputs.last_hidden_state, input_ids, attention_mask)
+
+        if self.use_linguistic_features:
+            if linguistic_features is None:
+                raise ValueError("linguistic_features required when use_linguistic_features=True")
+            pooled = torch.cat([pooled, linguistic_features], dim=-1)
+
+        pooled = self.dropout(pooled)
+        # Encoder may run in float16 on GPU; classification heads stay float32.
+        pooled = pooled.to(dtype=self.level_head.weight.dtype)
+
+        return ModelOutput(
+            level_logits=self.level_head(pooled),
+            reason_logits=self.reason_head(pooled),
+            pooled=pooled,
+        )
+
+
+def add_tgt_tokens(tokenizer: PreTrainedTokenizer) -> tuple[int, int]:
+    """Add [TGT] and [/TGT] special tokens; return their ids."""
+    special = {"additional_special_tokens": [TGT_TOKEN, TGT_END_TOKEN]}
+    tokenizer.add_special_tokens(special)
+    return (
+        tokenizer.convert_tokens_to_ids(TGT_TOKEN),
+        tokenizer.convert_tokens_to_ids(TGT_END_TOKEN),
+    )
+
+
+def add_tgt_token(tokenizer: PreTrainedTokenizer) -> int:
+    """Backward-compatible: add markers and return open [TGT] id."""
+    open_id, _ = add_tgt_tokens(tokenizer)
+    return open_id
+
+
+def compute_loss(
+    level_logits: torch.Tensor,
+    reason_logits: torch.Tensor,
+    level_ids: torch.Tensor,
+    reason_ids: torch.Tensor,
+    reason_mask: torch.Tensor,
+    reason_class_weights: Optional[torch.Tensor] = None,
+    lambda_reason: float = 1.0,
+    level_only: bool = False,
+) -> tuple[torch.Tensor, dict]:
+    level_loss = F.cross_entropy(level_logits, level_ids)
+
+    if level_only:
+        return level_loss, {"level_loss": level_loss.item(), "reason_loss": 0.0, "total_loss": level_loss.item()}
+
+    per_row = F.cross_entropy(
+        reason_logits,
+        reason_ids,
+        weight=reason_class_weights,
+        reduction="none",
+    )
+    masked = per_row * reason_mask
+    denom = reason_mask.sum().clamp(min=1.0)
+    reason_loss = masked.sum() / denom
+    total = level_loss + lambda_reason * reason_loss
+
+    return total, {
+        "level_loss": level_loss.item(),
+        "reason_loss": reason_loss.item(),
+        "total_loss": total.item(),
+    }
+
+
+def load_tokenizer(model_key: str = "deberta") -> PreTrainedTokenizer:
+    return AutoTokenizer.from_pretrained(MODELS[model_key])
+
+
+def build_model(
+    model_key: str = "deberta",
+    pooling_mode: str = POOLING_SPAN,
+    use_linguistic_features: bool = False,
+    use_cls_concat: bool | None = None,
+) -> TwoHeadModel:
+    return TwoHeadModel(
+        model_name=MODELS[model_key],
+        pooling_mode=pooling_mode,
+        use_linguistic_features=use_linguistic_features,
+        use_cls_concat=use_cls_concat,
+    )

src/ui/efficiency_panel.py

@@ -0,0 +1,69 @@
+"""
+Compare local model vs AI API: latency and estimated cost per query.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+LEVEL_PROMPT = """Rate the complexity of the target word in context.
+Levels: Very Easy, Easy, Medium, Hard, Very Hard.
+Sentence: {sentence}
+Target word: {target_word}
+Reply with only the level label."""
+
+# USD per 1k queries (rough; matches 08_efficiency.py defaults)
+COST_PER_1K = {
+    "openai_gpt4o_mini": 0.15,
+    "gemini_flash": 0.075,
+}
+
+# Typical latency when API is used (ms) — used when no live API call
+TYPICAL_AI_LATENCY_MS = {
+    "openai_gpt4o_mini": 1200.0,
+    "gemini_flash": 900.0,
+}
+
+
+@dataclass
+class QueryComparison:
+    local_latency_ms: float
+    ai_latency_ms: float
+    ai_provider: str
+    ai_cost_usd: float
+    ai_cost_per_1k_usd: float
+    local_cost_usd: float
+    speedup_factor: float
+    prompt_tokens_est: int
+
+
+def _estimate_tokens(text: str) -> int:
+    return max(1, len(text.split()) + len(text) // 4)
+
+
+def estimate_ai_cost(sentence: str, target_word: str, provider: str = "openai_gpt4o_mini") -> float:
+    prompt = LEVEL_PROMPT.format(sentence=sentence, target_word=target_word)
+    tokens = _estimate_tokens(prompt) + 10  # short reply
+    # gpt-4o-mini ballpark: ~$0.15/1M input + $0.60/1M output — ~$0.0002 per simple call
+    cost_per_1k = COST_PER_1K.get(provider, 0.15)
+    return (cost_per_1k / 1000.0) * (tokens / 200.0)
+
+
+def compare_query(sentence: str, target_word: str, local_latency_ms: float) -> QueryComparison:
+    provider = "openai_gpt4o_mini"
+    prompt = LEVEL_PROMPT.format(sentence=sentence, target_word=target_word)
+    tokens = _estimate_tokens(prompt)
+    ai_cost = estimate_ai_cost(sentence, target_word, provider)
+    ai_latency = TYPICAL_AI_LATENCY_MS[provider]
+    local_cost = 0.0
+    speedup = ai_latency / max(local_latency_ms, 0.1)
+    return QueryComparison(
+        local_latency_ms=local_latency_ms,
+        ai_latency_ms=ai_latency,
+        ai_provider="GPT-4o-mini (estimated)",
+        ai_cost_usd=ai_cost,
+        ai_cost_per_1k_usd=COST_PER_1K[provider],
+        local_cost_usd=local_cost,
+        speedup_factor=speedup,
+        prompt_tokens_est=tokens,
+    )

src/utils.py

@@ -0,0 +1,135 @@
+"""Shared constants and helpers for the word complexity project."""
+
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent
+DATA_DIR = PROJECT_ROOT / "data"
+OUTPUT_DIR = PROJECT_ROOT / "outputs"
+CHECKPOINT_DIR = PROJECT_ROOT / "checkpoints"
+EXPORT_DIR = PROJECT_ROOT / "exported_models"
+
+LEVEL_ORDER = ["Very Easy", "Easy", "Medium", "Hard", "Very Hard"]
+REASON_ORDER = ["Lexical Rarity", "Contextual Ambiguity", "Syntactic Complexity"]
+HARD_LEVELS = {"Hard", "Very Hard"}
+
+LEVEL_TO_ID = {level: idx for idx, level in enumerate(LEVEL_ORDER)}
+ID_TO_LEVEL = {idx: level for level, idx in LEVEL_TO_ID.items()}
+REASON_TO_ID = {reason: idx for idx, reason in enumerate(REASON_ORDER)}
+ID_TO_REASON = {idx: reason for reason, idx in REASON_TO_ID.items()}
+NONE_REASON_ID = -1
+
+TGT_TOKEN = "[TGT]"
+TGT_END_TOKEN = "[/TGT]"
+
+# Input encoding strategies (SemEval / ABSA literature)
+ENCODING_SPAN_MARK = "span_mark"          # [TGT] word [/TGT] inside sentence
+ENCODING_PAIR_CAMBRIDGE = "pair_cambridge"  # [CLS] target [SEP] sentence
+ENCODING_PAIR_CONTEXT = "pair_context"      # sentence [SEP] target (SemEval common)
+
+# Pooling strategies for target-focused readout
+POOLING_SPAN = "span"              # mean of tokens inside marked span (default)
+POOLING_CLS_CONCAT = "cls_concat"  # legacy: [CLS] + first [TGT]
+POOLING_TGT_MARKER = "tgt_marker"  # first [TGT] token only
+POOLING_CLS_ONLY = "cls_only"      # [CLS] only (for pair encoding)
+
+MODELS = {
+    "deberta": "microsoft/deberta-v3-base",
+    "distilbert": "distilbert-base-uncased",
+    "roberta": "roberta-base",
+}
+
+MERGE_KEYS = ["id", "sentence", "target_word", "complexity_level"]
+
+
+def level_id(level: str) -> int:
+    return LEVEL_TO_ID[level]
+
+
+def reason_id(reason: str) -> int:
+    if reason in ("NONE", None) or (isinstance(reason, float) and str(reason) == "nan"):
+        return NONE_REASON_ID
+    return REASON_TO_ID[reason]
+
+
+def is_hard_level(level: str) -> bool:
+    return level in HARD_LEVELS
+
+
+def wrap_target_word(sentence: str, target_word: str) -> str:
+    """Wrap target word with open/close span markers (ABSA aspect-marker style)."""
+    if not target_word or target_word not in sentence:
+        return sentence
+    marked = f"{TGT_TOKEN} {target_word} {TGT_END_TOKEN}"
+    return sentence.replace(target_word, marked, 1)
+
+
+def build_model_input(
+    sentence: str,
+    target_word: str,
+    encoding: str = ENCODING_SPAN_MARK,
+    corpus: str | None = None,
+) -> str | tuple[str, str]:
+    """
+    Build tokenizer input for LCP.
+
+    Returns a string for span marking, or (text_a, text_b) for pair encodings.
+    """
+    sentence = str(sentence)
+    target_word = str(target_word)
+
+    if encoding == ENCODING_SPAN_MARK:
+        return wrap_target_word(sentence, target_word)
+
+    if encoding == ENCODING_PAIR_CAMBRIDGE:
+        return target_word, sentence
+
+    if encoding == ENCODING_PAIR_CONTEXT:
+        if corpus:
+            return f"{corpus} {target_word}", sentence
+        return sentence, target_word
+
+    raise ValueError(f"Unknown encoding: {encoding}")
+
+
+def difficult_class_probability(level_probs) -> float:
+    """P(Hard) + P(Very Hard) from the 5-class softmax — auxiliary only, not a regression score."""
+    if hasattr(level_probs, "tolist"):
+        level_probs = level_probs.tolist()
+    if isinstance(level_probs, dict):
+        return float(level_probs.get("Hard", 0) + level_probs.get("Very Hard", 0))
+    return float(level_probs[3] + level_probs[4])
+
+
+def ensure_dirs() -> None:
+    DATA_DIR.mkdir(parents=True, exist_ok=True)
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True)
+    EXPORT_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def tokenize_lcp_input(
+    tokenizer,
+    sentence: str,
+    target_word: str,
+    encoding: str = ENCODING_SPAN_MARK,
+    max_length: int = 192,
+    corpus: str | None = None,
+):
+    """Tokenize a (sentence, target_word) pair for the LCP model."""
+    built = build_model_input(sentence, target_word, encoding=encoding, corpus=corpus)
+    if isinstance(built, tuple):
+        return tokenizer(
+            built[0],
+            built[1],
+            truncation=True,
+            max_length=max_length,
+            padding="max_length",
+            return_tensors="pt",
+        )
+    return tokenizer(
+        built,
+        truncation=True,
+        max_length=max_length,
+        padding="max_length",
+        return_tensors="pt",
+    )