Sync from GitHub 2025-12-17T12:18:53Z

.dockerignore

@@ -0,0 +1,27 @@
+.git
+.gitignore
+.env
+.env.*
+.venv
+venv
+__pycache__
+*.pyc
+*.pyo
+.pytest_cache
+.mypy_cache
+.ruff_cache
+.model_cache
+.cache
+node_modules
+frontend/node_modules
+frontend/.next
+frontend/.turbo
+frontend/.vercel
+runs
+logs
+*.log
+*.tmp
+*.swp
+*.swo
+*.orig
+*.DS_Store

.env.example

@@ -0,0 +1,16 @@
+# Application
+APP_ENV=local
+LOG_LEVEL=INFO
+CONFIG_PATH=configs/config.yaml
+USER_AGENT=llm-recommendation-engine/0.1 (+https://example.com)
+START_URL=https://www.shl.com/products/product-catalog/
+MAX_CONCURRENCY=2
+REQUEST_DELAY_SECONDS=1.5
+JITTER_SECONDS=0.5
+MAX_RETRIES=3
+ALLOW_ROBOTS_BYPASS=0
+
+# External services / secrets
+OPENAI_API_KEY=replace_me
+VECTOR_DB_URL=replace_me
+TRACING_ENDPOINT=replace_me

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.github/workflows/push_to_hf_space.yml

@@ -0,0 +1,38 @@
+name: Push to Hugging Face Space
+
+on:
+  push:
+    branches: [ "main" ]
+
+jobs:
+  sync-to-hf:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Push to HF Space
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          git config --global user.email "ci@github"
+          git config --global user.name "github-actions"
+
+          # Clone the Space repo
+          git clone https://AgamP:$HF_TOKEN@huggingface.co/spaces/AgamP/llm_recommendation_backend hf_space
+          
+          # Replace Space contents with GitHub repo contents (except .git)
+    
+          rsync -av --delete \
+            --exclude ".git" \
+            --exclude "hf_space" \
+            --exclude "*.pdf" \
+            ./ hf_space/
+
+
+          cd hf_space
+          git add -A
+          git commit -m "Sync from GitHub $(date -u +'%Y-%m-%dT%H:%M:%SZ')" || echo "No changes"
+          git push

.gitignore

@@ -0,0 +1,20 @@
+.venv/
+__pycache__/
+.pytest_cache/
+*.sqlite
+*.db
+logs/
+data/crawler.db
+data/catalog.parquet
+data/catalog.jsonl
+playwright-report/
+playwright/.cache/
+node_modules/
+venv/
+runs/
+frontend/.next/
+frontend/node_modules/
+.model_cache/
+data
+data/
+models/reranker_crossenc/v0.1.0/

Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
+
+WORKDIR /app
+
+# System deps for numpy/faiss/scipy style builds; drop if wheels suffice
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends build-essential && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy source after deps to leverage Docker layer caching
+COPY . .
+
+EXPOSE 8000
+# Render sets PORT; default to 8000 for local use
+ENV PORT=8000
+CMD ["sh", "-c", "uvicorn agent.server:app --host 0.0.0.0 --port ${PORT} --workers 2"]

Makefile

@@ -0,0 +1,30 @@
+PYTHON ?= python3
+VENV ?= .venv
+ACTIVATE = . $(VENV)/bin/activate
+APP_NAME ?= llm-recommender
+
+.PHONY: setup install config-check lint test docker-build docker-run clean
+
+setup:
+	$(PYTHON) -m venv $(VENV)
+
+install: setup
+	$(ACTIVATE) && pip install --upgrade pip && pip install -r requirements.txt
+
+config-check:
+	$(ACTIVATE) && PYTHONPATH=. $(PYTHON) config.py --print
+
+lint:
+	@echo "Add linting tools (ruff/black/flake8) here"
+
+test:
+	$(ACTIVATE) && PYTHONPATH=. pytest
+
+docker-build:
+	docker build -t $(APP_NAME):dev .
+
+docker-run:
+	docker run --rm -it -p 8000:8000 -p 3000:3000 --env-file .env.example $(APP_NAME):dev
+
+clean:
+	rm -rf $(VENV) __pycache__ */__pycache__

README.md

@@ -1,12 +1,149 @@
----
-title: Llm Recommendation Backend
-emoji: 📚
-colorFrom: indigo
-colorTo: purple
-sdk: docker
-pinned: false
-license: apache-2.0
-short_description: 'fastapi backend for llm recomemndation engine for shl '
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# llm_recommendation_engine
+Recommendation engine for SHL's product catalogue with conversational agents
+
+## Quick commands (crawler + export + QA)
+- Install deps (and Playwright browser): `python -m pip install -r requirements.txt && python -m playwright install chromium`
+- Clean DB: `rm -f data/crawler.db`
+- Crawl (bypass robots if needed): `ALLOW_ROBOTS_BYPASS=1 python -m crawler.run --mode=crawl_all --max-discover=20`
+  - Drop `--max-discover` for full crawl.
+- Export dataset: `python -m crawler.run --mode=export --limit-export=20`
+  - Outputs: `data/catalog.parquet`, `data/catalog.jsonl`
+  - Drop `--limit-export` for full export.
+- QA checks: `python -m crawler.qa_checks data/catalog.jsonl > data/qa_summary.json`
+  - Summary JSON saved to `data/qa_summary.json`
+
+## What’s implemented
+- Playwright-based crawler with catalog pagination, detail fetch, and structured storage in SQLite.
+- Field extraction: url, name, description, test_type (+full), remote/adaptive flags, duration (minutes/hours), job_levels, languages, downloads.
+- Export to Parquet/JSONL plus QA summary script for downstream sanity checks.
+
+## Evaluation harness (Phase 2)
+- Catalog loader with canonical IDs: `python -m data.catalog_loader --input data/catalog.jsonl --output data/catalog_with_ids.jsonl`
+- Train loader + label resolution report: `python -m data.train_loader --catalog data/catalog.jsonl --train <train_file> --report data/label_resolution_report.json`
+- Run eval (dummy baseline): `python -m eval.run_eval --catalog data/catalog.jsonl --train <train_file> --recommender dummy_random`
+  - Run eval (BM25 baseline): `python -m eval.run_eval --catalog data/catalog.jsonl --train <train_file> --recommender bm25`
+  - Outputs run folder under `runs/<timestamp>_<recommender>/` with `metrics.json`, `per_query_results.jsonl`, `worst_queries.csv`, `label_resolution_report.json`
+- Compare runs: `python -m eval.compare_runs runs/<run_a> runs/<run_b>`
+
+Recommender interface lives in `recommenders/base.py`; a random baseline is in `recommenders/dummy_random.py`. Metrics (Recall@k, MRR@10) are in `eval/metrics.py`.
+
+## Label probing & backfill (improve label coverage)
+- Probe unmatched label URLs (after a label match run): `python -m scripts.probe_unmatched_labels --labels data/label_resolution_report.json --output reports/label_url_probe.csv` — classifies label URLs (valid detail vs 404/blocked).
+- Backfill valid label pages into DB: `python -m crawler.backfill_labels --probe-csv reports/label_url_probe.csv --allow-robots-bypass` — fetches & inserts DETAIL_PAGE_VALID URLs.
+- Re-export and rematch after backfill:
+  - `python -m crawler.run --mode=export`
+  - `python -m data.catalog_loader --input data/catalog.jsonl --output data/catalog_with_ids.jsonl`
+  - `python -m data.train_loader --catalog data/catalog.jsonl --train <train_file> --sheet "Train-Set" --report data/label_resolution_report.json`
+
+## Vector pipeline (semantic retrieval)
+- Build doc_text: `python -m data.document_builder --input data/catalog.jsonl --output data/catalog_docs.jsonl`
+- Generate embeddings: `python -m embeddings.generator --catalog data/catalog_docs.jsonl --model sentence-transformers/all-MiniLM-L6-v2 --output-dir data/embeddings`
+- Build FAISS index: `python -m retrieval.build_index --embeddings data/embeddings/embeddings.npy --ids data/embeddings/assessment_ids.json --index-path data/faiss_index/index.faiss`
+- Vector components:
+  - Model wrapper: `models/embedding_model.py`
+  - Index wrapper: `retrieval/vector_index.py`
+  - Index builder script: `retrieval/build_index.py`
+  - Vector recommender scaffold: `recommenders/vector_recommender.py` (wire with assessment_ids + index)
+
+## Hybrid retrieval (BM25 + vector with RRF)
+- Run hybrid eval: `python -m eval.run_eval --catalog data/catalog_docs.jsonl --train data/Gen_AI\ Dataset.xlsx --recommender hybrid_rrf --vector-index data/faiss_index/index.faiss --assessment-ids data/embeddings/assessment_ids.json --model sentence-transformers/all-MiniLM-L6-v2 --topn-candidates 200 --rrf-k 60`
+- Run hybrid + cross-encoder rerank: `python -m eval.run_eval --catalog data/catalog_docs.jsonl --train data/Gen_AI\ Dataset.xlsx --recommender hybrid_rrf_rerank --vector-index data/faiss_index/index.faiss --assessment-ids data/embeddings/assessment_ids.json --model sentence-transformers/all-MiniLM-L6-v2 --reranker-model cross-encoder/ms-marco-MiniLM-L-6-v2 --topn-candidates 200 --rrf-k 60`
+- Run hybrid + LGBM rerank: `python -m eval.run_eval --catalog data/catalog_docs.jsonl --train data/Gen_AI\ Dataset.xlsx --recommender hybrid_rrf_lgbm --vector-index data/faiss_index/index.faiss --assessment-ids data/embeddings/assessment_ids.json --model sentence-transformers/all-MiniLM-L6-v2 --topn-candidates 200 --rrf-k 60 --lgbm-model models/reranker/v0.1.0/lgbm_model.txt --lgbm-features models/reranker/v0.1.0/feature_schema.json`
+- Diagnostics (positives in top-N vs top-10): `python -m eval.diagnostic_topk --catalog data/catalog_docs.jsonl --train data/Gen_AI\ Dataset.xlsx --vector-index data/faiss_index/index.faiss --assessment-ids data/embeddings/assessment_ids.json --model sentence-transformers/all-MiniLM-L6-v2 --topn 200`
+- Run ablation (bm25/vector/hybrid across topN): `python -m scripts.run_ablation --catalog data/catalog_docs.jsonl --train data/Gen_AI\ Dataset.xlsx --vector-index data/faiss_index/index.faiss --assessment-ids data/embeddings/assessment_ids.json --model sentence-transformers/all-MiniLM-L6-v2 --topn-list 100,200,377`
+
+## Current findings & next steps
+- Candidate coverage is solved by top200; ranking is the bottleneck. Use union fusion + rerank.
+- Locked decisions:
+  - Candidate pool (train): top200
+  - Candidate pool (infer): top100–200
+  - Base retriever: hybrid (BM25 + vector), union fusion, dual-query (raw + rewritten).
+- Next: focus on reranking and constraint handling; no more embedding/model swaps.
+
+## Core pipeline (concise commands)
+
+### Build rich docs, embeddings, index (BGE)
+```bash
+python -m data.document_builder \
+  --input data/catalog.jsonl \
+  --output data/catalog_docs_rich.jsonl \
+  --variant rich \
+  --version v2_struct
+
+python -m embeddings.generator \
+  --catalog data/catalog_docs_rich.jsonl \
+  --model BAAI/bge-small-en-v1.5 \
+  --batch-size 32 \
+  --output-dir data/embeddings_bge
+
+python -m retrieval.build_index \
+  --embeddings data/embeddings_bge/embeddings.npy \
+  --ids data/embeddings_bge/assessment_ids.json \
+  --index-path data/faiss_index/index_bge.faiss
+```
+
+### Build vocab for query rewriter (optional, recommended)
+```bash
+python -m scripts.build_role_vocab \
+  --catalog data/catalog_docs_rich.jsonl \
+  --out data/catalog_role_vocab.json
+```
+
+### Evaluate hybrid + cross-encoder rerank (with rewriting and union fusion)
+```bash
+python -m eval.run_eval \
+  --catalog data/catalog_docs_rich.jsonl \
+  --train data/Gen_AI\ Dataset.xlsx \
+  --recommender hybrid_rrf_rerank \
+  --vector-index data/faiss_index/index_bge.faiss \
+  --assessment-ids data/embeddings_bge/assessment_ids.json \
+  --model BAAI/bge-small-en-v1.5 \
+  --reranker-model models/reranker_crossenc/v0.1.0 \
+  --topn-candidates 200 --rrf-k 60 \
+  --use-rewriter --vocab data/catalog_role_vocab.json \
+  --out-dir runs/$(date +%Y%m%d_%H%M%S)_hybrid_rrf_rerank_rewrite
+```
+
+### Candidate coverage (bm25 vs vector vs hybrid; grouped per query)
+```bash
+python -m scripts.candidate_coverage \
+  --catalog data/catalog_docs_rich.jsonl \
+  --train data/Gen_AI\ Dataset.xlsx \
+  --vector-index data/faiss_index/index_bge.faiss \
+  --assessment-ids data/embeddings_bge/assessment_ids.json \
+  --embedding-model BAAI/bge-small-en-v1.5 \
+  --topn 200 \
+  --use-rewriter --vocab data/catalog_role_vocab.json \
+  --out runs/candidate_coverage.jsonl
+
+python -m scripts.summarize_candidate_coverage \
+  --input runs/candidate_coverage.jsonl \
+  --out runs/candidate_coverage_stats.json
+```
+
+### Rewrite impact (optional)
+```bash
+python -m scripts.eval_rewrite_impact \
+  --catalog data/catalog_docs_rich.jsonl \
+  --train data/Gen_AI\ Dataset.xlsx \
+  --vector-index data/faiss_index/index_bge.faiss \
+  --assessment-ids data/embeddings_bge/assessment_ids.json \
+  --embedding-model BAAI/bge-small-en-v1.5 \
+  --topn 200 \
+  --vocab data/catalog_role_vocab.json \
+  --out runs/rewrite_impact.jsonl
+```
+
+## Frontend + backend (Next.js + FastAPI)
+
+Backend (FastAPI):
+- Start: `uvicorn agent.server:app --reload --port 8000`
+- Health: `GET /health`
+- Chat: `POST /chat` (returns compact top-10 + optional summary when verbose=true)
+- Recommend: `POST /recommend` with `{"query": "..."}` returns `{"recommended_assessments": [...]}` (top-10)
+
+Frontend (Next.js in `frontend/`):
+- Install deps: `cd frontend && npm install`
+- Dev: `npm run dev` (will start on port 3000; ensure backend is running on 8000 or set API base in UI)
+- Build/start: `npm run build && npm run start`
+- UI is at `http://localhost:3000/` (API base defaults to `http://localhost:8000`, editable in the UI)

agent/app.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+"""
+Lightweight agent harness (no LangChain server) to demonstrate the tool stack end-to-end.
+This keeps ranking deterministic; LLM can be plugged later for structured QueryPlan.
+"""
+
+import json
+from typing import Callable
+
+import pandas as pd
+
+from data.catalog_loader import load_catalog
+from recommenders.bm25 import BM25Recommender
+from recommenders.vector_recommender import VectorRecommender
+from retrieval.vector_index import VectorIndex
+from models.embedding_model import EmbeddingModel
+from rerankers.cross_encoder import CrossEncoderReranker
+
+from tools.query_plan_tool import build_query_plan
+from tools.retrieve_tool import retrieve_candidates
+from tools.rerank_tool import rerank_candidates
+from tools.constraints_tool import apply_constraints
+from tools.explain_tool import explain
+
+
+def load_resources():
+    df_catalog, _, _ = load_catalog("data/catalog_docs_rich.jsonl")
+    bm25 = BM25Recommender(df_catalog)
+    embed = EmbeddingModel("BAAI/bge-small-en-v1.5")
+    index = VectorIndex.load("data/faiss_index/index_bge.faiss")
+    with open("data/embeddings_bge/assessment_ids.json") as f:
+        ids = json.load(f)
+    vec = VectorRecommender(embed, index, df_catalog, ids, k_candidates=200)
+    return df_catalog, bm25, vec
+
+
+def make_catalog_lookup(df_catalog: pd.DataFrame) -> Callable[[str], dict]:
+    cat = df_catalog.set_index("assessment_id")
+
+    def lookup(aid: str) -> dict:
+        if aid in cat.index:
+            return cat.loc[aid].to_dict()
+        return {}
+
+    return lookup
+
+
+def run_query(user_text: str, vocab_path="data/catalog_role_vocab.json"):
+    vocab = json.load(open(vocab_path)) if vocab_path else {}
+    df_catalog, bm25, vec = load_resources()
+    catalog_lookup = make_catalog_lookup(df_catalog)
+
+    # Step 1: plan (deterministic rewriter for now; swap with LLM structured plan if desired)
+    plan = build_query_plan(user_text, vocab=vocab)
+
+    # Step 2: retrieve (union)
+    cand_set = retrieve_candidates(plan, bm25, vec, topn=200, catalog_df=df_catalog)
+
+    # Step 3: rerank (use best reranker)
+    reranker = CrossEncoderReranker(model_name="models/reranker_crossenc/v0.1.0")
+    ranked = rerank_candidates(plan, cand_set, reranker, df_catalog, k=10)
+
+    # Step 4: constraints (hook; currently passthrough)
+    final_list = apply_constraints(plan, ranked)
+
+    # Step 5: explanation
+    summary = explain(plan, final_list, catalog_lookup)
+    return summary
+
+
+if __name__ == "__main__":
+    import sys
+
+    user_text = " ".join(sys.argv[1:]) or "Find a 1 hour culture fit assessment for a COO"
+    print(run_query(user_text))

agent/chat_agent.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+"""
+Chat-style agent using Gemini for planning + explanation, deterministic tools for retrieval/rerank.
+Set GOOGLE_API_KEY in your environment.
+"""
+import json
+import os
+from typing import Callable
+
+import pandas as pd
+
+from data.catalog_loader import load_catalog
+from recommenders.bm25 import BM25Recommender
+from recommenders.vector_recommender import VectorRecommender
+from retrieval.vector_index import VectorIndex
+from models.embedding_model import EmbeddingModel
+from rerankers.cross_encoder import CrossEncoderReranker
+
+from tools.query_plan_tool_llm import build_query_plan_llm
+from tools.query_plan_tool import build_query_plan as deterministic_plan
+from tools.retrieve_tool import retrieve_candidates
+from tools.rerank_tool import rerank_candidates
+from tools.constraints_tool import apply_constraints
+from tools.explain_tool import explain
+from schemas.query_plan import QueryPlan
+
+
+def load_resources():
+    df_catalog, _, _ = load_catalog("data/catalog_docs_rich.jsonl")
+    bm25 = BM25Recommender(df_catalog)
+    embed = EmbeddingModel("BAAI/bge-small-en-v1.5")
+    index = VectorIndex.load("data/faiss_index/index_bge.faiss")
+    with open("data/embeddings_bge/assessment_ids.json") as f:
+        ids = json.load(f)
+    vec = VectorRecommender(embed, index, df_catalog, ids, k_candidates=200)
+    catalog_by_id = {row["assessment_id"]: row for _, row in df_catalog.iterrows()}
+    return df_catalog, bm25, vec, catalog_by_id
+
+
+def make_catalog_lookup(df_catalog: pd.DataFrame) -> Callable[[str], dict]:
+    cat = df_catalog.set_index("assessment_id")
+
+    def lookup(aid: str) -> dict:
+        if aid in cat.index:
+            return cat.loc[aid].to_dict()
+        return {}
+
+    return lookup
+
+
+def _maybe_clarify(plan: QueryPlan, cand_count: int, topn: int) -> str | None:
+    # LLM-flagged clarification
+    if plan.needs_clarification and plan.clarifying_question:
+        return plan.clarifying_question
+    # Coverage-based triggers
+    if cand_count < max(10, int(0.25 * topn)):
+        return "Results look thin. Clarify: are you looking for (1) personality/culture fit, (2) leadership judgment (SJT), or (3) role capability?"
+    if plan.intent in {"BEHAVIORAL", "UNKNOWN", "MIXED"} and cand_count < max(20, int(0.5 * topn)):
+        return "For culture/behavioral focus, choose: (1) personality/culture fit, (2) leadership judgment (SJT), or (3) role capability. Please pick one."
+    return None
+
+
+def run_chat(
+    user_text: str,
+    vocab_path: str = "data/catalog_role_vocab.json",
+    model_name: str = "gemini-2.5-flash-lite",
+    clarification_answer: str | None = None,
+    topn: int = 200,
+    verbose: bool = False,
+):
+    vocab = json.load(open(vocab_path)) if vocab_path and os.path.exists(vocab_path) else {}
+    df_catalog, bm25, vec, catalog_by_id = load_resources()
+    catalog_lookup = make_catalog_lookup(df_catalog)
+
+    trace_id = f"trace-{abs(hash(user_text))}"
+    log = {"trace_id": trace_id, "raw_query": user_text}
+
+    # Plan with LLM; fallback deterministic if LLM fails
+    try:
+        plan = build_query_plan_llm(user_text, vocab=vocab, model_name=model_name)
+        QueryPlan.model_validate(plan.dict())  # schema guard
+        log["plan_source"] = "llm"
+    except Exception as e:
+        plan = deterministic_plan(user_text, vocab=vocab)
+        log["plan_source"] = f"deterministic (llm_fail={str(e)})"
+    log["query_plan"] = plan.dict()
+
+    # Retrieve union
+    cand_set = retrieve_candidates(plan, bm25, vec, topn=topn, catalog_df=df_catalog)
+    if verbose:
+        log["candidates"] = [c.model_dump() for c in cand_set.candidates[:10]]
+
+    # Clarification loop
+    question = _maybe_clarify(plan, cand_count=len(cand_set.candidates), topn=topn)
+    if question and not clarification_answer:
+        log["clarification"] = question
+        if verbose:
+            print(json.dumps(log, indent=2))
+        return f"Clarification needed: {question}"
+    if question and clarification_answer:
+        clarified_text = f"{user_text}\nUser clarification: {clarification_answer}"
+        try:
+            plan = build_query_plan_llm(clarified_text, vocab=vocab, model_name=model_name)
+            QueryPlan.model_validate(plan.dict())
+        except Exception:
+            plan = deterministic_plan(clarified_text, vocab=vocab)
+        log["query_plan_clarified"] = plan.dict()
+        cand_set = retrieve_candidates(plan, bm25, vec, topn=topn, catalog_df=df_catalog)
+        if verbose:
+            log["candidates_clarified"] = [c.model_dump() for c in cand_set.candidates[:10]]
+
+    # Rerank
+    reranker = CrossEncoderReranker(model_name="models/reranker_crossenc/v0.1.0")
+    ranked = rerank_candidates(plan, cand_set, reranker, df_catalog, k=10)
+    log["rerank"] = [item.model_dump() for item in ranked.items]
+
+    # Constraints
+    final_list = apply_constraints(plan, ranked, catalog_by_id, k=10)
+    log["final"] = [item.model_dump() for item in final_list.items]
+
+    # Explain
+    summary = explain(plan, final_list, catalog_lookup)
+    log["summary"] = summary
+
+    # Compact output: top-10 with metadata
+    final_results = []
+    for item in final_list.items:
+        meta = catalog_lookup(item.assessment_id)
+        final_results.append(
+            {
+                "assessment_id": item.assessment_id,
+                "score": item.score,
+                "name": meta.get("name"),
+                "url": meta.get("url"),
+                "test_type_full": meta.get("test_type_full") or meta.get("test_type"),
+                "duration": meta.get("duration_minutes") or meta.get("duration"),
+            }
+        )
+
+    if verbose:
+        log["final_results"] = final_results
+        print(json.dumps(log, indent=2))
+    else:
+        print(json.dumps({"trace_id": trace_id, "final_results": final_results}, indent=2))
+
+    return summary
+
+
+if __name__ == "__main__":
+    import sys
+
+    if "GOOGLE_API_KEY" not in os.environ:
+        print("Please set GOOGLE_API_KEY for Gemini.")
+    user_text = " ".join(sys.argv[1:]) or "Find a 1 hour culture fit assessment for a COO"
+    print(run_chat(user_text, verbose=False))

agent/router_agent.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+"""
+Router-style agent (minimal, deterministic) that orchestrates the tool stack:
+- build_query_plan
+- retrieve_candidates
+- rerank_candidates
+- apply_constraints
+- explain
+
+This is intentionally simple and does not require an LLM. You can swap
+build_query_plan with an LLM-based planner that emits the same QueryPlan schema.
+"""
+import json
+from typing import Callable
+
+import pandas as pd
+
+from data.catalog_loader import load_catalog
+from recommenders.bm25 import BM25Recommender
+from recommenders.vector_recommender import VectorRecommender
+from retrieval.vector_index import VectorIndex
+from models.embedding_model import EmbeddingModel
+from rerankers.cross_encoder import CrossEncoderReranker
+
+from tools.query_plan_tool import build_query_plan
+from tools.retrieve_tool import retrieve_candidates
+from tools.rerank_tool import rerank_candidates
+from tools.constraints_tool import apply_constraints
+from tools.explain_tool import explain
+
+
+def load_resources():
+    df_catalog, _, _ = load_catalog("data/catalog_docs_rich.jsonl")
+    bm25 = BM25Recommender(df_catalog)
+    embed = EmbeddingModel("BAAI/bge-small-en-v1.5")
+    index = VectorIndex.load("data/faiss_index/index_bge.faiss")
+    with open("data/embeddings_bge/assessment_ids.json") as f:
+        ids = json.load(f)
+    vec = VectorRecommender(embed, index, df_catalog, ids, k_candidates=200)
+    return df_catalog, bm25, vec
+
+
+def make_catalog_lookup(df_catalog: pd.DataFrame) -> Callable[[str], dict]:
+    cat = df_catalog.set_index("assessment_id")
+
+    def lookup(aid: str) -> dict:
+        if aid in cat.index:
+            return cat.loc[aid].to_dict()
+        return {}
+
+    return lookup
+
+
+def route_query(user_text: str, vocab_path: str = "data/catalog_role_vocab.json") -> str:
+    vocab = json.load(open(vocab_path)) if vocab_path else {}
+    df_catalog, bm25, vec = load_resources()
+    catalog_lookup = make_catalog_lookup(df_catalog)
+
+    # 1) Plan (deterministic rewriter; swap with LLM-structured plan if desired)
+    plan = build_query_plan(user_text, vocab=vocab)
+
+    # 2) Clarification hook
+    # Placeholder: in an interactive app, if plan.needs_clarification or coverage is weak,
+    # ask a question and rebuild the plan with the user response.
+
+    # 3) Retrieve (union of BM25 + vector)
+    cand_set = retrieve_candidates(plan, bm25, vec, topn=200, catalog_df=df_catalog)
+
+    # 4) Rerank (cross-encoder)
+    reranker = CrossEncoderReranker(model_name="models/reranker_crossenc/v0.1.0")
+    ranked = rerank_candidates(plan, cand_set, reranker, df_catalog, k=10)
+
+    # 5) Apply constraints (stub; extend for duration/remote/adaptive)
+    final_list = apply_constraints(plan, ranked)
+
+    # 6) Explain
+    summary = explain(plan, final_list, catalog_lookup)
+    return summary
+
+
+if __name__ == "__main__":
+    import sys
+
+    user_text = " ".join(sys.argv[1:]) or "Find a 1 hour culture fit assessment for a COO"
+    print(route_query(user_text))

agent/server.py

@@ -0,0 +1,324 @@
+from __future__ import annotations
+
+"""
+Minimal chat backend (FastAPI) that delegates to the agent app pipeline.
+
+Run:
+  uvicorn agent.server:app --reload --port 8000
+"""
+
+import uuid
+import json
+from typing import Optional, Callable
+from collections import deque
+import time
+import math
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+from pydantic import BaseModel
+
+from functools import lru_cache
+import os
+from data.catalog_loader import load_catalog
+from recommenders.bm25 import BM25Recommender
+from recommenders.vector_recommender import VectorRecommender
+from retrieval.vector_index import VectorIndex
+from models.embedding_model import EmbeddingModel
+from rerankers.cross_encoder import CrossEncoderReranker
+from tools.query_plan_tool import build_query_plan
+from tools.query_plan_tool_llm import build_query_plan_llm
+from llm.nu_extract import NuExtractWrapper, default_query_rewrite_examples
+from llm.qwen_rewriter import QwenRewriter
+from tools.retrieve_tool import retrieve_candidates
+from tools.rerank_tool import rerank_candidates
+from tools.constraints_tool import apply_constraints
+
+
+class ChatRequest(BaseModel):
+    query: str
+    clarification_answer: Optional[str] = None
+    verbose: bool = False
+
+class RecommendRequest(BaseModel):
+    query: str
+    llm_model: Optional[str] = None
+    verbose: bool = False
+
+
+def _make_catalog_lookup(df_catalog) -> Callable[[str], dict]:
+    cat = df_catalog.set_index("assessment_id")
+    def lookup(aid: str) -> dict:
+        if aid in cat.index:
+            return cat.loc[aid].to_dict()
+        return {}
+    return lookup
+
+
+@lru_cache(maxsize=1)
+def load_resources(llm_model_override: Optional[str] = None):
+    df_catalog, _, _ = load_catalog("data/catalog_docs_rich.jsonl")
+    bm25 = BM25Recommender(df_catalog)
+    embed = EmbeddingModel("BAAI/bge-small-en-v1.5")
+    index = VectorIndex.load("data/faiss_index/index_bge.faiss")
+    with open("data/embeddings_bge/assessment_ids.json") as f:
+        ids = json.load(f)
+    vec = VectorRecommender(embed, index, df_catalog, ids, k_candidates=200)
+    reranker = CrossEncoderReranker(model_name="models/reranker_crossenc/v0.1.0")
+    lookup = _make_catalog_lookup(df_catalog)
+    catalog_by_id = {row["assessment_id"]: row for _, row in df_catalog.iterrows()}
+    vocab = {}
+    vocab_path = "data/catalog_role_vocab.json"
+    if os.path.exists(vocab_path):
+        try:
+            with open(vocab_path) as vf:
+                vocab = json.load(vf)
+        except Exception:
+            vocab = {}
+    # Optional LLM rewriter; choose via request override or env LLM_MODEL
+    llm_extractor = None
+    llm_model = llm_model_override or os.getenv("LLM_MODEL", "").strip()
+    if not llm_model:
+        llm_model = "Qwen/Qwen2.5-1.5B-Instruct"
+    try:
+        if llm_model.lower().startswith("qwen"):
+            llm_extractor = QwenRewriter(model_name=llm_model, default_examples=default_query_rewrite_examples())
+        elif not os.getenv("GOOGLE_API_KEY"):
+            llm_extractor = NuExtractWrapper(default_examples=default_query_rewrite_examples())
+    except Exception:
+        llm_extractor = None
+    return df_catalog, bm25, vec, reranker, lookup, vocab, llm_extractor, catalog_by_id
+
+
+def _infer_remote_adaptive(meta: dict) -> (Optional[bool], Optional[bool]):
+    remote = meta.get("remote_support", True if meta.get("remote_support") is None else meta.get("remote_support"))
+    adaptive = meta.get("adaptive_support")
+    text_blob = " ".join([str(meta.get("name", "")), str(meta.get("description", "")), str(meta.get("doc_text", ""))]).lower()
+    if adaptive is None and "adaptive" in text_blob:
+        adaptive = True
+    return remote, adaptive
+
+
+def _build_plan_with_fallback(query: str, vocab: dict, llm_extractor):
+    """
+    Build the query plan using the LLM rewriter (Qwen) when available, otherwise
+    fall back to deterministic rewrite. No Gemini refinement to keep behavior predictable.
+    """
+    try:
+        return build_query_plan(query, vocab=vocab, llm_extractor=llm_extractor)
+    except Exception:
+        return build_query_plan(query, vocab=vocab)
+
+
+def _safe_num(val):
+    try:
+        if val is None:
+            return None
+        f = float(val)
+        if math.isfinite(f):
+            return f
+    except Exception:
+        return None
+    return None
+
+
+def _sanitize_debug(obj):
+    """Recursively replace NaN/inf with None to keep JSON safe."""
+    if isinstance(obj, dict):
+        return {k: _sanitize_debug(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_sanitize_debug(v) for v in obj]
+    if isinstance(obj, tuple):
+        return tuple(_sanitize_debug(v) for v in obj)
+    if isinstance(obj, (int, float)):
+        return _safe_num(obj)
+    return obj
+
+
+CODE_TO_FULL = {
+    "A": "Ability & Aptitude",
+    "B": "Biodata & Situational Judgement",
+    "C": "Competencies",
+    "D": "Development & 360",
+    "E": "Assessment Exercises",
+    "K": "Knowledge & Skills",
+    "P": "Personality & Behavior",
+    "S": "Simulations",
+}
+
+
+def _format_test_types(meta: dict) -> list[str]:
+    if meta.get("test_type_full"):
+        raw = meta["test_type_full"]
+    elif meta.get("test_type"):
+        raw = meta["test_type"]
+    else:
+        return []
+    if isinstance(raw, list):
+        vals = raw
+    else:
+        vals = str(raw).replace("/", ",").split(",")
+    out = []
+    for v in vals:
+        v = v.strip()
+        if not v:
+            continue
+        # Map letter codes to full names when applicable
+        if len(v) == 1 and v in CODE_TO_FULL:
+            out.append(CODE_TO_FULL[v])
+        else:
+            out.append(v)
+    return out
+
+
+def _run_pipeline(query: str, topn: int = 200, verbose: bool = False, llm_model: Optional[str] = None):
+    if verbose:
+        # For debugging, bypass cached resources to ensure fresh state
+        load_resources.cache_clear()
+    df_catalog, bm25, vec, reranker, lookup, vocab, llm_extractor, catalog_by_id = load_resources(llm_model_override=llm_model)
+    plan = _build_plan_with_fallback(query, vocab=vocab, llm_extractor=llm_extractor)
+    cand_set = retrieve_candidates(plan, bm25, vec, topn=topn, catalog_df=df_catalog)
+    ranked = rerank_candidates(plan, cand_set, reranker, df_catalog, k=10)
+    final_list = apply_constraints(plan, ranked, catalog_by_id, k=10)
+
+    debug_payload = {}
+    if verbose:
+        debug_payload["plan"] = plan.dict()
+        # If plan carries a source (from planner), include it
+        if hasattr(plan, "plan_source"):
+            debug_payload["plan_source"] = getattr(plan, "plan_source")
+        # Capture NuExtract LLM debug if present
+        if hasattr(plan, "llm_debug") and plan.llm_debug:
+            debug_payload["llm_debug"] = plan.llm_debug
+        if hasattr(cand_set, "fusion") and cand_set.fusion:
+            debug_payload["fusion"] = cand_set.fusion
+        debug_payload["candidates"] = [
+            {
+                "assessment_id": c.assessment_id,
+                "bm25_rank": c.bm25_rank,
+                "vector_rank": c.vector_rank,
+                "hybrid_rank": c.hybrid_rank,
+                "bm25_score": _safe_num(c.bm25_score),
+                "vector_score": _safe_num(c.vector_score),
+                "score": _safe_num(c.score),
+            }
+            for c in cand_set.candidates[: min(20, len(cand_set.candidates))]
+        ]
+        debug_payload["rerank"] = [
+            {"assessment_id": r.assessment_id, "score": _safe_num(r.score)}
+            for r in ranked.items[: min(20, len(ranked.items))]
+        ]
+        debug_payload["constraints"] = [
+            {
+                "assessment_id": r.assessment_id,
+                "score": _safe_num(r.score),
+                "debug": r.debug,
+            }
+            for r in final_list.items
+        ]
+
+    final_results = []
+    for item in final_list.items:
+        meta = lookup(item.assessment_id)
+        remote, adaptive = _infer_remote_adaptive(meta)
+        score = _safe_num(item.score)
+        duration = _safe_num(meta.get("duration_minutes") or meta.get("duration"))
+        duration_int = int(duration) if duration is not None else None
+        description = meta.get("description") or meta.get("doc_text") or ""
+        test_types = _format_test_types(meta)
+        final_results.append(
+            {
+                "url": meta.get("url"),
+                "name": meta.get("name"),
+                "adaptive_support": "Yes" if adaptive else "No",
+                "description": description,
+                "duration": duration_int if duration_int is not None else 0,
+                "remote_support": "Yes" if remote else "No",
+                "test_type": test_types,
+            }
+        )
+    # Guarantee at least one result if pipeline produced candidates
+    if not final_results and ranked.items:
+        item = ranked.items[0]
+        meta = lookup(item.assessment_id)
+        remote, adaptive = _infer_remote_adaptive(meta)
+        duration = _safe_num(meta.get("duration_minutes") or meta.get("duration"))
+        duration_int = int(duration) if duration is not None else 0
+        final_results.append(
+            {
+                "url": meta.get("url"),
+                "name": meta.get("name"),
+                "adaptive_support": "Yes" if adaptive else "No",
+                "description": meta.get("description") or meta.get("doc_text") or "",
+                "duration": duration_int,
+                "remote_support": "Yes" if remote else "No",
+                "test_type": _format_test_types(meta),
+            }
+        )
+    summary = {"plan": plan.intent, "top": len(final_results)}
+    return final_results, summary, debug_payload
+
+
+app = FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=False,  # '*' cannot be used with credentials
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Serve frontend assets
+app.mount("/static", StaticFiles(directory="frontend"), name="static")
+
+# Simple in-process rate limiter (max 5 requests per second)
+_timestamps = deque()
+_RATE_LIMIT = 5
+_WINDOW = 1.0
+
+def _allow_request() -> bool:
+    now = time.time()
+    while _timestamps and now - _timestamps[0] > _WINDOW:
+        _timestamps.popleft()
+    if len(_timestamps) < _RATE_LIMIT:
+        _timestamps.append(now)
+        return True
+    return False
+
+
+@app.post("/chat")
+def chat(req: ChatRequest):
+    if not _allow_request():
+        return {"error": "rate limit exceeded"}
+    trace_id = str(uuid.uuid4())
+    final_results, summary, debug_payload = _run_pipeline(req.query, verbose=req.verbose)
+    payload = {"trace_id": trace_id, "final_results": final_results}
+    if req.verbose:
+        payload["summary"] = summary
+        payload["debug"] = _sanitize_debug(debug_payload)
+    return payload
+
+
+@app.post("/recommend")
+def recommend(req: RecommendRequest):
+    if not _allow_request():
+        return {"error": "rate limit exceeded"}
+    final_results, summary, debug_payload = _run_pipeline(req.query, verbose=req.verbose, llm_model=req.llm_model)
+    resp = {"recommended_assessments": final_results}
+    if req.verbose:
+        resp["debug"] = _sanitize_debug(debug_payload)
+        resp["summary"] = summary
+    return resp
+
+
+@app.get("/health")
+def health():
+    return {"status": "ok"}
+
+
+@app.get("/")
+def index():
+    # Serve the SPA entry point
+    return FileResponse("frontend/index.html")

api/__init__.py

@@ -0,0 +1 @@
+"""API surface (REST/gRPC/WebSocket) for serving recommendations."""

config.py

@@ -0,0 +1,75 @@
+"""Centralized config loader using YAML with ENV overrides.
+
+Environment variables prefixed with ``LRE_`` can override nested keys using
+double-underscores, e.g. ``LRE_APP__LOG_LEVEL=DEBUG``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import pathlib
+from typing import Any, Dict
+
+import yaml
+
+DEFAULT_CONFIG_PATH = pathlib.Path(os.environ.get("CONFIG_PATH", "configs/config.yaml"))
+ENV_PREFIX = "LRE_"
+
+
+def _parse_env_value(value: str) -> Any:
+    """Best-effort casting for env values."""
+    lowered = value.lower()
+    if lowered in {"true", "false"}:
+        return lowered == "true"
+    try:
+        return int(value)
+    except ValueError:
+        pass
+    try:
+        return float(value)
+    except ValueError:
+        pass
+    return value
+
+
+def _set_nested(config: Dict[str, Any], path: list[str], value: Any) -> None:
+    cursor = config
+    for part in path[:-1]:
+        cursor = cursor.setdefault(part, {})
+    cursor[path[-1]] = value
+
+
+def apply_env_overrides(config: Dict[str, Any], prefix: str = ENV_PREFIX) -> Dict[str, Any]:
+    """Apply ENV overrides in-place and return config."""
+    for key, raw_value in os.environ.items():
+        if not key.startswith(prefix):
+            continue
+        path = key[len(prefix) :].lower().split("__")
+        _set_nested(config, path, _parse_env_value(raw_value))
+    return config
+
+
+def load_config(config_path: pathlib.Path | str | None = None) -> Dict[str, Any]:
+    """Load YAML config and apply ENV overrides."""
+    path = pathlib.Path(config_path or DEFAULT_CONFIG_PATH)
+    with path.open() as f:
+        config: Dict[str, Any] = yaml.safe_load(f) or {}
+    return apply_env_overrides(config)
+
+
+def _cli() -> None:
+    parser = argparse.ArgumentParser(description="Config loader helper")
+    parser.add_argument(
+        "--print", dest="print_config", action="store_true", help="Print resolved config"
+    )
+    parser.add_argument("--path", dest="config_path", type=str, help="Optional config path")
+    args = parser.parse_args()
+
+    cfg = load_config(args.config_path)
+    if args.print_config:
+        print(yaml.dump(cfg, sort_keys=False))
+
+
+if __name__ == "__main__":
+    _cli()

configs/__init__.py

@@ -0,0 +1 @@
+"""Config package placeholder."""

configs/config.yaml

@@ -0,0 +1,43 @@
+app:
+  name: llm-recommendation-engine
+  environment: local
+  log_level: INFO
+
+data:
+  raw_dir: data/raw
+  processed_dir: data/processed
+  cache_dir: data/cache
+
+index:
+  type: faiss
+  dim: 384
+  store_path: data/index/faiss.index
+
+models:
+  embedder: sentence-transformers/all-MiniLM-L6-v2
+  reranker: cross-encoder/ms-marco-MiniLM-L-6-v2
+
+services:
+  api:
+    host: 0.0.0.0
+    port: 8000
+  ui:
+    host: 0.0.0.0
+    port: 3000
+
+observability:
+  tracing_enabled: false
+  metrics_endpoint: /metrics
+
+storage:
+  bucket: s3://placeholder-bucket
+  prefix: recommendations
+
+crawler:
+  start_url: https://www.shl.com/products/product-catalog/
+  user_agent: llm-recommendation-engine/0.1 (+https://example.com)
+  max_concurrency: 2
+  request_delay_seconds: 1.5
+  jitter_seconds: 0.5
+  max_retries: 3
+  sqlite_path: data/crawler.db

configs/embedding_config.yaml

@@ -0,0 +1,10 @@
+model:
+  name: sentence-transformers/all-MiniLM-L6-v2
+  cache_dir: .model_cache
+  device: cpu
+  normalize_embeddings: true
+  batch_size: 32
+
+preprocessing:
+  max_length: 512
+  padding: false

configs/retrieval.yaml

@@ -0,0 +1,5 @@
+retrieval:
+  method: hybrid_rrf
+  train_topn_candidates: 200
+  infer_topn_candidates: 100
+  rrf_k: 60

crawler/__init__.py

@@ -0,0 +1 @@
+"""Data ingestion and crawling utilities."""

crawler/backfill_labels.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import csv
+import os
+from pathlib import Path
+
+import structlog
+
+from config import load_config
+from crawler.fetcher import PlaywrightFetcher
+from crawler.parser_detail import parse_detail_page
+from crawler.robots import RobotsManager
+from crawler.storage import PAGE_TYPE_DETAIL, PARSE_PARSED, PageRecord, Storage
+from crawler.utils import RateLimiter
+
+logger = structlog.get_logger(__name__)
+
+
+async def backfill_from_probe(probe_csv: str, storage: Storage, fetcher: PlaywrightFetcher, robots: RobotsManager, allow_bypass: bool):
+    with open(probe_csv) as f:
+        reader = csv.DictReader(f)
+        rows = [row for row in reader if row.get("classification") == "DETAIL_PAGE_VALID"]
+    logger.info("backfill.labels.start", count=len(rows))
+    for row in rows:
+        url = row["url"]
+        allowed = allow_bypass or robots.is_allowed(url)
+        if not allowed:
+            logger.warning("backfill.detail.disallowed", url=url)
+            continue
+        if allow_bypass:
+            logger.warning("backfill.detail.disallowed.bypassed", url=url)
+        result = await fetcher.fetch(url, page_type=PAGE_TYPE_DETAIL)
+        storage.upsert_page(result.record)
+        if result.error or not result.html:
+            logger.error("backfill.detail.fetch_failed", url=url, error=result.error)
+            continue
+        parse_detail_page(result.html, url=url, storage=storage)
+        storage.update_parse_status(url, PARSE_PARSED)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Backfill assessments from probed label URLs")
+    parser.add_argument("--probe-csv", required=True, help="CSV from scripts/probe_unmatched_labels.py")
+    parser.add_argument("--config", type=str, default=os.environ.get("CONFIG_PATH", "configs/config.yaml"))
+    parser.add_argument("--sqlite", type=str, default="data/crawler.db")
+    parser.add_argument("--allow-robots-bypass", action="store_true", help="Bypass robots.txt disallow (use responsibly)")
+    args = parser.parse_args()
+
+    config = load_config(args.config)
+    rate_limiter = RateLimiter(
+        base_delay=float(os.environ.get("REQUEST_DELAY_SECONDS", config.get("crawler", {}).get("request_delay_seconds", 1.5))),
+        jitter=float(os.environ.get("JITTER_SECONDS", config.get("crawler", {}).get("jitter_seconds", 0.5))),
+    )
+    user_agent = os.environ.get("USER_AGENT", config.get("crawler", {}).get("user_agent"))
+    max_retries = int(os.environ.get("MAX_RETRIES", config.get("crawler", {}).get("max_retries", 3)))
+
+    storage = Storage(args.sqlite)
+    robots = RobotsManager(robots_url="https://www.shl.com/robots.txt", user_agent=user_agent)
+    robots.load()
+
+    async def _runner():
+        async with PlaywrightFetcher(user_agent=user_agent, rate_limiter=rate_limiter, max_retries=max_retries) as fetcher:
+            await backfill_from_probe(args.probe_csv, storage, fetcher, robots, allow_bypass=args.allow_robots_bypass)
+
+    asyncio.run(_runner())
+    logger.info("backfill.labels.done")
+
+
+if __name__ == "__main__":
+    main()

crawler/export.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Optional
+
+import pandas as pd
+import structlog
+
+from crawler.storage import Storage
+from crawler.utils import now_iso
+
+logger = structlog.get_logger(__name__)
+
+
+def _normalize_row(row) -> dict:
+    downloads = row["downloads"]
+    if isinstance(downloads, str):
+        try:
+            downloads = json.loads(downloads)
+        except Exception:
+            downloads = None
+    job_levels = row["job_levels"]
+    if isinstance(job_levels, str):
+        try:
+            job_levels = json.loads(job_levels)
+        except Exception:
+            job_levels = [j.strip() for j in job_levels.split(",") if j.strip()]
+    languages = row.get("languages")
+    if isinstance(languages, str):
+        try:
+            languages = json.loads(languages)
+        except Exception:
+            languages = [l.strip() for l in languages.split(",") if l.strip()]
+    duration_minutes = row["duration_minutes"]
+    duration_hours = None
+    if duration_minutes is not None:
+        try:
+            duration_hours = float(duration_minutes) / 60.0
+        except Exception:
+            duration_hours = None
+    return {
+        "url": row["url"],
+        "name": row["name"],
+        "description": row["description"],
+        "test_type": row["test_type"],
+        "test_type_full": row.get("test_type_full"),
+        "remote_support": bool(row["remote_support"]) if row["remote_support"] is not None else None,
+        "adaptive_support": bool(row["adaptive_support"]) if row["adaptive_support"] is not None else None,
+        "duration": duration_minutes,
+        "duration_hours": duration_hours,
+        "job_levels": job_levels,
+        "languages": languages,
+        "downloads": downloads,
+        "source": "shl_product_catalog",
+        "crawled_at": now_iso(),
+    }
+
+
+def export_catalog(
+    storage: Storage,
+    parquet_path: str,
+    jsonl_path: Optional[str] = None,
+    min_count: int = 377,
+    limit: Optional[int] = None,
+) -> None:
+    rows = storage.fetch_assessments()
+    logger.info("export.assessments.fetched", count=len(rows))
+
+    if len(rows) < min_count:
+        raise RuntimeError(f"Validation failed: expected at least {min_count} assessments, got {len(rows)}")
+
+    records = [_normalize_row(dict(r)) for r in rows]
+    df = pd.DataFrame.from_records(records)
+    if limit:
+        df = df.head(limit)
+        logger.info("export.limit.applied", limit=limit, rows=len(df))
+
+    Path(parquet_path).parent.mkdir(parents=True, exist_ok=True)
+    df.to_parquet(parquet_path, index=False)
+    logger.info("export.parquet.write", path=parquet_path, rows=len(df))
+
+    if jsonl_path:
+        df.to_json(jsonl_path, orient="records", lines=True, force_ascii=False)
+        logger.info("export.jsonl.write", path=jsonl_path, rows=len(df))
+
+    missing_desc = df["description"].isna().sum()
+    missing_duration = df["duration"].isna().sum()
+    logger.info(
+        "export.summary",
+        missing_description=missing_desc,
+        missing_duration=missing_duration,
+        test_type_counts=df["test_type"].value_counts(dropna=False).to_dict(),
+    )

crawler/fetcher.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import Optional
+
+import structlog
+from playwright.async_api import async_playwright, Browser, Page
+
+from crawler.storage import PageRecord
+from crawler.utils import RateLimiter
+
+logger = structlog.get_logger(__name__)
+
+
+@dataclass
+class FetchResult:
+    record: PageRecord
+    status: Optional[int]
+    html: Optional[str]
+    error: Optional[str]
+
+
+class PlaywrightFetcher:
+    """Thin wrapper around Playwright with polite rate limiting."""
+
+    def __init__(
+        self,
+        user_agent: str,
+        rate_limiter: RateLimiter,
+        max_retries: int = 3,
+    ) -> None:
+        self.user_agent = user_agent
+        self.rate_limiter = rate_limiter
+        self.max_retries = max_retries
+        self._playwright = None
+        self._browser: Optional[Browser] = None
+        self._page: Optional[Page] = None
+
+    async def __aenter__(self) -> "PlaywrightFetcher":
+        await self.start()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        await self.close()
+
+    async def start(self) -> None:
+        if self._page:
+            return
+        self._playwright = await async_playwright().start()
+        self._browser = await self._playwright.chromium.launch(headless=True)
+        context = await self._browser.new_context(user_agent=self.user_agent)
+        self._page = await context.new_page()
+        logger.info("fetcher.started", user_agent=self.user_agent)
+
+    async def close(self) -> None:
+        if self._browser:
+            await self._browser.close()
+        if self._playwright:
+            await self._playwright.stop()
+        self._browser = None
+        self._page = None
+        logger.info("fetcher.closed")
+
+    async def fetch(self, url: str, page_type: str) -> FetchResult:
+        assert self._page, "Fetcher must be started before fetch()"
+        attempt = 0
+        last_error: Optional[str] = None
+        html: Optional[str] = None
+        status: Optional[int] = None
+
+        while attempt < self.max_retries:
+            attempt += 1
+            self.rate_limiter.sleep()
+            logger.info("fetcher.request", url=url, attempt=attempt)
+            try:
+                response = await self._page.goto(url, wait_until="networkidle", timeout=20000)
+                status = response.status if response else None
+                html = await self._page.content()
+                return FetchResult(
+                    record=PageRecord(url=url, page_type=page_type, http_status=status, html=html),
+                    status=status,
+                    html=html,
+                    error=None,
+                )
+            except Exception as exc:  # pragma: no cover - network variability
+                last_error = str(exc)
+                logger.warning("fetcher.request.error", url=url, error=last_error, attempt=attempt)
+        return FetchResult(
+            record=PageRecord(url=url, page_type=page_type, http_status=status, html=html, error=last_error),
+            status=status,
+            html=html,
+            error=last_error,
+        )
+
+
+def fetch_sync(url: str, page_type: str, user_agent: str, rate_limiter: RateLimiter, max_retries: int = 3) -> FetchResult:
+    async def _runner():
+        async with PlaywrightFetcher(user_agent=user_agent, rate_limiter=rate_limiter, max_retries=max_retries) as fetcher:
+            return await fetcher.fetch(url, page_type)
+
+    return asyncio.run(_runner())

crawler/parser_catalog.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+from typing import List, Tuple
+from urllib.parse import urljoin
+
+import structlog
+from bs4 import BeautifulSoup
+
+from crawler.storage import (
+    PAGE_TYPE_DETAIL,
+    PARSE_PARSED,
+    PageRecord,
+    Storage,
+)
+from crawler.utils import canonicalize_url, now_iso
+
+logger = structlog.get_logger(__name__)
+
+ALLOWED_TEST_TYPES = {"A", "B", "C", "D", "E", "K", "P", "S"}
+GREEN_TOKENS = ["green", "#8ac640", "rgb(138", "rgb(103", "0, 167, 83", "8ac640"]
+
+
+def _has_green_indicator(cell) -> bool:
+    for el in cell.find_all(True):
+        style = (el.get("style") or "").lower()
+        classes = " ".join(el.get("class", [])).lower() if isinstance(el.get("class"), list) else str(el.get("class") or "").lower()
+        combined = f"{style} {classes}"
+        if any(tok in combined for tok in GREEN_TOKENS):
+            return True
+        if "-yes" in classes or "catalogue__circle" in classes:
+            return True
+        fill = (el.get("fill") or "").lower()
+        if any(tok in fill for tok in GREEN_TOKENS):
+            return True
+        # Generic icon/dot detection (when color is applied via CSS)
+        if el.name in {"svg", "circle", "path", "i"}:
+            return True
+        if "dot" in classes or "indicator" in classes:
+            return True
+    return False
+
+
+def extract_catalog_entries(html: str) -> List[dict]:
+    """Parse catalog page for individual test solutions.
+
+    This is intentionally defensive; selectors may change on shl.com. We look for anchors within
+    sections that mention "Individual Test Solutions" or tables with product rows.
+    """
+    soup = BeautifulSoup(html, "lxml")
+    entries = []
+
+    tables = soup.find_all("table")
+    for table in tables:
+        headers = " ".join(th.get_text(" ", strip=True) for th in table.find_all("th"))
+        if "Individual Test Solutions" not in headers and "Assessment" not in headers:
+            continue
+        for row in table.find_all("tr"):
+            link = row.find("a", href=True)
+            if not link:
+                continue
+            name = link.get_text(strip=True)
+            detail_url = link["href"]
+            badges_text = [span.get_text("", strip=True) for span in row.find_all("span")]
+            test_letters = []
+            for token in badges_text:
+                token = token.strip()
+                if len(token) == 1 and token in ALLOWED_TEST_TYPES:
+                    test_letters.append(token)
+            test_type = ",".join(dict.fromkeys(test_letters)) or None
+            tds = row.find_all("td")
+            remote = None
+            adaptive = None
+            if len(tds) >= 3:
+                remote = _has_green_indicator(tds[1])
+                adaptive = _has_green_indicator(tds[2])
+            else:
+                flat_badges = " ".join(badges_text).lower()
+                remote = "remote" in flat_badges
+                adaptive = "adaptive" in flat_badges or "irt" in flat_badges
+            entries.append(
+                {
+                    "name": name,
+                    "url": detail_url,
+                    "test_type": test_type or None,
+                    "remote_support": remote if remote else None,
+                    "adaptive_support": adaptive if adaptive else None,
+                }
+            )
+    return entries
+
+
+def find_next_pages(html: str, source_url: str) -> List[str]:
+    """Find pagination links (Next or numbered) and resolve to absolute URLs."""
+    soup = BeautifulSoup(html, "lxml")
+    urls = []
+    for link in soup.find_all("a", href=True):
+        text = link.get_text(" ", strip=True).lower()
+        if "next" in text or text.isdigit():
+            urls.append(canonicalize_url(urljoin(source_url, link["href"])))
+    # de-duplicate while preserving order
+    seen = set()
+    deduped = []
+    for u in urls:
+        if u not in seen:
+            seen.add(u)
+            deduped.append(u)
+    return deduped
+
+
+def parse_catalog_page(html: str, source_url: str, storage: Storage) -> Tuple[int, List[str], List[str]]:
+    entries = extract_catalog_entries(html)
+    discovered_urls: List[str] = []
+
+    for entry in entries:
+        detail_url = canonicalize_url(urljoin(source_url, entry["url"]))
+        discovered_urls.append(detail_url)
+        storage.upsert_page(
+            PageRecord(
+                url=detail_url,
+                page_type=PAGE_TYPE_DETAIL,
+            )
+        )
+        storage.upsert_assessment(
+            {
+                "url": detail_url,
+                "name": entry.get("name"),
+                "test_type": entry.get("test_type"),
+                "remote_support": entry.get("remote_support"),
+                "adaptive_support": entry.get("adaptive_support"),
+                "source_catalog_page": canonicalize_url(source_url),
+                "discovered_at": now_iso(),
+            }
+        )
+
+    storage.update_parse_status(source_url, PARSE_PARSED)
+    next_pages = find_next_pages(html, source_url)
+    logger.info(
+        "catalog.parse.summary",
+        source_url=source_url,
+        discovered=len(discovered_urls),
+        next_pages=len(next_pages),
+    )
+    return len(entries), discovered_urls, next_pages

crawler/parser_detail.py

@@ -0,0 +1,320 @@
+from __future__ import annotations
+
+import re
+from typing import Dict, Iterable, List, Optional
+
+import structlog
+from bs4 import BeautifulSoup, Tag
+
+from crawler.storage import PARSE_PARSED, Storage
+from crawler.utils import canonicalize_url, now_iso
+
+logger = structlog.get_logger(__name__)
+
+ALLOWED_TEST_TYPES = {"A", "B", "C", "D", "E", "K", "P", "S"}
+STOP_LABELS = [
+    "Job levels",
+    "Job level",
+    "Languages",
+    "Language",
+    "Assessment length",
+    "Assessment Length",
+    "Test Type",
+    "Remote Testing",
+    "Adaptive/IRT",
+    "Adaptive",
+    "Downloads",
+]
+STOP_LABELS_LOWER = [s.lower() for s in STOP_LABELS]
+TEST_TYPE_LABELS = {
+    "A": "Ability & Aptitude",
+    "B": "Biodata & Situational Judgement",
+    "C": "Competencies",
+    "D": "Development & 360",
+    "E": "Assessment Exercises",
+    "K": "Knowledge & Skills",
+    "P": "Personality & Behavior",
+    "S": "Simulations",
+}
+
+
+def _normalize(text: str) -> str:
+    return re.sub(r"\s+", " ", (text or "")).strip()
+
+
+def _extract_text(soup: BeautifulSoup, selector: str) -> Optional[str]:
+    node = soup.select_one(selector)
+    if not node:
+        return None
+    text = _normalize(node.get_text(" ", strip=True))
+    return text or None
+
+
+def _find_label_node(soup: BeautifulSoup, label: str) -> Optional[Tag]:
+    label_l = label.lower()
+    candidates = soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6", "p", "div", "span", "strong", "dt", "th", "li"])
+    for node in candidates:
+        txt = _normalize(node.get_text(" ", strip=True)).lower()
+        if txt == label_l or txt.startswith(label_l + ":") or txt.startswith(label_l):
+            return node
+    for node in candidates:
+        txt = _normalize(node.get_text(" ", strip=True)).lower()
+        if re.search(rf"\b{re.escape(label_l)}\b", txt):
+            return node
+    return None
+
+
+def _extract_section_until(soup: BeautifulSoup, start_label: str, stop_labels: Iterable[str]) -> Optional[str]:
+    start = _find_label_node(soup, start_label)
+    if not start:
+        return None
+
+    chunks: List[str] = []
+
+    start_txt = _normalize(start.get_text(" ", strip=True))
+    if re.match(rf"^{re.escape(start_label)}\s*:", start_txt, flags=re.I):
+        after = re.split(rf"^{re.escape(start_label)}\s*:\s*", start_txt, flags=re.I)[-1]
+        if after:
+            chunks.append(after)
+
+    for node in start.find_all_next():
+        if node == start:
+            continue
+        if not isinstance(node, Tag):
+            continue
+
+        node_txt = _normalize(node.get_text(" ", strip=True))
+        if not node_txt:
+            continue
+
+        for stop in stop_labels:
+            if re.match(rf"^{re.escape(stop)}\b", node_txt, flags=re.I):
+                return _normalize(" ".join(chunks)) or None
+
+        if node.name in {"p", "li"}:
+            chunks.append(node_txt)
+        elif node.name in {"div", "span"} and len(node_txt) > 40:
+            chunks.append(node_txt)
+
+    return _normalize(" ".join(chunks)) or None
+
+
+def _extract_segment(text: str, label: str, stop_labels: Iterable[str]) -> Optional[str]:
+    """Extract substring after a label up to the next stop label in raw text."""
+    text_norm = _normalize(text)
+    lower = text_norm.lower()
+    label_l = label.lower()
+    start = lower.find(label_l)
+    if start == -1:
+        return None
+    start = start + len(label_l)
+    while start < len(text_norm) and text_norm[start] in " :":
+        start += 1
+    stop_pos = len(text_norm)
+    for stop in stop_labels:
+        pos = lower.find(stop, start)
+        if pos != -1 and pos < stop_pos:
+            stop_pos = pos
+    segment = text_norm[start:stop_pos].strip(" :-")
+    return segment or None
+
+
+def _extract_kv_value(soup: BeautifulSoup, label: str) -> Optional[str]:
+    node = _find_label_node(soup, label)
+    if not node:
+        return None
+
+    txt = _normalize(node.get_text(" ", strip=True))
+    m = re.match(rf"^{re.escape(label)}\s*:\s*(.+)$", txt, flags=re.I)
+    if m:
+        return m.group(1).strip() or None
+
+    remainder = re.sub(rf"^{re.escape(label)}\s*", "", txt, flags=re.I).strip(" :-")
+    if remainder and remainder.lower() != label.lower():
+        return remainder
+
+    for sib in node.next_siblings:
+        if isinstance(sib, Tag):
+            v = _normalize(sib.get_text(" ", strip=True))
+            if v:
+                return v
+
+    parent = node.parent if isinstance(node.parent, Tag) else None
+    if parent:
+        parent_txt = _normalize(parent.get_text(" ", strip=True))
+        parent_remainder = re.sub(rf"\b{re.escape(label)}\b", "", parent_txt, flags=re.I).strip(" :-")
+        if parent_remainder:
+            return parent_remainder
+        for sib in parent.find_next_siblings():
+            v = _normalize(sib.get_text(" ", strip=True))
+            if v:
+                return v
+
+    return None
+
+
+def _extract_duration_minutes(soup: BeautifulSoup) -> Optional[int]:
+    text = _normalize(soup.get_text(" ", strip=True))
+    patterns = [
+        r"minutes?\s*=\s*(\d+)",
+        r"(\d+)\s*(?:minute|min)\b",
+        r"completion time.*?(\d+)\s*(?:minute|min)\b",
+    ]
+    for pat in patterns:
+        m = re.search(pat, text, flags=re.I)
+        if m:
+            try:
+                return int(m.group(1))
+            except Exception:
+                continue
+    return None
+
+
+def _extract_test_type_from_meta(soup: BeautifulSoup) -> Optional[str]:
+    label = _find_label_node(soup, "Test Type")
+    scope = label.parent if label and isinstance(label.parent, Tag) else label or soup
+
+    tokens: List[str] = []
+    for el in scope.find_all(["span", "button", "a"], limit=30):
+        t = _normalize(el.get_text("", strip=True))
+        if len(t) == 1 and t in ALLOWED_TEST_TYPES:
+            tokens.append(t)
+    if not tokens:
+        for el in label.find_all_next(["span", "button", "a"], limit=30) if label else []:
+            t = _normalize(el.get_text("", strip=True))
+            if len(t) == 1 and t in ALLOWED_TEST_TYPES:
+                tokens.append(t)
+    if not tokens:
+        return None
+    out = []
+    seen = set()
+    for t in tokens:
+        if t not in seen:
+            seen.add(t)
+            out.append(t)
+    return ",".join(out)
+
+
+def _map_test_types_full(test_type: Optional[str]) -> Optional[str]:
+    if not test_type:
+        return None
+    parts = []
+    for token in test_type.split(","):
+        token = token.strip()
+        if not token:
+            continue
+        full = TEST_TYPE_LABELS.get(token)
+        if full:
+            parts.append(full)
+    return ", ".join(parts) if parts else None
+
+
+def _split_list(value: Optional[str]) -> Optional[list[str]]:
+    if not value:
+        return None
+    parts = [p.strip() for p in value.replace(";", ",").split(",") if p.strip()]
+    return parts or None
+
+
+def _is_positive_indicator(node: Tag) -> bool:
+    if not node:
+        return False
+    attrs = " ".join(
+        [
+            " ".join(node.get("class", [])) if isinstance(node.get("class"), list) else str(node.get("class") or ""),
+            str(node.get("aria-label") or ""),
+            str(node.get("title") or ""),
+            str(node.get("style") or ""),
+        ]
+    ).lower()
+    positive_tokens = ["green", "yes", "true", "available", "supported", "active", "enabled", "tick", "check", "on"]
+    return any(tok in attrs for tok in positive_tokens)
+
+
+def _extract_boolean_from_meta(soup: BeautifulSoup, label_text: str) -> Optional[bool]:
+    label = _find_label_node(soup, label_text)
+    if not label:
+        return None
+
+    container = label.parent if isinstance(label.parent, Tag) else label
+    for el in container.find_all(["span", "i", "svg", "img"], limit=20):
+        if _is_positive_indicator(el):
+            return True
+
+    for el in label.find_all_next(["span", "i", "svg", "img"], limit=20):
+        if _is_positive_indicator(el):
+            return True
+
+    return False
+
+
+def extract_detail_fields(html: str) -> Dict:
+    soup = BeautifulSoup(html, "lxml")
+
+    title = _extract_text(soup, "h1") or _extract_text(soup, "title")
+    full_text = _normalize(soup.get_text(" ", strip=True))
+    description = _extract_segment(full_text, "description", STOP_LABELS_LOWER)
+    if not description:
+        description = _extract_section_until(soup, "Description", STOP_LABELS)
+
+    job_levels_raw = _extract_kv_value(soup, "Job levels") or _extract_segment(full_text, "job levels", STOP_LABELS_LOWER)
+    job_levels = _split_list(job_levels_raw)
+    languages_raw = _extract_kv_value(soup, "Languages") or _extract_segment(full_text, "languages", STOP_LABELS_LOWER)
+    languages = _split_list(languages_raw)
+
+    duration = _extract_duration_minutes(soup)
+    if duration is None:
+        segment = _extract_segment(full_text, "assessment length", STOP_LABELS_LOWER)
+        if segment:
+            match = re.search(r"(\d+)\s*(?:minute|min)", segment, flags=re.I)
+            if match:
+                try:
+                    duration = int(match.group(1))
+                except Exception:
+                    duration = None
+
+    test_type = _extract_test_type_from_meta(soup)
+    test_type_full = _map_test_types_full(test_type)
+
+    remote_support = _extract_boolean_from_meta(soup, "Remote Testing")
+    adaptive_support = _extract_boolean_from_meta(soup, "Adaptive/IRT")
+    if adaptive_support is None:
+        adaptive_support = _extract_boolean_from_meta(soup, "Adaptive")
+    if adaptive_support is None:
+        adaptive_support = _extract_boolean_from_meta(soup, "Adaptive/IRT Testing")
+
+    downloads = []
+    downloads_label = _find_label_node(soup, "Downloads")
+    scope = downloads_label.parent if downloads_label and isinstance(downloads_label.parent, Tag) else soup
+    for link in scope.find_all("a", href=True):
+        text = _normalize(link.get_text(" ", strip=True))
+        href = link["href"]
+        if text and any(keyword in text.lower() for keyword in ["report", "fact sheet", "sample", "pdf", "download", "brochure"]):
+            downloads.append({"text": text, "url": href})
+
+    return {
+        "name": title,
+        "description": description,
+        "test_type": test_type,
+        "test_type_full": test_type_full,
+        "remote_support": remote_support,
+        "adaptive_support": adaptive_support,
+        "duration_minutes": duration,
+        "job_levels": job_levels,
+        "languages": languages,
+        "downloads": downloads or None,
+    }
+
+
+def parse_detail_page(html: str, url: str, storage: Storage) -> Dict:
+    fields = extract_detail_fields(html)
+    storage.upsert_assessment(
+        {
+            "url": canonicalize_url(url),
+            **fields,
+            "last_updated_at": now_iso(),
+        }
+    )
+    storage.update_parse_status(url, PARSE_PARSED)
+    logger.info("detail.parse.success", url=url, name=fields.get("name"))
+    return fields

crawler/qa_checks.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import pandas as pd
+
+
+def load_catalog(path: str) -> pd.DataFrame:
+    p = Path(path)
+    if not p.exists():
+        raise FileNotFoundError(f"Catalog file not found: {path}")
+    if p.suffix == ".jsonl":
+        return pd.read_json(path, lines=True)
+    if p.suffix in {".parquet", ".pq"}:
+        return pd.read_parquet(path)
+    raise ValueError(f"Unsupported catalog format: {path}")
+
+
+def qa_checks(df: pd.DataFrame) -> Dict[str, Any]:
+    total = len(df)
+
+    def pct_missing(col: str) -> float:
+        return float(df[col].isna().mean()) * 100.0 if col in df else 100.0
+
+    bool_sanity = {}
+    for col in ["remote_support", "adaptive_support"]:
+        if col in df:
+            bool_sanity[col] = bool(
+                df[col].dropna().apply(lambda x: isinstance(x, (bool, int))).all()
+            )
+        else:
+            bool_sanity[col] = False
+
+    description_lengths = df["description"].dropna().apply(lambda x: len(str(x))) if "description" in df else pd.Series(dtype=int)
+    min_desc_len: Optional[int] = int(description_lengths.min()) if not description_lengths.empty else None
+
+    return {
+        "total": total,
+        "count_gate": total >= 377,
+        "missing_pct": {
+            "description": pct_missing("description"),
+            "test_type": pct_missing("test_type"),
+            "remote_support": pct_missing("remote_support"),
+            "adaptive_support": pct_missing("adaptive_support"),
+            "duration_minutes": pct_missing("duration") if "duration" in df else pct_missing("duration_minutes"),
+        },
+        "url_uniqueness": {
+            "unique_urls": int(df["url"].nunique()) if "url" in df else 0,
+            "matches_row_count": bool("url" in df and df["url"].nunique() == total),
+        },
+        "description_quality": {
+            "min_length": min_desc_len,
+            "passed_min_30": bool(min_desc_len is not None and min_desc_len >= 30),
+        },
+        "test_type_distribution": df["test_type"].value_counts(dropna=False).to_dict() if "test_type" in df else {},
+        "boolean_sanity": bool_sanity,
+    }
+
+
+def main() -> None:
+    if len(sys.argv) < 2:
+        print("Usage: python qa_checks.py <catalog.jsonl|catalog.parquet>")
+        sys.exit(1)
+    path = sys.argv[1]
+    df = load_catalog(path)
+    results = qa_checks(df)
+    print(json.dumps(results, indent=2))
+
+
+if __name__ == "__main__":
+    main()

crawler/robots.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import logging
+import urllib.robotparser
+from dataclasses import dataclass
+
+import structlog
+
+
+logger = structlog.get_logger(__name__)
+
+
+@dataclass
+class RobotsManager:
+    robots_url: str
+    user_agent: str
+
+    def __post_init__(self) -> None:
+        self._parser = urllib.robotparser.RobotFileParser()
+
+    def load(self) -> None:
+        logger.info("robots.load.start", robots_url=self.robots_url)
+        self._parser.set_url(self.robots_url)
+        try:
+            self._parser.read()
+            logger.info("robots.load.success", can_fetch_all=self._parser.can_fetch(self.user_agent, "*"))
+        except Exception as exc:  # pragma: no cover - network errors are logged
+            logger.warning("robots.load.failed", error=str(exc))
+
+    def is_allowed(self, url: str) -> bool:
+        try:
+            return self._parser.can_fetch(self.user_agent, url)
+        except Exception as exc:  # pragma: no cover
+            logger.warning("robots.check.error", url=url, error=str(exc))
+            return False

crawler/run.py

@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import os
+from typing import Optional
+
+import logging
+import structlog
+
+from config import load_config
+from crawler.export import export_catalog
+from crawler.fetcher import PlaywrightFetcher
+from crawler.parser_catalog import parse_catalog_page
+from crawler.parser_detail import parse_detail_page
+from crawler.robots import RobotsManager
+from crawler.storage import (
+    PAGE_TYPE_CATALOG,
+    PAGE_TYPE_DETAIL,
+    PARSE_PENDING,
+    Storage,
+)
+from crawler.utils import RateLimiter
+
+logger = structlog.get_logger(__name__)
+
+
+def configure_logging(log_level: str = "INFO") -> None:
+    logging.basicConfig(level=getattr(logging, log_level.upper(), logging.INFO))
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(getattr(logging, log_level.upper(), logging.INFO)),
+        processors=[
+            structlog.processors.add_log_level,
+            structlog.processors.TimeStamper(fmt="iso"),
+            structlog.processors.JSONRenderer(),
+        ],
+    )
+
+
+async def crawl_catalog(
+    start_url: str,
+    storage: Storage,
+    fetcher: PlaywrightFetcher,
+    robots: RobotsManager,
+    max_discover: int | None = None,
+    allow_robots_bypass: bool = False,
+) -> None:
+    frontier = [start_url]
+    seen = set()
+    total_discovered = 0
+
+    while frontier:
+        url = frontier.pop(0)
+        if url in seen:
+            continue
+        seen.add(url)
+        allowed = allow_robots_bypass or robots.is_allowed(url)
+        if not allowed:
+            logger.warning("catalog.fetch.disallowed", url=url)
+            continue
+        if allow_robots_bypass:
+            logger.warning("catalog.fetch.disallowed.bypassed", url=url)
+        result = await fetcher.fetch(url, page_type=PAGE_TYPE_CATALOG)
+        storage.upsert_page(result.record)
+        if result.error or not result.html:
+            logger.error("catalog.fetch.failed", url=url, error=result.error)
+            continue
+        _, discovered_urls, next_pages = parse_catalog_page(result.html, source_url=url, storage=storage)
+        total_discovered += len(discovered_urls)
+        for next_url in next_pages:
+            if next_url not in seen:
+                frontier.append(next_url)
+        if max_discover and total_discovered >= max_discover:
+            logger.info("catalog.max_discover.reached", total=total_discovered, max=max_discover)
+            break
+
+
+async def crawl_details(
+    storage: Storage,
+    fetcher: PlaywrightFetcher,
+    robots: RobotsManager,
+    allow_robots_bypass: bool = False,
+) -> None:
+    pending = storage.get_pages_by_type(PAGE_TYPE_DETAIL, parse_status=PARSE_PENDING)
+    logger.info("detail.queue", pending=len(pending))
+    for page in pending:
+        url = page["url"]
+        allowed = allow_robots_bypass or robots.is_allowed(url)
+        if not allowed:
+            logger.warning("detail.fetch.disallowed", url=url)
+            continue
+        if allow_robots_bypass:
+            logger.warning("detail.fetch.disallowed.bypassed", url=url)
+        result = await fetcher.fetch(url, page_type=PAGE_TYPE_DETAIL)
+        storage.upsert_page(result.record)
+        if result.error or not result.html:
+            logger.error("detail.fetch.failed", url=url, error=result.error)
+            continue
+        parse_detail_page(result.html, url=url, storage=storage)
+
+
+def main(argv: Optional[list[str]] = None) -> None:
+    parser = argparse.ArgumentParser(description="Crawler pipeline")
+    parser.add_argument("--mode", choices=["crawl_all", "discover", "details", "export"], default="crawl_all")
+    parser.add_argument("--config", type=str, default=os.environ.get("CONFIG_PATH", "configs/config.yaml"))
+    parser.add_argument("--parquet", type=str, default="data/catalog.parquet")
+    parser.add_argument("--jsonl", type=str, default="data/catalog.jsonl")
+    parser.add_argument(
+        "--max-discover",
+        type=int,
+        default=None,
+        help="Limit number of detail URLs discovered (for smoke tests)",
+    )
+    parser.add_argument(
+        "--limit-export",
+        type=int,
+        default=None,
+        help="Limit number of rows exported (for smoke tests)",
+    )
+    parser.add_argument(
+        "--allow-robots-bypass",
+        action="store_true",
+        help="Bypass robots.txt disallow (for testing; use responsibly)",
+    )
+    args = parser.parse_args(argv)
+
+    config = load_config(args.config)
+    configure_logging(config.get("app", {}).get("log_level", "INFO"))
+    crawler_cfg = config.get("crawler", {})
+    rate_limiter = RateLimiter(
+        base_delay=float(os.environ.get("REQUEST_DELAY_SECONDS", crawler_cfg.get("request_delay_seconds", 1.5))),
+        jitter=float(os.environ.get("JITTER_SECONDS", crawler_cfg.get("jitter_seconds", 0.5))),
+    )
+    user_agent = os.environ.get("USER_AGENT", crawler_cfg.get("user_agent"))
+    start_url = os.environ.get("START_URL", crawler_cfg.get("start_url"))
+    max_retries = int(os.environ.get("MAX_RETRIES", crawler_cfg.get("max_retries", 3)))
+    sqlite_path = crawler_cfg.get("sqlite_path", "data/crawler.db")
+    allow_bypass = args.allow_robots_bypass or os.environ.get("ALLOW_ROBOTS_BYPASS", "").lower() in {"1", "true", "yes"}
+
+    storage = Storage(sqlite_path)
+    robots = RobotsManager(robots_url="https://www.shl.com/robots.txt", user_agent=user_agent)
+    robots.load()
+
+    async def _runner():
+        async with PlaywrightFetcher(user_agent=user_agent, rate_limiter=rate_limiter, max_retries=max_retries) as fetcher:
+            if args.mode in {"crawl_all", "discover"}:
+                await crawl_catalog(start_url, storage, fetcher, robots, max_discover=args.max_discover, allow_robots_bypass=allow_bypass)
+            if args.mode in {"crawl_all", "details"}:
+                await crawl_details(storage, fetcher, robots, allow_robots_bypass=allow_bypass)
+
+    if args.mode in {"crawl_all", "discover", "details"}:
+        asyncio.run(_runner())
+
+    if args.mode == "export":
+        export_catalog(
+            storage,
+            parquet_path=args.parquet,
+            jsonl_path=args.jsonl,
+            limit=args.limit_export,
+            min_count=1 if args.limit_export else 377,
+        )
+
+
+if __name__ == "__main__":
+    main()

crawler/storage.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional
+
+import structlog
+
+from crawler.utils import canonicalize_url, make_assessment_id, now_iso
+
+logger = structlog.get_logger(__name__)
+
+
+PAGE_TYPE_CATALOG = "CATALOG"
+PAGE_TYPE_DETAIL = "DETAIL"
+
+PARSE_PENDING = "PENDING"
+PARSE_PARSED = "PARSED"
+PARSE_FAILED = "FAILED"
+
+
+@dataclass
+class PageRecord:
+    url: str
+    page_type: str
+    http_status: Optional[int] = None
+    html: Optional[str] = None
+    error: Optional[str] = None
+    retry_count: int = 0
+    parse_status: str = PARSE_PENDING
+
+
+class Storage:
+    def __init__(self, db_path: str) -> None:
+        self.db_path = db_path
+        Path(db_path).parent.mkdir(parents=True, exist_ok=True)
+        self.conn = sqlite3.connect(self.db_path)
+        self.conn.row_factory = sqlite3.Row
+        self.ensure_schema()
+
+    def ensure_schema(self) -> None:
+        logger.info("storage.schema.ensure", db_path=self.db_path)
+        cur = self.conn.cursor()
+        cur.execute(
+            """
+            CREATE TABLE IF NOT EXISTS pages (
+                url TEXT PRIMARY KEY,
+                url_canonical TEXT UNIQUE,
+                page_type TEXT,
+                http_status INTEGER,
+                fetched_at TEXT,
+                html TEXT,
+                error TEXT,
+                retry_count INTEGER DEFAULT 0,
+                parse_status TEXT DEFAULT 'PENDING'
+            )
+            """
+        )
+        cur.execute(
+            """
+            CREATE TABLE IF NOT EXISTS assessments (
+                assessment_id TEXT PRIMARY KEY,
+                url TEXT UNIQUE,
+                name TEXT,
+                description TEXT,
+                test_type TEXT,
+                test_type_full TEXT,
+                remote_support INTEGER,
+                adaptive_support INTEGER,
+                duration_minutes INTEGER,
+                job_levels TEXT,
+                languages TEXT,
+                downloads TEXT,
+                source_catalog_page TEXT,
+                discovered_at TEXT,
+                last_updated_at TEXT
+            )
+            """
+        )
+        cur.execute(
+            """
+            CREATE TABLE IF NOT EXISTS crawl_meta (
+                run_id TEXT,
+                started_at TEXT,
+                finished_at TEXT,
+                total_catalog_pages INTEGER,
+                total_detail_pages INTEGER,
+                individual_assessment_count INTEGER,
+                notes TEXT
+            )
+            """
+        )
+        self.conn.commit()
+
+    def upsert_page(self, record: PageRecord) -> None:
+        canonical = canonicalize_url(record.url)
+        logger.debug("storage.page.upsert", url=record.url, page_type=record.page_type)
+        self.conn.execute(
+            """
+            INSERT INTO pages (url, url_canonical, page_type, http_status, fetched_at, html, error, retry_count, parse_status)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(url) DO UPDATE SET
+                page_type=excluded.page_type,
+                http_status=excluded.http_status,
+                fetched_at=excluded.fetched_at,
+                html=excluded.html,
+                error=excluded.error,
+                retry_count=excluded.retry_count,
+                parse_status=excluded.parse_status
+            """,
+            (
+                record.url,
+                canonical,
+                record.page_type,
+                record.http_status,
+                now_iso(),
+                record.html,
+                record.error,
+                record.retry_count,
+                record.parse_status,
+            ),
+        )
+        self.conn.commit()
+
+    def update_parse_status(self, url: str, status: str) -> None:
+        self.conn.execute("UPDATE pages SET parse_status=? WHERE url=?", (status, url))
+        self.conn.commit()
+
+    def get_pages_by_type(self, page_type: str, parse_status: Optional[str] = None) -> List[sqlite3.Row]:
+        cur = self.conn.cursor()
+        if parse_status:
+            cur.execute(
+                "SELECT * FROM pages WHERE page_type=? AND parse_status=? ORDER BY url", (page_type, parse_status)
+            )
+        else:
+            cur.execute("SELECT * FROM pages WHERE page_type=? ORDER BY url", (page_type,))
+        return cur.fetchall()
+
+    def upsert_assessment(self, data: Dict[str, Any]) -> None:
+        url = data["url"]
+        assessment_id = data.get("assessment_id") or make_assessment_id(url)
+        data = {**data, "assessment_id": assessment_id}
+        downloads = data.get("downloads")
+        if downloads is not None and not isinstance(downloads, str):
+            downloads = json.dumps(downloads)
+        job_levels = data.get("job_levels")
+        if isinstance(job_levels, (list, tuple)):
+            job_levels = json.dumps(job_levels)
+        languages = data.get("languages")
+        if isinstance(languages, (list, tuple)):
+            languages = json.dumps(languages)
+
+        logger.debug("storage.assessment.upsert", url=url)
+        self.conn.execute(
+            """
+            INSERT INTO assessments (
+                assessment_id, url, name, description, test_type, test_type_full, remote_support, adaptive_support,
+                duration_minutes, job_levels, languages, downloads, source_catalog_page, discovered_at, last_updated_at
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(assessment_id) DO UPDATE SET
+                url=excluded.url,
+                name=COALESCE(excluded.name, assessments.name),
+                description=COALESCE(excluded.description, assessments.description),
+                test_type=COALESCE(excluded.test_type, assessments.test_type),
+                test_type_full=COALESCE(excluded.test_type_full, assessments.test_type_full),
+                remote_support=COALESCE(excluded.remote_support, assessments.remote_support),
+                adaptive_support=COALESCE(excluded.adaptive_support, assessments.adaptive_support),
+                duration_minutes=COALESCE(excluded.duration_minutes, assessments.duration_minutes),
+                job_levels=COALESCE(excluded.job_levels, assessments.job_levels),
+                languages=COALESCE(excluded.languages, assessments.languages),
+                downloads=COALESCE(excluded.downloads, assessments.downloads),
+                source_catalog_page=COALESCE(excluded.source_catalog_page, assessments.source_catalog_page),
+                last_updated_at=excluded.last_updated_at
+            """,
+            (
+                data["assessment_id"],
+                url,
+                data.get("name"),
+                data.get("description"),
+                data.get("test_type"),
+                data.get("test_type_full"),
+                data.get("remote_support"),
+                data.get("adaptive_support"),
+                data.get("duration_minutes"),
+                job_levels,
+                languages,
+                downloads,
+                data.get("source_catalog_page"),
+                data.get("discovered_at") or now_iso(),
+                data.get("last_updated_at") or now_iso(),
+            ),
+        )
+        self.conn.commit()
+
+    def fetch_assessments(self) -> List[sqlite3.Row]:
+        cur = self.conn.cursor()
+        cur.execute("SELECT * FROM assessments ORDER BY name")
+        return cur.fetchall()
+
+    def count_assessments(self) -> int:
+        cur = self.conn.cursor()
+        cur.execute("SELECT COUNT(*) FROM assessments")
+        return cur.fetchone()[0]
+
+    def close(self) -> None:
+        self.conn.close()

crawler/utils.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import hashlib
+import random
+import time
+import urllib.parse
+from datetime import datetime, timezone
+from typing import Iterable
+
+
+def canonicalize_url(url: str) -> str:
+    """Normalize URL by stripping fragments/query trackers and trailing slashes."""
+    parsed = urllib.parse.urlparse(url)
+    query = urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)
+    filtered_query = [(k, v) for k, v in query if not k.lower().startswith("utm_")]
+    cleaned_query = urllib.parse.urlencode(filtered_query, doseq=True)
+    path = parsed.path if parsed.path != "/" else ""
+    # Keep trailing slash for non-root paths to avoid 404s on detail pages.
+    if path and not path.endswith("/"):
+        path = path
+    normalized = parsed._replace(query=cleaned_query, fragment="", path=path).geturl()
+    return normalized or url
+
+
+def make_assessment_id(url: str) -> str:
+    """Deterministic ID from canonical URL."""
+    canonical = canonicalize_url(url)
+    return hashlib.sha1(canonical.encode("utf-8")).hexdigest()
+
+
+def now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+class RateLimiter:
+    """Coarse rate limiter with jitter to respect polite crawling."""
+
+    def __init__(self, base_delay: float, jitter: float) -> None:
+        self.base_delay = base_delay
+        self.jitter = jitter
+        self._last_ts = 0.0
+
+    def sleep(self) -> None:
+        now = time.monotonic()
+        elapsed = now - self._last_ts
+        delay = self.base_delay + random.uniform(0, self.jitter)
+        if elapsed < delay:
+            time.sleep(delay - elapsed)
+        self._last_ts = time.monotonic()
+
+
+def batched(iterable: Iterable, size: int):
+    """Yield fixed-size batches from an iterable."""
+    batch = []
+    for item in iterable:
+        batch.append(item)
+        if len(batch) == size:
+            yield batch
+            batch = []
+    if batch:
+        yield batch

docker-compose.yml

@@ -0,0 +1,26 @@
+version: "3.9"
+
+services:
+  api:
+    build: .
+    ports:
+      - "8000:8000"
+    environment:
+      LLM_MODEL: Qwen/Qwen2.5-1.5B-Instruct
+      HF_HOME: /cache/hf
+    volumes:
+      - ./data:/app/data:ro
+      - ./models:/app/models:ro
+      - hf-cache:/cache/hf
+
+  web:
+    build: ./frontend
+    ports:
+      - "3000:3000"
+    environment:
+      NEXT_PUBLIC_API_BASE: http://api:8000
+    depends_on:
+      - api
+
+volumes:
+  hf-cache:

embeddings/generator.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import List, Tuple
+
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+from data.catalog_loader import make_assessment_id
+from models.embedding_model import EmbeddingModel
+
+
+def generate_embeddings(catalog_path: str, model_name: str, batch_size: int = 32, output_dir: str = "data/embeddings") -> Tuple[np.ndarray, List[str]]:
+    df = pd.read_json(catalog_path, lines=True) if catalog_path.endswith(".jsonl") else pd.read_parquet(catalog_path)
+    if "assessment_id" not in df.columns:
+        if "url" in df.columns:
+            df["assessment_id"] = df["url"].apply(make_assessment_id)
+        else:
+            raise KeyError("assessment_id not found and url missing to derive it.")
+    df = df.sort_values("assessment_id")
+    texts = df["doc_text"].tolist()
+    ids = df["assessment_id"].tolist()
+
+    model = EmbeddingModel(model_name)
+    embeddings: List[np.ndarray] = []
+    start = time.time()
+    for i in tqdm(range(0, len(texts), batch_size), desc="Embedding"):
+        batch = texts[i : i + batch_size]
+        embeds = model.encode(batch, normalize=True, batch_size=batch_size, is_query=False)
+        embeddings.append(embeds)
+    embeddings_arr = np.vstack(embeddings).astype(np.float32)
+
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+    np.save(Path(output_dir) / "embeddings.npy", embeddings_arr)
+    with open(Path(output_dir) / "assessment_ids.json", "w") as f:
+        json.dump(ids, f, indent=2)
+
+    total_time = time.time() - start
+    log = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "model_name": model_name,
+        "num_documents": len(texts),
+        "embedding_dim": embeddings_arr.shape[1],
+        "batch_size": batch_size,
+        "total_time_seconds": total_time,
+        "avg_time_per_doc_ms": (total_time / len(texts) * 1000) if len(texts) else None,
+        "normalized": True,
+        "catalog_path": catalog_path,
+    }
+    with open(Path(output_dir) / "generation_log.json", "w") as f:
+        json.dump(log, f, indent=2)
+    return embeddings_arr, ids
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--catalog", required=True, help="Enriched catalog with doc_text")
+    parser.add_argument("--model", default="sentence-transformers/all-MiniLM-L6-v2")
+    parser.add_argument("--batch-size", type=int, default=32)
+    parser.add_argument("--output-dir", default="data/embeddings")
+    args = parser.parse_args()
+
+    generate_embeddings(args.catalog, args.model, batch_size=args.batch_size, output_dir=args.output_dir)

eval/__init__.py

@@ -0,0 +1 @@
+"""Evaluation datasets, metrics, and experiments."""

eval/compare_runs.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+
+def load_metrics(path: str) -> dict:
+    with open(path) as f:
+        return json.load(f)
+
+
+def compare(run_a: str, run_b: str) -> dict:
+    m_a = load_metrics(Path(run_a) / "metrics.json")
+    m_b = load_metrics(Path(run_b) / "metrics.json")
+    def extract(m):
+        return {
+            "train_r10": m["train"]["recall@10"],
+            "val_r10": m["val"]["recall@10"],
+            "val_mrr10": m["val"]["mrr@10"],
+        }
+    return {"run_a": run_a, "run_b": run_b, "metrics_a": extract(m_a), "metrics_b": extract(m_b)}
+
+
+def main():
+    if len(sys.argv) != 3:
+        print("Usage: python -m eval.compare_runs <run_dir_a> <run_dir_b>")
+        sys.exit(1)
+    result = compare(sys.argv[1], sys.argv[2])
+    print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()

eval/diagnostic_topk.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import pandas as pd
+
+from data.catalog_loader import load_catalog
+from data.train_loader import load_train
+from recommenders.bm25 import BM25Recommender
+from recommenders.vector_recommender import VectorRecommender
+from recommenders.hybrid_rrf import HybridRRFRecommender, HybridRerankRecommender
+from retrieval.vector_index import VectorIndex
+from models.embedding_model import EmbeddingModel
+from rerankers.cross_encoder import CrossEncoderReranker
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Diagnostics: positives coverage in top-N candidates and top-10 rerank.")
+    parser.add_argument("--catalog", default="data/catalog_docs.jsonl")
+    parser.add_argument("--train", required=True)
+    parser.add_argument("--vector-index", required=True)
+    parser.add_argument("--assessment-ids", required=True)
+    parser.add_argument("--model", default="sentence-transformers/all-MiniLM-L6-v2")
+    parser.add_argument("--reranker-model", default="cross-encoder/ms-marco-MiniLM-L-6-v2")
+    parser.add_argument("--topn", type=int, default=200, help="Top-N candidates to inspect")
+    parser.add_argument("--rrf-k", type=int, default=60)
+    parser.add_argument("--output-dir", default="runs/diagnostic_topk")
+    args = parser.parse_args()
+
+    df_catalog, _, id_by_url = load_catalog(args.catalog)
+    with open(args.assessment_ids) as f:
+        ids = json.load(f)
+    index = VectorIndex.load(args.vector_index)
+    embed_model = EmbeddingModel(args.model)
+    vector_rec = VectorRecommender(embed_model, index, df_catalog, ids, k_candidates=args.topn)
+    bm25_rec = BM25Recommender(df_catalog)
+    hybrid = HybridRRFRecommender(bm25_rec, vector_rec, topn_candidates=args.topn, rrf_k=args.rrf_k)
+    reranker = CrossEncoderReranker(model_name=args.reranker_model)
+    hybrid_rerank = HybridRerankRecommender(bm25_rec, vector_rec, reranker, df_catalog, topn_candidates=args.topn, rrf_k=args.rrf_k)
+
+    examples, label_report = load_train(args.train, id_by_url)
+    Path(args.output_dir).mkdir(parents=True, exist_ok=True)
+    Path(args.output_dir, "label_resolution_report.json").write_text(json.dumps(label_report, indent=2))
+
+    rows = []
+    coverage_fail = 0
+    zero_topn = 0
+    zero_top10 = 0
+    for ex in examples:
+        candidates = hybrid.recommend(ex.query, k=args.topn)
+        reranked = hybrid_rerank.recommend(ex.query, k=10)
+        pos_topn = len(set(candidates).intersection(ex.relevant_ids))
+        pos_top10 = len(set(reranked).intersection(ex.relevant_ids))
+        if pos_topn == 0:
+            zero_topn += 1
+        if pos_top10 == 0:
+            zero_top10 += 1
+        if pos_topn == 0:
+            coverage_fail += 1
+        rows.append(
+            {
+                "query": ex.query,
+                "relevant_ids": list(ex.relevant_ids),
+                "pos_in_topn": pos_topn,
+                "pos_in_top10": pos_top10,
+                "candidates": candidates,
+                "reranked_top10": reranked,
+            }
+        )
+
+    summary = {
+        "total_queries": len(examples),
+        "topn": args.topn,
+        "zero_pos_in_topn": zero_topn,
+        "zero_pos_in_top10": zero_top10,
+        "coverage_failures": coverage_fail,
+        "label_match_pct": label_report.get("matched_pct"),
+    }
+    with open(Path(args.output_dir) / "summary.json", "w") as f:
+        json.dump(summary, f, indent=2)
+    pd.DataFrame(rows).to_json(Path(args.output_dir) / "per_query.jsonl", orient="records", lines=True)
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()

eval/metrics.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import Iterable, List, Sequence, Set
+
+
+def recall_at_k(ground_truth: Set[str], preds: Sequence[str], k: int) -> float:
+    if not ground_truth:
+        return 0.0
+    topk = preds[:k]
+    hits = len(ground_truth.intersection(topk))
+    return hits / len(ground_truth)
+
+
+def mrr_at_k(ground_truth: Set[str], preds: Sequence[str], k: int) -> float:
+    if not ground_truth:
+        return 0.0
+    for idx, pid in enumerate(preds[:k], start=1):
+        if pid in ground_truth:
+            return 1.0 / idx
+    return 0.0
+
+
+def mean_metric(queries: Iterable[Set[str]], preds_list: Iterable[Sequence[str]], fn, k: int) -> float:
+    scores = []
+    for g, p in zip(queries, preds_list):
+        scores.append(fn(g, p, k))
+    return sum(scores) / len(scores) if scores else 0.0

eval/run_eval.py

@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List
+
+import pandas as pd
+
+from data.catalog_loader import load_catalog
+from data.train_loader import load_train, save_label_resolution_report
+from eval.metrics import recall_at_k, mrr_at_k
+from recommenders.dummy_random import DummyRandomRecommender
+from recommenders.bm25 import BM25Recommender
+from recommenders.vector_recommender import VectorRecommender
+from recommenders.hybrid_rrf import HybridRRFRecommender, HybridRerankRecommender
+from recommenders.hybrid_rrf_lgbm import HybridRRFLGBMRecommender
+from retrieval.vector_index import VectorIndex
+from models.embedding_model import EmbeddingModel
+from rerankers.cross_encoder import CrossEncoderReranker
+from rerankers.lgbm_reranker import LGBMReranker
+from retrieval.query_rewriter import rewrite_query
+
+
+def split_examples(examples, val_ratio=0.2, seed=42):
+    import random
+
+    rnd = random.Random(seed)
+    shuffled = examples[:]
+    rnd.shuffle(shuffled)
+    cut = int(len(shuffled) * (1 - val_ratio))
+    return shuffled[:cut], shuffled[cut:]
+
+
+def run_eval(catalog_path: str, train_path: str, recommender_name: str, out_dir: str, seed: int = 42):
+    df_catalog, catalog_by_id, id_by_url = load_catalog(catalog_path)
+    examples, label_report = load_train(train_path, id_by_url)
+    save_label_resolution_report(label_report, Path(out_dir) / "label_resolution_report.json")
+
+    train_split, val_split = split_examples(examples, val_ratio=0.2, seed=seed)
+
+    def make_recommender():
+        if recommender_name == "dummy_random":
+            return DummyRandomRecommender(df_catalog["assessment_id"].tolist(), seed=seed)
+        if recommender_name == "bm25":
+            return BM25Recommender(df_catalog)
+        if recommender_name == "vector":
+            # Expect doc_text present in df_catalog and provided index/ids/model via env/args; set below in main()
+            raise RuntimeError("Vector recommender should be constructed in main with index and ids.")
+        raise ValueError(f"Unknown recommender: {recommender_name}")
+
+    recommender = make_recommender()
+
+    def eval_split(split, split_name):
+        preds_list: List[List[str]] = []
+        gt_list: List[set] = []
+        rows = []
+        for ex in split:
+            preds_raw = recommender.recommend(ex.query, k=10)
+            preds = []
+            for pr in preds_raw:
+                if isinstance(pr, str):
+                    preds.append(pr)
+                elif isinstance(pr, dict) and "assessment_id" in pr:
+                    preds.append(pr["assessment_id"])
+            preds = preds[:10]
+            preds_list.append(preds)
+            gt_list.append(ex.relevant_ids)
+            hits = len(set(preds).intersection(ex.relevant_ids))
+            rows.append(
+                {
+                    "query": ex.query,
+                    "relevant_ids": list(ex.relevant_ids),
+                    "predicted_ids": preds,
+                    "hits": hits,
+                }
+            )
+        recall10 = sum(recall_at_k(g, p, 10) for g, p in zip(gt_list, preds_list)) / len(gt_list) if gt_list else 0.0
+        recall5 = sum(recall_at_k(g, p, 5) for g, p in zip(gt_list, preds_list)) / len(gt_list) if gt_list else 0.0
+        mrr10 = sum(mrr_at_k(g, p, 10) for g, p in zip(gt_list, preds_list)) / len(gt_list) if gt_list else 0.0
+        return recall10, recall5, mrr10, rows
+
+    train_r10, train_r5, train_mrr10, train_rows = eval_split(train_split, "train")
+    val_r10, val_r5, val_mrr10, val_rows = eval_split(val_split, "val")
+
+    Path(out_dir).mkdir(parents=True, exist_ok=True)
+    metrics = {
+        "recommender": recommender_name,
+        "label_match_pct": label_report.get("matched_pct"),
+        "train": {"recall@10": train_r10, "recall@5": train_r5, "mrr@10": train_mrr10, "n": len(train_split)},
+        "val": {"recall@10": val_r10, "recall@5": val_r5, "mrr@10": val_mrr10, "n": len(val_split)},
+    }
+    with open(Path(out_dir) / "metrics.json", "w") as f:
+        json.dump(metrics, f, indent=2)
+    pd.DataFrame(train_rows + val_rows).to_json(Path(out_dir) / "per_query_results.jsonl", orient="records", lines=True)
+    worst = sorted(val_rows, key=lambda r: r["hits"])[:10]
+    pd.DataFrame(worst).to_csv(Path(out_dir) / "worst_queries.csv", index=False)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--catalog", default="data/catalog.jsonl")
+    parser.add_argument("--train", required=True)
+    parser.add_argument("--recommender", default="dummy_random")
+    parser.add_argument("--out-dir", default=None)
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--vector-index", type=str, help="Path to FAISS index (for recommender=vector/hybrid_rrf)")
+    parser.add_argument("--assessment-ids", type=str, help="Path to assessment_ids.json aligned with embeddings/index")
+    parser.add_argument("--model", type=str, default="sentence-transformers/all-MiniLM-L6-v2", help="Embedding model for vector recommender")
+    parser.add_argument("--topn-candidates", type=int, default=200, help="Top-N candidates to retrieve before fusion/rerank")
+    parser.add_argument("--rrf-k", type=int, default=60, help="RRF smoothing constant")
+    parser.add_argument("--reranker-model", type=str, default="cross-encoder/ms-marco-MiniLM-L-6-v2", help="Cross-encoder model for reranking")
+    parser.add_argument("--lgbm-model", type=str, help="Path to trained LGBM model (for hybrid_rrf_lgbm)")
+    parser.add_argument("--lgbm-features", type=str, help="Path to feature_schema.json for LGBM reranker")
+    parser.add_argument("--use-rewriter", action="store_true", help="Rewrite queries before retrieval/rerank.")
+    parser.add_argument("--vocab", type=str, help="Optional vocab JSON for rewriter boosts.")
+    args = parser.parse_args()
+
+    run_id = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
+    out_dir = args.out_dir or f"runs/{run_id}_{args.recommender}"
+    if args.recommender in {"vector", "hybrid_rrf", "hybrid_rrf_rerank", "hybrid_rrf_lgbm"}:
+        if not args.vector_index or not args.assessment_ids:
+            raise ValueError("Vector/hybrid recommender requires --vector-index and --assessment-ids")
+        df_catalog, _, id_by_url = load_catalog(args.catalog)
+        with open(args.assessment_ids) as f:
+            ids = json.load(f)
+        index = VectorIndex.load(args.vector_index)
+        embed_model = EmbeddingModel(args.model)
+        examples, label_report = load_train(args.train, id_by_url)
+        Path(out_dir).mkdir(parents=True, exist_ok=True)
+        save_label_resolution_report(label_report, Path(out_dir) / "label_resolution_report.json")
+        vocab = {}
+        if args.use_rewriter and args.vocab:
+            with open(args.vocab) as f:
+                vocab = json.load(f)
+
+        train_split, val_split = split_examples(examples, val_ratio=0.2, seed=args.seed)
+        vector_rec = VectorRecommender(embed_model, index, df_catalog, ids, k_candidates=args.topn_candidates)
+        if args.recommender == "vector":
+            recommender = vector_rec
+        elif args.recommender == "hybrid_rrf":
+            bm25_rec = BM25Recommender(df_catalog)
+            recommender = HybridRRFRecommender(bm25_rec, vector_rec, topn_candidates=args.topn_candidates, rrf_k=args.rrf_k)
+        elif args.recommender == "hybrid_rrf_rerank":
+            bm25_rec = BM25Recommender(df_catalog)
+            reranker = CrossEncoderReranker(model_name=args.reranker_model)
+            recommender = HybridRerankRecommender(
+                bm25_rec,
+                vector_rec,
+                reranker,
+                df_catalog,
+                topn_candidates=args.topn_candidates,
+                rrf_k=args.rrf_k,
+            )
+        else:
+            if not args.lgbm_model or not args.lgbm_features:
+                raise ValueError("hybrid_rrf_lgbm requires --lgbm-model and --lgbm-features")
+            bm25_rec = BM25Recommender(df_catalog)
+            feature_cols = json.load(open(args.lgbm_features))
+            if isinstance(feature_cols, dict) and "features" in feature_cols:
+                feature_cols = feature_cols["features"]
+            recommender = HybridRRFLGBMRecommender(
+                bm25_rec,
+                vector_rec,
+                lgbm_model_path=args.lgbm_model,
+                feature_cols=feature_cols,
+                catalog_df=df_catalog,
+                topn_candidates=args.topn_candidates,
+                rrf_k=args.rrf_k,
+            )
+
+        def eval_split(split, split_name):
+            preds_list = []
+            gt_list = []
+            rows = []
+            for ex in split:
+                retrieval_query = ex.query
+                rerank_query = ex.query
+                if args.use_rewriter:
+                    rw = rewrite_query(ex.query, catalog_vocab=vocab)
+                    retrieval_query = rw.retrieval_query
+                    rerank_query = rw.rerank_query
+                if args.recommender == "hybrid_rrf_rerank":
+                    preds_raw = recommender.recommend(retrieval_query, k=10, rerank_query=rerank_query)
+                else:
+                    preds_raw = recommender.recommend(retrieval_query, k=10)
+                preds = []
+                for pr in preds_raw:
+                    if isinstance(pr, str):
+                        preds.append(pr)
+                    elif isinstance(pr, dict) and "assessment_id" in pr:
+                        preds.append(pr["assessment_id"])
+                preds = preds[:10]
+                preds_list.append(preds)
+                gt_list.append(ex.relevant_ids)
+                hits = len(set(preds).intersection(ex.relevant_ids))
+                rows.append(
+                    {
+                        "query": ex.query,
+                        "relevant_ids": list(ex.relevant_ids),
+                        "predicted_ids": preds,
+                        "hits": hits,
+                    }
+                )
+            recall10 = sum(recall_at_k(g, p, 10) for g, p in zip(gt_list, preds_list)) / len(gt_list) if gt_list else 0.0
+            recall5 = sum(recall_at_k(g, p, 5) for g, p in zip(gt_list, preds_list)) / len(gt_list) if gt_list else 0.0
+            mrr10 = sum(mrr_at_k(g, p, 10) for g, p in zip(gt_list, preds_list)) / len(gt_list) if gt_list else 0.0
+            return recall10, recall5, mrr10, rows
+
+        train_r10, train_r5, train_mrr10, train_rows = eval_split(train_split, "train")
+        val_r10, val_r5, val_mrr10, val_rows = eval_split(val_split, "val")
+        metrics = {
+            "recommender": args.recommender,
+            "label_match_pct": label_report.get("matched_pct"),
+            "train": {"recall@10": train_r10, "recall@5": train_r5, "mrr@10": train_mrr10, "n": len(train_split)},
+            "val": {"recall@10": val_r10, "recall@5": val_r5, "mrr@10": val_mrr10, "n": len(val_split)},
+            "config": {
+                "topn_candidates": args.topn_candidates,
+                "rrf_k": args.rrf_k,
+                "model": args.model,
+                "index": args.vector_index,
+            },
+        }
+        with open(Path(out_dir) / "metrics.json", "w") as f:
+            json.dump(metrics, f, indent=2)
+        pd.DataFrame(train_rows + val_rows).to_json(Path(out_dir) / "per_query_results.jsonl", orient="records", lines=True)
+        worst = sorted(val_rows, key=lambda r: r["hits"])[:10]
+        pd.DataFrame(worst).to_csv(Path(out_dir) / "worst_queries.csv", index=False)
+        print(f"Run saved to {out_dir}")
+    else:
+        run_eval(args.catalog, args.train, args.recommender, out_dir, seed=args.seed)
+        print(f"Run saved to {out_dir}")
+
+
+if __name__ == "__main__":
+    main()

frontend/.dockerignore

@@ -0,0 +1,9 @@
+.env
+.env.*
+.next
+node_modules
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.turbo
+.vercel

frontend/Dockerfile

@@ -0,0 +1,19 @@
+FROM node:20-alpine AS deps
+WORKDIR /app
+COPY package*.json ./
+RUN npm ci
+
+FROM deps AS builder
+WORKDIR /app
+COPY . .
+RUN npm run build
+
+FROM node:20-alpine AS runner
+WORKDIR /app
+ENV NODE_ENV=production
+COPY --from=builder /app/.next ./.next
+COPY --from=builder /app/public ./public
+COPY --from=builder /app/package*.json ./
+RUN npm ci --omit=dev
+EXPOSE 3000
+CMD ["npm", "start"]

frontend/index.html

@@ -0,0 +1,43 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>Assessment Recommender</title>
+  <style>
+    body { font-family: Arial, sans-serif; margin: 24px; background: #f7f7f7; }
+    .container { max-width: 960px; margin: 0 auto; background: #fff; padding: 20px; border-radius: 8px; box-shadow: 0 2px 6px rgba(0,0,0,0.1); }
+    textarea, input { width: 100%; }
+    label { display: block; margin: 8px 0 4px; font-weight: 600; }
+    button { margin-top: 12px; padding: 10px 16px; cursor: pointer; }
+    pre { background: #111; color: #0f0; padding: 12px; border-radius: 4px; overflow: auto; max-height: 420px; }
+    .row { display: flex; gap: 8px; align-items: center; }
+    .row input[type="checkbox"] { width: auto; }
+  </style>
+</head>
+<body>
+  <div class="container">
+    <h2>Assessment Recommender</h2>
+    <label for="api">API base URL</label>
+    <input id="api" type="text" placeholder="http://localhost:8000" />
+
+    <label for="query">Query</label>
+    <textarea id="query" rows="4" placeholder="Enter your query..."></textarea>
+
+    <label for="clarification">Clarification (optional)</label>
+    <input id="clarification" type="text" placeholder="If a clarification question was asked, answer here" />
+
+    <div class="row">
+      <input id="verbose" type="checkbox" />
+      <label for="verbose" style="margin: 0; font-weight: 400;">Verbose (debug)</label>
+    </div>
+
+    <button id="submit">Submit</button>
+
+    <h3>Response</h3>
+    <pre id="output">Awaiting input...</pre>
+  </div>
+
+  <script type="module" src="/static/main.js"></script>
+</body>
+</html>

frontend/next-env.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/basic-features/typescript for more information.

frontend/next.config.mjs

@@ -0,0 +1,8 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  reactStrictMode: false,
+  // Static export for hosting on static platforms (Render static site, etc.)
+  output: "export"
+};
+
+export default nextConfig;

frontend/out/404.html

@@ -0,0 +1 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/_next/static/css/e2d6b4bec72e8797.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-879f858537244e02.js"/><script src="/_next/static/chunks/fd9d1056-0eb575322ff5015c.js" async=""></script><script src="/_next/static/chunks/23-02b97631d99e6f05.js" async=""></script><script src="/_next/static/chunks/main-app-df951a18dbec0e17.js" async=""></script><title>404: This page could not be found.</title><title>SHL Assessment Recommender</title><meta name="description" content="Chat + recommendations UI powered by FastAPI backend"/><script src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js" noModule=""></script></head><body class="bg-slate-100"><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding:0 23px 0 0;font-size:24px;font-weight:500;vertical-align:top;line-height:49px">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:49px;margin:0">This page could not be found.</h2></div></div></div><script src="/_next/static/chunks/webpack-879f858537244e02.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/_next/static/css/e2d6b4bec72e8797.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"2:I[5751,[],\"\"]\n4:I[9275,[],\"\"]\n5:I[1343,[],\"\"]\nb:I[6130,[],\"\"]\n6:{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"}\n7:{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"}\n8:{\"display\":\"inline-block\"}\n9:{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0}\nc:[]\n"])</script><script>self.__next_f.push([1,"0:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/e2d6b4bec72e8797.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}]],[\"$\",\"$L2\",null,{\"buildId\":\"tvK2sxvsuv7CccL1KsVpv\",\"assetPrefix\":\"\",\"initialCanonicalUrl\":\"/_not-found\",\"initialTree\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{}]}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{},[[\"$L3\",[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]]],null],null]},[\"$\",\"$L4\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"/_not-found\",\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L5\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\",\"styles\":null}],null]},[[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"bg-slate-100\",\"children\":[\"$\",\"$L4\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L5\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":\"$6\",\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":\"$7\",\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":\"$8\",\"children\":[\"$\",\"h2\",null,{\"style\":\"$9\",\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null],null],\"couldBeIntercepted\":false,\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"SHL Assessment Recommender\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"Chat + recommendations UI powered by FastAPI backend\"}]]\n3:null\n"])</script></body></html>

frontend/out/_next/static/chunks/app/_not-found/page-a99a188ec9244b3f.js

@@ -0,0 +1 @@
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[409],{7589:function(e,t,n){(window.__NEXT_P=window.__NEXT_P||[]).push(["/_not-found/page",function(){return n(5457)}])},5457:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),Object.defineProperty(t,"default",{enumerable:!0,get:function(){return s}}),n(9920);let i=n(7437);n(2265);let o={fontFamily:'system-ui,"Segoe UI",Roboto,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji"',height:"100vh",textAlign:"center",display:"flex",flexDirection:"column",alignItems:"center",justifyContent:"center"},l={display:"inline-block"},r={display:"inline-block",margin:"0 20px 0 0",padding:"0 23px 0 0",fontSize:24,fontWeight:500,verticalAlign:"top",lineHeight:"49px"},d={fontSize:14,fontWeight:400,lineHeight:"49px",margin:0};function s(){return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)("title",{children:"404: This page could not be found."}),(0,i.jsx)("div",{style:o,children:(0,i.jsxs)("div",{children:[(0,i.jsx)("style",{dangerouslySetInnerHTML:{__html:"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}),(0,i.jsx)("h1",{className:"next-error-h1",style:r,children:"404"}),(0,i.jsx)("div",{style:l,children:(0,i.jsx)("h2",{style:d,children:"This page could not be found."})})]})})]})}("function"==typeof t.default||"object"==typeof t.default&&null!==t.default)&&void 0===t.default.__esModule&&(Object.defineProperty(t.default,"__esModule",{value:!0}),Object.assign(t.default,t),e.exports=t.default)}},function(e){e.O(0,[971,23,744],function(){return e(e.s=7589)}),_N_E=e.O()}]);

frontend/out/_next/static/chunks/app/layout-fc95adeb217fd9c8.js

@@ -0,0 +1 @@
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{3711:function(n,e,u){Promise.resolve().then(u.t.bind(u,3054,23))},3054:function(){}},function(n){n.O(0,[141,971,23,744],function(){return n(n.s=3711)}),_N_E=n.O()}]);

frontend/out/_next/static/chunks/app/page-73ea6ec0ec8fa438.js

@@ -0,0 +1,16 @@
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[931],{5531:function(e,t,s){Promise.resolve().then(s.bind(s,9306))},9306:function(e,t,s){"use strict";s.r(t),s.d(t,{default:function(){return M}});var a=s(7437),l=s(2265);/**
+ * @license lucide-react v0.561.0 - ISC
+ *
+ * This source code is licensed under the ISC license.
+ * See the LICENSE file in the root directory of this source tree.
+ */let r=e=>e.replace(/([a-z0-9])([A-Z])/g,"$1-$2").toLowerCase(),n=e=>e.replace(/^([A-Z])|[\s-_]+(\w)/g,(e,t,s)=>s?s.toUpperCase():t.toLowerCase()),i=e=>{let t=n(e);return t.charAt(0).toUpperCase()+t.slice(1)},o=function(){for(var e=arguments.length,t=Array(e),s=0;s<e;s++)t[s]=arguments[s];return t.filter((e,t,s)=>!!e&&""!==e.trim()&&s.indexOf(e)===t).join(" ").trim()},d=e=>{for(let t in e)if(t.startsWith("aria-")||"role"===t||"title"===t)return!0};/**
+ * @license lucide-react v0.561.0 - ISC
+ *
+ * This source code is licensed under the ISC license.
+ * See the LICENSE file in the root directory of this source tree.
+ */var c={xmlns:"http://www.w3.org/2000/svg",width:24,height:24,viewBox:"0 0 24 24",fill:"none",stroke:"currentColor",strokeWidth:2,strokeLinecap:"round",strokeLinejoin:"round"};/**
+ * @license lucide-react v0.561.0 - ISC
+ *
+ * This source code is licensed under the ISC license.
+ * See the LICENSE file in the root directory of this source tree.
+ */let u=(0,l.forwardRef)((e,t)=>{let{color:s="currentColor",size:a=24,strokeWidth:r=2,absoluteStrokeWidth:n,className:i="",children:u,iconNode:m,...x}=e;return(0,l.createElement)("svg",{ref:t,...c,width:a,height:a,stroke:s,strokeWidth:n?24*Number(r)/Number(a):r,className:o("lucide",i),...!u&&!d(x)&&{"aria-hidden":"true"},...x},[...m.map(e=>{let[t,s]=e;return(0,l.createElement)(t,s)}),...Array.isArray(u)?u:[u]])}),m=(e,t)=>{let s=(0,l.forwardRef)((s,a)=>{let{className:n,...d}=s;return(0,l.createElement)(u,{ref:a,iconNode:t,className:o("lucide-".concat(r(i(e))),"lucide-".concat(e),n),...d})});return s.displayName=i(e),s},x=m("refresh-cw",[["path",{d:"M3 12a9 9 0 0 1 9-9 9.75 9.75 0 0 1 6.74 2.74L21 8",key:"v9h5vc"}],["path",{d:"M21 3v5h-5",key:"1q7to0"}],["path",{d:"M21 12a9 9 0 0 1-9 9 9.75 9.75 0 0 1-6.74-2.74L3 16",key:"3uifl3"}],["path",{d:"M8 16H3v5",key:"1cv678"}]]),h=m("send",[["path",{d:"M14.536 21.686a.5.5 0 0 0 .937-.024l6.5-19a.496.496 0 0 0-.635-.635l-19 6.5a.5.5 0 0 0-.024.937l7.93 3.18a2 2 0 0 1 1.112 1.11z",key:"1ffxy3"}],["path",{d:"m21.854 2.147-10.94 10.939",key:"12cjpa"}]]),p=m("bug",[["path",{d:"M12 20v-9",key:"1qisl0"}],["path",{d:"M14 7a4 4 0 0 1 4 4v3a6 6 0 0 1-12 0v-3a4 4 0 0 1 4-4z",key:"uouzyp"}],["path",{d:"M14.12 3.88 16 2",key:"qol33r"}],["path",{d:"M21 21a4 4 0 0 0-3.81-4",key:"1b0z45"}],["path",{d:"M21 5a4 4 0 0 1-3.55 3.97",key:"5cxbf6"}],["path",{d:"M22 13h-4",key:"1jl80f"}],["path",{d:"M3 21a4 4 0 0 1 3.81-4",key:"1fjd4g"}],["path",{d:"M3 5a4 4 0 0 0 3.55 3.97",key:"1d7oge"}],["path",{d:"M6 13H2",key:"82j7cp"}],["path",{d:"m8 2 1.88 1.88",key:"fmnt4t"}],["path",{d:"M9 7.13V6a3 3 0 1 1 6 0v1.13",key:"1vgav8"}]]),v=m("settings",[["path",{d:"M9.671 4.136a2.34 2.34 0 0 1 4.659 0 2.34 2.34 0 0 0 3.319 1.915 2.34 2.34 0 0 1 2.33 4.033 2.34 2.34 0 0 0 0 3.831 2.34 2.34 0 0 1-2.33 4.033 2.34 2.34 0 0 0-3.319 1.915 2.34 2.34 0 0 1-4.659 0 2.34 2.34 0 0 0-3.32-1.915 2.34 2.34 0 0 1-2.33-4.033 2.34 2.34 0 0 0 0-3.831A2.34 2.34 0 0 1 6.35 6.051a2.34 2.34 0 0 0 3.319-1.915",key:"1i5ecw"}],["circle",{cx:"12",cy:"12",r:"3",key:"1v7zrd"}]]),f=m("funnel",[["path",{d:"M10 20a1 1 0 0 0 .553.895l2 1A1 1 0 0 0 14 21v-7a2 2 0 0 1 .517-1.341L21.74 4.67A1 1 0 0 0 21 3H3a1 1 0 0 0-.742 1.67l7.225 7.989A2 2 0 0 1 10 14z",key:"sc7q7i"}]]),g=m("search",[["path",{d:"m21 21-4.34-4.34",key:"14j7rj"}],["circle",{cx:"11",cy:"11",r:"8",key:"4ej97u"}]]),b=m("sliders-horizontal",[["path",{d:"M10 5H3",key:"1qgfaw"}],["path",{d:"M12 19H3",key:"yhmn1j"}],["path",{d:"M14 3v4",key:"1sua03"}],["path",{d:"M16 17v4",key:"1q0r14"}],["path",{d:"M21 12h-9",key:"1o4lsq"}],["path",{d:"M21 19h-5",key:"1rlt1p"}],["path",{d:"M21 5h-7",key:"1oszz2"}],["path",{d:"M8 10v4",key:"tgpxqk"}],["path",{d:"M8 12H3",key:"a7s4jb"}]]),j=m("link",[["path",{d:"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71",key:"1cjeqo"}],["path",{d:"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71",key:"19qd67"}]]);async function y(e,t,s){return w("".concat(e.replace(/\/$/,""),"/chat"),t,s)}async function N(e,t,s){return w("".concat(e.replace(/\/$/,""),"/recommend"),t,s)}async function w(e,t,s){let a=new AbortController,l=setTimeout(()=>a.abort(),3e4);try{let l=await fetch(e,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(t),signal:s||a.signal});if(!l.ok){let e=await l.text();throw Error("HTTP ".concat(l.status,": ").concat(e))}return l.json()}finally{clearTimeout(l)}}function k(e,t){let[s,a]=(0,l.useState)(t);return(0,l.useEffect)(()=>{try{let t=localStorage.getItem(e);null!==t&&a(JSON.parse(t))}catch(e){}},[]),(0,l.useEffect)(()=>{try{localStorage.setItem(e,JSON.stringify(s))}catch(e){}},[e,s]),[s,a]}let C=["Java dev + collaboration + 40 minutes","Sales graduate assessment for 60 minutes","Culture fit assessment for COO, 60 minutes"];function M(){var e,t,s;let[r,n]=k("api_base","http://localhost:8000"),[i,o]=k("mode","recommend"),[d,c]=k("verbose",!1),[u,m]=k("llm_model","Qwen/Qwen2.5-1.5B-Instruct"),[w,M]=(0,l.useState)(""),[S,A]=(0,l.useState)(""),[_,z]=(0,l.useState)([]),[L,q]=(0,l.useState)(!1),[E,O]=(0,l.useState)(null),[T,H]=(0,l.useState)({search:"",remote:"any",adaptive:"any",duration:"any",sort:"match"}),R=(0,l.useRef)(null);(0,l.useEffect)(()=>{_.length&&null===E&&O(_.length-1)},[_,E]);let D=null!==E?_[E]:null,I=(null==D?void 0:null===(e=D.response)||void 0===e?void 0:e.recommended_assessments)||(null==D?void 0:null===(t=D.response)||void 0===t?void 0:t.final_results)||[],J=null==D?void 0:null===(s=D.response)||void 0===s?void 0:s.debug,U=(0,l.useMemo)(()=>{let e=[...I],{search:t,remote:s,adaptive:a,duration:l,sort:r}=T;if(t.trim()){let s=t.toLowerCase();e=e.filter(e=>{var t,a,l;return(null===(t=e.name)||void 0===t?void 0:t.toLowerCase().includes(s))||(null===(a=e.description)||void 0===a?void 0:a.toLowerCase().includes(s))||(null===(l=e.test_type)||void 0===l?void 0:l.some(e=>e.toLowerCase().includes(s)))})}return"any"!==s&&(e=e.filter(e=>(e.remote_support||"").toLowerCase()===s.toLowerCase())),"any"!==a&&(e=e.filter(e=>(e.adaptive_support||"").toLowerCase()===a.toLowerCase())),"any"!==l&&(e=e.filter(e=>{let t=e.duration;return null==t?"unknown"===l:"<=20"===l?t<=20:"<=40"===l?t<=40:"<=60"!==l||t<=60})),"short"===r?e.sort((e,t)=>(e.duration||999)-(t.duration||999)):"adaptive"===r&&e.sort((e,t)=>("Yes"===t.adaptive_support?1:0)-("Yes"===e.adaptive_support?1:0)),e},[I,T]),P=async()=>{var e;if(!w.trim())return;q(!0),null===(e=R.current)||void 0===e||e.abort();let t=new AbortController;R.current=t;let s={query:w,verbose:d};S.trim()&&(s.clarification_answer=S.trim()),"recommend"===i&&u&&(s.llm_model=u);let a=crypto.randomUUID(),l=Date.now();z(e=>[...e,{id:a,query:w,response:null,ts:l}]);try{let e="chat"===i?await y(r,s,t.signal):await N(r,s,t.signal);z(t=>t.map(t=>t.id===a?{...t,response:e,error:void 0}:t)),O(_.length),M(""),A("")}catch(e){z(t=>t.map(t=>t.id===a?{...t,error:e.message}:t))}finally{q(!1)}},Y=(0,a.jsxs)("div",{className:"flex items-center justify-between mb-3",children:[(0,a.jsxs)("div",{children:[(0,a.jsx)("h1",{className:"text-3xl font-semibold text-slate-900",children:"SHL Assessment Recommender"}),(0,a.jsx)("p",{className:"text-sm text-slate-600",children:"Chat to get top-10 assessments. Filters and debug on the right."})]}),(0,a.jsxs)("div",{className:"hidden md:flex items-center gap-2 text-xs text-slate-500",children:[(0,a.jsx)(x,{size:16})," Live against FastAPI backend"]})]}),B=(0,a.jsxs)("div",{className:"flex flex-wrap gap-3 text-sm",children:[(0,a.jsxs)("div",{className:"flex items-center gap-2",children:[(0,a.jsx)("label",{className:"font-medium",children:"Mode"}),(0,a.jsxs)("select",{className:"border rounded px-2 py-1",value:i,onChange:e=>o(e.target.value),children:[(0,a.jsx)("option",{value:"recommend",children:"/recommend"}),(0,a.jsx)("option",{value:"chat",children:"/chat"})]})]}),(0,a.jsxs)("div",{className:"flex items-center gap-2",children:[(0,a.jsx)("label",{className:"font-medium",children:"LLM"}),(0,a.jsx)("input",{className:"border rounded px-2 py-1",value:u,onChange:e=>m(e.target.value),placeholder:"Qwen/Qwen2.5-1.5B-Instruct"})]}),(0,a.jsxs)("label",{className:"flex items-center gap-2",children:[(0,a.jsx)("input",{type:"checkbox",checked:d,onChange:e=>c(e.target.checked)}),"Verbose debug"]})]}),Q=(0,a.jsxs)("div",{className:"flex flex-col h-full",children:[(0,a.jsxs)("div",{className:"flex flex-col gap-3 flex-1 overflow-hidden bg-white border rounded-xl shadow-sm p-4",children:[(0,a.jsxs)("div",{className:"flex items-center justify-between",children:[(0,a.jsxs)("div",{className:"text-lg font-semibold flex items-center gap-2",children:[(0,a.jsx)(h,{size:18})," Chat"]}),(0,a.jsx)("button",{onClick:()=>{M(C[0])},className:"text-xs text-blue-600 hover:underline",children:"Use sample"})]}),(0,a.jsxs)("div",{className:"flex gap-2 items-center text-sm",children:[(0,a.jsx)("label",{className:"font-medium min-w-[70px]",children:"API base"}),(0,a.jsx)("input",{className:"border rounded px-2 py-1 w-full",value:r,onChange:e=>n(e.target.value)})]}),(0,a.jsx)("textarea",{className:"border rounded-lg p-3 w-full text-sm min-h-[140px] resize-none focus:ring-2 focus:ring-blue-200",placeholder:"Enter job description or query",value:w,onChange:e=>M(e.target.value),onKeyDown:e=>{"Enter"!==e.key||e.shiftKey||(e.preventDefault(),P())}}),(0,a.jsx)("div",{className:"flex gap-2",children:C.map(e=>(0,a.jsx)("button",{onClick:()=>M(e),className:"text-xs bg-slate-100 hover:bg-slate-200 px-2 py-1 rounded",children:e},e))}),(0,a.jsxs)("div",{className:"flex gap-3 items-center",children:[(0,a.jsx)("input",{className:"border rounded px-2 py-1 text-sm flex-1",placeholder:"Clarification (if asked)",value:S,onChange:e=>A(e.target.value)}),(0,a.jsxs)("button",{onClick:P,disabled:L,className:"bg-blue-600 text-white px-4 py-2 rounded-lg flex items-center gap-2 hover:bg-blue-700 disabled:opacity-60",children:[(0,a.jsx)(h,{size:16})," ",L?"Sending...":"Send"]}),(0,a.jsx)("button",{onClick:()=>c(!d),className:"p-2 border rounded-lg hover:bg-slate-100",title:"Toggle verbose debug",children:(0,a.jsx)(p,{size:16})}),(0,a.jsx)("button",{onClick:()=>o("recommend"===i?"chat":"recommend"),className:"p-2 border rounded-lg hover:bg-slate-100",title:"Toggle endpoint",children:(0,a.jsx)(v,{size:16})})]}),B]}),(0,a.jsxs)("div",{className:"mt-3 bg-white border rounded-xl shadow-sm p-3 text-sm text-slate-600 max-h-48 overflow-auto",children:[(0,a.jsx)("div",{className:"font-semibold mb-2",children:"History"}),0===_.length&&(0,a.jsx)("div",{className:"text-slate-400",children:"No queries yet."}),_.map((e,t)=>(0,a.jsxs)("button",{onClick:()=>O(t),className:"block w-full text-left px-2 py-1 rounded ".concat(t===E?"bg-blue-50 text-blue-700":"hover:bg-slate-100"),children:[(0,a.jsx)("div",{className:"font-medium text-sm truncate",children:e.query}),(0,a.jsx)("div",{className:"text-xs text-slate-500",children:new Date(e.ts).toLocaleTimeString()}),e.error&&(0,a.jsxs)("div",{className:"text-xs text-red-600",children:["Error: ",e.error]})]},e.id))]})]}),$=(0,a.jsxs)("div",{className:"flex flex-col h-full",children:[(0,a.jsxs)("div",{className:"bg-white border rounded-xl shadow-sm p-4 flex flex-col gap-3",children:[(0,a.jsxs)("div",{className:"flex items-center justify-between",children:[(0,a.jsxs)("div",{className:"text-lg font-semibold flex items-center gap-2",children:[(0,a.jsx)(f,{size:18})," Results"]}),(0,a.jsxs)("div",{className:"flex items-center gap-2",children:[(0,a.jsxs)("div",{className:"relative",children:[(0,a.jsx)(g,{className:"absolute left-2 top-2.5 h-4 w-4 text-slate-400"}),(0,a.jsx)("input",{className:"pl-8 pr-3 py-2 border rounded-lg text-sm",placeholder:"Search results",value:T.search,onChange:e=>H(t=>({...t,search:e.target.value}))})]}),(0,a.jsx)(b,{size:16,className:"text-slate-500"})]})]}),(0,a.jsxs)("div",{className:"flex flex-wrap gap-3 text-xs",children:[(0,a.jsxs)("select",{className:"border rounded px-2 py-1",value:T.remote,onChange:e=>H(t=>({...t,remote:e.target.value})),children:[(0,a.jsx)("option",{value:"any",children:"Remote: Any"}),(0,a.jsx)("option",{value:"Yes",children:"Remote: Yes"}),(0,a.jsx)("option",{value:"No",children:"Remote: No"})]}),(0,a.jsxs)("select",{className:"border rounded px-2 py-1",value:T.adaptive,onChange:e=>H(t=>({...t,adaptive:e.target.value})),children:[(0,a.jsx)("option",{value:"any",children:"Adaptive: Any"}),(0,a.jsx)("option",{value:"Yes",children:"Adaptive: Yes"}),(0,a.jsx)("option",{value:"No",children:"Adaptive: No"})]}),(0,a.jsxs)("select",{className:"border rounded px-2 py-1",value:T.duration,onChange:e=>H(t=>({...t,duration:e.target.value})),children:[(0,a.jsx)("option",{value:"any",children:"Duration: Any"}),(0,a.jsx)("option",{value:"<=20",children:"≤ 20 min"}),(0,a.jsx)("option",{value:"<=40",children:"≤ 40 min"}),(0,a.jsx)("option",{value:"<=60",children:"≤ 60 min"}),(0,a.jsx)("option",{value:"unknown",children:"Unknown only"})]}),(0,a.jsxs)("select",{className:"border rounded px-2 py-1",value:T.sort,onChange:e=>H(t=>({...t,sort:e.target.value})),children:[(0,a.jsx)("option",{value:"match",children:"Sort: Best match"}),(0,a.jsx)("option",{value:"short",children:"Sort: Shortest"}),(0,a.jsx)("option",{value:"adaptive",children:"Sort: Adaptive first"})]})]}),(0,a.jsxs)("div",{className:"grid md:grid-cols-2 lg:grid-cols-2 gap-3",children:[0===U.length&&(0,a.jsx)("div",{className:"text-sm text-slate-500",children:"No results yet. Submit a query to see recommendations."}),U.map((e,t)=>{var s;return(0,a.jsxs)("div",{className:"border rounded-xl p-4 shadow-sm hover:shadow-md transition bg-slate-50",children:[(0,a.jsxs)("div",{className:"flex items-start justify-between gap-2",children:[(0,a.jsx)("a",{href:e.url,target:"_blank",rel:"noreferrer",className:"font-semibold text-slate-900 hover:text-blue-600",children:e.name||"Untitled"}),(0,a.jsx)("button",{className:"text-slate-500 hover:text-blue-600",onClick:()=>e.url&&navigator.clipboard.writeText(e.url),children:(0,a.jsx)(j,{size:16})})]}),(0,a.jsxs)("div",{className:"flex flex-wrap gap-2 mt-2",children:[null===(s=e.test_type)||void 0===s?void 0:s.map(e=>(0,a.jsx)("span",{className:"text-[11px] bg-blue-50 text-blue-700 px-2 py-1 rounded-full border border-blue-100",children:e},e)),(0,a.jsx)("span",{className:"text-[11px] bg-slate-100 text-slate-700 px-2 py-1 rounded-full border border-slate-200",children:e.duration?"".concat(e.duration," min"):"Duration unknown"}),(0,a.jsxs)("span",{className:"text-[11px] bg-emerald-50 text-emerald-700 px-2 py-1 rounded-full border border-emerald-100",children:["Remote: ",e.remote_support||"?"]}),(0,a.jsxs)("span",{className:"text-[11px] bg-indigo-50 text-indigo-700 px-2 py-1 rounded-full border border-indigo-100",children:["Adaptive: ",e.adaptive_support||"?"]})]}),(0,a.jsx)("p",{className:"text-sm text-slate-700 mt-2 overflow-hidden text-ellipsis",children:e.description||"No description."})]},t)})]})]}),d&&J&&(0,a.jsxs)("div",{className:"mt-3 bg-white border rounded-xl shadow-sm p-4",children:[(0,a.jsxs)("div",{className:"flex items-center gap-2 text-sm font-semibold mb-2",children:[(0,a.jsx)(p,{size:16})," Debug"]}),(0,a.jsxs)("div",{className:"grid md:grid-cols-2 gap-3 text-xs",children:[(0,a.jsxs)("div",{className:"bg-slate-50 border rounded p-2",children:[(0,a.jsx)("div",{className:"font-semibold mb-1",children:"Plan"}),(0,a.jsx)("pre",{className:"overflow-auto max-h-48 text-slate-700",children:JSON.stringify(J.plan,null,2)})]}),J.fusion&&(0,a.jsxs)("div",{className:"bg-slate-50 border rounded p-2",children:[(0,a.jsx)("div",{className:"font-semibold mb-1",children:"Fusion"}),(0,a.jsx)("pre",{className:"overflow-auto max-h-48 text-slate-700",children:JSON.stringify(J.fusion,null,2)})]}),J.candidates&&(0,a.jsxs)("div",{className:"bg-slate-50 border rounded p-2 col-span-2",children:[(0,a.jsx)("div",{className:"font-semibold mb-1",children:"Top candidates"}),(0,a.jsx)("pre",{className:"overflow-auto max-h-60 text-slate-700",children:JSON.stringify(J.candidates,null,2)})]})]})]})]});return(0,a.jsx)("main",{className:"min-h-screen bg-slate-100",children:(0,a.jsxs)("div",{className:"app-shell py-6",children:[Y,(0,a.jsxs)("div",{className:"grid lg:grid-cols-2 gap-6 mt-4",children:[Q,$]})]})})}}},function(e){e.O(0,[971,23,744],function(){return e(e.s=5531)}),_N_E=e.O()}]);

frontend/out/_next/static/chunks/main-app-df951a18dbec0e17.js

@@ -0,0 +1 @@
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{3332:function(e,n,t){Promise.resolve().then(t.t.bind(t,5751,23)),Promise.resolve().then(t.t.bind(t,6513,23)),Promise.resolve().then(t.t.bind(t,6130,23)),Promise.resolve().then(t.t.bind(t,9275,23)),Promise.resolve().then(t.t.bind(t,5324,23)),Promise.resolve().then(t.t.bind(t,1343,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,23],function(){return n(1028),n(3332)}),_N_E=e.O()}]);