deploy: 59cac6b1dd28

.dockerignore

@@ -0,0 +1,43 @@
+# Hindari bloat build context (yang Docker daemon kirim ke builder)
+
+# Python
+**/__pycache__
+**/*.pyc
+**/*.pyo
+**/.venv
+**/venv
+**/.pytest_cache
+**/.coverage
+**/htmlcov
+**/coverage.xml
+
+# Node
+**/node_modules
+**/dist
+**/.vite
+
+# Env files (NEVER ship)
+**/.env
+**/.env.local
+
+# Git
+.git
+.gitignore
+**/.gitkeep
+
+# IDE / OS
+**/.vscode
+**/.idea
+**/*.swp
+**/.DS_Store
+**/Thumbs.db
+
+# Dev artifacts
+**/.scrape_cache
+**/notebooks/.ipynb_checkpoints
+**/*.log
+
+# Tests + docs (gak perlu di runtime image)
+docs/
+.github/
+**/tests

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/indexes/indobert/faiss.index filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,118 @@
+# ============================================================================
+# Python
+# ============================================================================
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# Jupyter
+.ipynb_checkpoints/
+
+# pytest / coverage
+.coverage
+htmlcov/
+.pytest_cache/
+.tox/
+.nox/
+coverage.xml
+*.cover
+
+# ============================================================================
+# Node / Frontend
+# ============================================================================
+node_modules/
+dist/
+dist-ssr/
+*.local
+.vite/
+
+# ============================================================================
+# Environment variables (NEVER COMMIT)
+# ============================================================================
+.env
+.env.local
+.env.*.local
+!.env.example
+
+# ============================================================================
+# Data files (large, re-buildable)
+# ============================================================================
+data/raw/*.json
+data/raw/*.csv
+data/raw/*.html
+data/raw/*.jsonl
+data/processed/*.json
+data/processed/*.csv
+data/processed/*.parquet
+data/indexes/*.pkl
+data/indexes/*.index
+data/indexes/*.bin
+data/indexes/*.faiss
+data/indexes/*.npy
+!data/raw/.gitkeep
+!data/processed/.gitkeep
+!data/indexes/.gitkeep
+# Whitelist: real Mamikos corpus (source of truth) + built indexes untuk Docker COPY
+!data/raw/mamikos_real_v2.jsonl
+!data/raw/_discovered_slugs.txt
+!data/processed/corpus.json
+!data/indexes/tfidf.pkl
+!data/indexes/bm25.pkl
+!data/indexes/indobert/
+
+# Scrape cache (during dev)
+.scrape_cache/
+
+# Database dumps
+*.db
+*.sqlite
+*.sqlite3
+*.dump
+
+# ============================================================================
+# Logs
+# ============================================================================
+*.log
+logs/
+
+# ============================================================================
+# IDE / OS
+# ============================================================================
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+Thumbs.db
+desktop.ini
+
+# ============================================================================
+# Misc
+# ============================================================================
+.render-buildlog
+*.pid

Dockerfile

@@ -0,0 +1,106 @@
+# ============================================================================
+# KozyNear single-container deploy
+# React frontend + FastAPI backend serve dari 1 service
+#
+# Multi-stage build:
+#   Stage 1 (Node)  -> npm install + vite build -> /app/frontend/dist
+#   Stage 2 (Python) -> copy frontend dist + install Python deps + uvicorn
+#
+# Routes di runtime:
+#   GET /                  -> React SPA (index.html)
+#   GET /assets/*          -> built JS/CSS chunks
+#   GET /favicon.ico, dll  -> static files
+#   GET /health            -> liveness probe (Render uses this)
+#   GET /api/search, /api/listings/{id}, /api/eval/*  -> FastAPI endpoints
+#   GET /api/docs          -> Swagger UI
+# ============================================================================
+
+# ----------------------------------------------------------------------------
+# Stage 1: Build React frontend
+# ----------------------------------------------------------------------------
+FROM node:20-alpine AS frontend-builder
+
+WORKDIR /app/frontend
+
+# Layer caching: install deps dulu (gak invalidate kalau code berubah)
+COPY frontend/package*.json ./
+RUN npm install
+
+# Build
+COPY frontend ./
+# VITE_API_URL kosong -> frontend pakai relative URL (same origin sebagai API)
+ENV VITE_API_URL=""
+RUN npm run build
+
+
+# ----------------------------------------------------------------------------
+# Stage 2: Python runtime
+# ----------------------------------------------------------------------------
+FROM python:3.11-slim
+
+# System deps untuk lxml (parsing HTML scraper) + psycopg2 + faiss (libgomp1:
+# wheel faiss-cpu link ke libgomp yang tidak ada di python-slim)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        libxml2-dev \
+        libxslt-dev \
+        libpq-dev \
+        libgomp1 \
+        curl \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install RUNTIME deps + NEURAL deps. Satu image untuk dua platform:
+# - Render free 512MB: ENABLE_NEURAL=false -> fastembed/faiss tidak di-import,
+#   RAM aman (paket cuma duduk di disk).
+# - HF Spaces 16GB ($0): ENABLE_NEURAL=true -> MiniLM + Hybrid live.
+# Full set dev (scraping, pandas, dev tools) ada di requirements.txt.
+COPY backend/requirements-runtime.txt backend/requirements-neural.txt ./
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements-runtime.txt \
+        -r requirements-neural.txt
+
+# Pre-download model ONNX MiniLM ke cache yang dipakai runtime
+# (FASTEMBED_CACHE_PATH) supaya startup dengan ENABLE_NEURAL=true tidak
+# download ~120MB tiap cold start (HF Spaces disk-nya ephemeral).
+ENV FASTEMBED_CACHE_PATH=/app/.fastembed_cache
+RUN python -c "import os; from fastembed import TextEmbedding; \
+    TextEmbedding('sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2', \
+    cache_dir=os.environ['FASTEMBED_CACHE_PATH'])"
+
+# Copy backend code
+COPY backend ./backend
+# Copy data folder (mamikos_real_v2 source + corpus.json + pre-built indexes)
+COPY data ./data
+# Copy hasil evaluasi (CSV) — dibaca /api/eval/* (file-based dashboard)
+COPY eval ./eval
+
+# Copy built frontend dari stage 1
+COPY --from=frontend-builder /app/frontend/dist ./static
+
+# Working dir backend supaya alembic + relative imports berfungsi
+WORKDIR /app/backend
+
+# Render expose $PORT via env (default 10000 kalau gak set)
+ENV PORT=10000
+EXPOSE 10000
+
+# Startup pipeline:
+#   1. alembic upgrade head -- create/update schema (idempotent)
+#   2. seed_db.py --truncate --skip-if-synced: reconcile listings table ke
+#      mamikos_real_v2.jsonl (227 real, sesudah filter deskripsi kosong).
+#      --skip-if-synced: kalau count + digest id DB == source, seed di-skip
+#      (cold start lebih cepat); kalau beda, TRUNCATE CASCADE + reseed atomic.
+#   3. uvicorn start FastAPI -- lifespan load TF-IDF + BM25 + gazetteer.
+#      Neural (MiniLM) ikut di-load HANYA kalau ENABLE_NEURAL=true
+#      (HF Spaces 16GB); di Render free biarkan false. Default search
+#      model = "smart" (query understanding + geo + BM25).
+#   Catatan resiliency: alembic/seed TIDAK boleh mematikan container kalau
+#   DB unreachable (mis. Space baru yang DATABASE_URL-nya belum di-set).
+#   App tetap up: /health, frontend, /api/stats (fallback corpus), /api/eval,
+#   /api/preprocess jalan; /api/search yang butuh DB akan error jelas dan
+#   /api/status menampilkan database.connected=false untuk debugging.
+CMD (alembic upgrade head || echo "[startup] alembic SKIPPED (DB unreachable? cek DATABASE_URL; continuing)") \
+    && (python -m scripts.seed_db --input ../data/raw/mamikos_real_v2.jsonl --truncate --skip-if-synced || echo "[startup] seed_db SKIPPED (continuing)") \
+    && uvicorn app.main:app --host 0.0.0.0 --port $PORT

LAPORAN.md

@@ -0,0 +1,722 @@
+# LAPORAN AKHIR — KozyNear
+
+**Mata Kuliah**: Temu Kembali Informasi (STKI, COM620321, 3 SKS)
+**Universitas**: Lampung (UNILA), FMIPA
+**Project**: Indonesian Kos-Kosan Search Engine — **seluruh Bandar Lampung untuk semua universitas**
+**Submission**: 17 Juni 2026 (vClass)
+**Repository**: https://github.com/DYmazeh/KozyNear
+**Live Demo**: https://dymazeh-kozynear.hf.space (HF Spaces, neural ON) | backup: https://kozynear.onrender.com (Render, smart-only)
+
+---
+
+## 1. Abstract
+
+KozyNear adalah Information Retrieval (IR) system untuk listing kos-kosan
+di **Bandar Lampung** (18 kecamatan dengan data real, 9 universitas: UNILA,
+ITERA, Darmajaya, UBL, UIN Raden Intan, Teknokrat, Polinela, Malahayati, Saburai).
+Sistem mendukung natural language query bahasa Indonesia dengan
+domain-specific preprocessing pipeline dan membandingkan **lima model
+retrieval**: TF-IDF, BM25, neural semantic (MiniLM multilingual, label
+kode "IndoBERT"), hybrid (BM25 → MiniLM rerank), dan **Smart** — pipeline
+produksi berbasis query understanding (parser rule-based) + geo ranking
+(gazetteer kampus terverifikasi + haversine) + BM25 fusion.
+
+Corpus terdiri dari **227 listing kos REAL** hasil scrape Mamikos.com
+(deskripsi pemilik asli + koordinat asli, 18 kecamatan), bukan synthetic.
+Evaluasi pada **30 query × 900 ground truth annotations** (AI-assisted
+3-annotator simulation, Cohen's Kappa 0.58-0.87) menempatkan **Smart
+sebagai best performer standard top-K by MAP** (MAP 0.301, P@5 0.580,
+MRR 0.868) diikuti BM25 (MAP 0.270, P@5 0.553); signifikansi pairwise
+diuji Wilcoxon signed-rank dengan koreksi **Holm-Bonferroni** (4 dari 10
+pasangan signifikan; semua melibatkan model neural pada standard eval).
+Pada lensa **Constraint Satisfaction@5** (proporsi top-5 yang memenuhi
+semua kebutuhan eksplisit user: gender, budget, fasilitas, radius 3 km
+dari kampus), Smart unggul jauh atas BM25 (**0.86 vs 0.45**, Wilcoxon
+p<0.001). Hyperparameter dipilih lewat grid search: bobot fusion Smart
+0.2/0.4/0.4 (teks/geo/atribut) dan alpha Hybrid 0.9.
+
+Finding penting: **model neural terlihat underperform di standard top-K
+eval (P@5 0.160)** semata-mata karena *pooling bias* — ground truth
+di-pool dari kandidat BM25, sehingga hasil semantic yang unik tidak
+ter-judge. Pada **pool-restricted evaluation** (fair, hanya judged docs),
+neural justru kompetitif (MAP 0.557 vs BM25 0.583). Ini menegaskan
+pentingnya melaporkan kedua angka untuk corpus IR yang di-pool secara
+lexical. Koordinat real 226/227 listing dipakai untuk geo ranking Smart
+dan integrasi Google Maps.
+
+---
+
+## 2. Latar Belakang
+
+### 2.1 Motivasi
+
+Mahasiswa di Bandar Lampung — baik dari UNILA, ITERA, Darmajaya, UBL,
+UIN, Teknokrat, Polinela, Malahayati, maupun Saburai — membutuhkan
+informasi kos-kosan strategis sekitar kampus mereka. Platform existing
+seperti Mamikos kurang optimal untuk Indonesian natural language karena
+tidak mendukung sinonim domain (e.g., "kmd" vs "kamar mandi dalam") dan
+tidak fokus geografis ke Bandar Lampung.
+
+### 2.2 Tujuan
+
+1. Membangun search engine khusus kos-kosan **seluruh Bandar Lampung**
+   untuk **semua universitas** (universitas-agnostic)
+2. Membandingkan tiga paradigma IR utama + hybrid: lexical (TF-IDF),
+   probabilistic (BM25), neural/semantic (IndoBERT+FAISS)
+3. Mengevaluasi performa quantitative via metric standar IR dengan
+   inter-annotator agreement protocol
+4. Deploy sebagai web app yang accessible publik
+
+### 2.3 Ruang Lingkup
+
+**In scope**: text retrieval atas data REAL Mamikos, 18 kecamatan Bandar
+Lampung, 9 universitas target, deskripsi pemilik asli, koordinat real
+untuk Google Maps, deployment Render.com free tier.
+
+**Out of scope**: image retrieval, real-time price updates, user auth,
+booking, multi-city (kabupaten lain). Geo-spatial distance kini masuk
+scope sebagai SALAH SATU sinyal fusion di model Smart (bobot 0.4
+bersama teks 0.4 + atribut 0.2), bukan primary signal tunggal. Tipe
+`pasutri` di luar scope — Mamikos hanya
+mengklasifikasi gender 0/1/2 (campur/putra/putri); "pasutri" adalah
+kategori marketing (campur yang boleh berdua), bukan tipe gender.
+
+---
+
+## 3. Dataset
+
+### 3.1 Acquisition Methodology — Real Mamikos Scrape
+
+Corpus adalah **227 listing kos REAL** dari Mamikos.com, di-scrape via
+pipeline 3-tahap (sumber: `data/raw/mamikos_real_v2.jsonl`):
+
+1. **Discovery** (`scripts/discover_mamikos_slugs.py` + WebSearch): kumpulkan
+   URL halaman detail `/room/`. Halaman kategori Mamikos JS-rendered dengan
+   API listing ter-enkripsi (AES), jadi discovery dilakukan via ~50 query
+   `site:mamikos.com inurl:/room/` (per-kecamatan, per-universitas,
+   per-fasilitas, per-harga, street-level) → **312 URL unik** terkumpul
+   dalam 5 ronde.
+2. **Detail extraction** (`scripts/extract_mamikos_detail.py`): setiap
+   halaman detail Mamikos meng-*inject* `var detail = {...}` (JSON 146
+   field) di static HTML — bisa di-fetch HTTP-only tanpa browser. Parse
+   field penting: `_id`, `room_title`, `description` (cerita pemilik asli),
+   `latitude`/`longitude`, `price_monthly`, `gender`, fasilitas
+   (fac_room/share/bath/park), booking_rules, owner_name, verification.
+3. **Canonical build** (`scripts/rebuild_v2.py`):
+   normalisasi kecamatan, hitung `jarak_kampus_km` via haversine ke 9
+   universitas, drop deskripsi kosong → `kozynear_combined.jsonl`.
+
+Success rate ekstraksi ~76% (sisanya listing inactive/removed). Field
+gender Mamikos hanya 0/1/2 → tipe **campur/putra/putri** (tidak ada pasutri).
+
+### 3.2 Honest Methodology Disclosure
+
+Iterasi awal project memakai synthetic corpus (template-generated) untuk
+mengejar volume. Namun karena web app ditujukan **production** (pengguna
+mencari kos nyata + tampilan Google Maps), synthetic dibuang total:
+koordinat synthetic yang fabricated akan menaruh pin di lokasi salah.
+
+**Trade-off yang diterima secara sadar**: corpus real lebih kecil (227 vs
+2000 synthetic) karena Mamikos Bandar Lampung hanya memiliki ~300-400
+listing unik dan WebSearch hanya meng-surface yang ter-index Google. Kami
+memilih **kualitas + keaslian** di atas volume. Semua deskripsi adalah
+tulisan asli pemilik kos (terverifikasi 0 template-leak), bukan generated.
+Setiap listing punya `url_source` untuk traceability balik ke Mamikos.
+
+### 3.3 Statistik Corpus
+
+| Metric | Value |
+|--------|-------|
+| Total listings | **227 (100% real Mamikos)** |
+| Kecamatan covered | 18 unique (Bandar Lampung) |
+| Tipe | campur 114 (50%), putri 85 (37%), putra 28 (13%) |
+| Harga (per bulan) | min Rp 300k, median Rp 800k, max Rp 6jt |
+| Deskripsi (kata) | median ~23 (cerita pemilik, terse — bukan padded) |
+| Koordinat valid (Google Maps) | 226/227 (99.6%) |
+| url_source / Mamikos-verified | 100% / 100% |
+| owner_name present | 96% |
+| File size | ~0.3 MB raw / ~0.6 MB processed |
+
+**Distribusi listings per kecamatan** (top: Sukarame 40, Kedaton 37,
+Rajabasa 32, Tanjung Karang Timur 14, Sukabumi 13):
+
+![Distribusi Kecamatan](notebooks/charts/01_kecamatan.png)
+
+**Karakteristik data real vs synthetic**: deskripsi pemilik asli jauh lebih
+*terse* (median 23 kata) dibanding synthetic (110+ kata padded). Contoh
+nyata: *"belakang UNILA (sangat dekat, jalan kaki 5mnt), dekat rel kereta"*.
+Ini menurunkan absolute IR metric (lebih sedikit term overlap) tetapi
+mencerminkan **realita yang sebenarnya** dihadapi sistem production.
+
+**Distribusi tipe + harga** (lihat notebook 01_eda.ipynb untuk chart lain):
+
+![Distribusi Tipe Kos](notebooks/charts/01_tipe.png)
+
+![Distribusi Harga](notebooks/charts/01_price.png)
+
+---
+
+## 4. Preprocessing
+
+### 4.1 Pipeline 9-Stage
+
+```
+raw text
+  → 1. HTML strip (BeautifulSoup)
+  → 2. Whitespace normalize
+  → 3. Price extraction (SEBELUM lowercase — preserve Rp)
+  → 4. Lowercase
+  → 5. Custom jargon dictionary substitution (longest-first regex)
+  → 6. Spelling correction (typo dictionary)
+  → 7. Tokenize (whitespace + punctuation)
+  → 8. Stopword removal (Sastrawi default + custom domain)
+  → 9. Stem (Sastrawi StemmerFactory, LRU-cached)
+```
+
+**Anti-pattern**: lowercase sebelum extract harga → regex `[Rr][Pp]` masih
+match, tapi edge case `Rp850k` (tanpa space) jadi rapuh.
+
+### 4.2 Custom Jargon Dictionary
+
+**~106 entries** (course requirement ≥100):
+- ABBREVIATIONS: 43 (kmd, wc dlm, jt, rb, yg, krn, ...)
+- LOCATIONS: 24 (gdg meneng, sumbro, unyila, ...)
+- TYPE_SLANG: 11 (cowo, cewe, pria, wanita, ...) — entri `pasutri` dihapus
+  (bukan tipe gender valid di Mamikos)
+- RULES: 5 (jamal, no smoking, ...)
+- PAYMENT: 8 (dp, deposit, all in, ...)
+- KOS_FORM: 15 (kostan, kontrakan, eksklusif, ...)
+
+### 4.3 Pipeline Performance + Stage Impact
+
+Run pada 227 listing real:
+- **Latency**: ~5 detik (Sastrawi LRU cache, deskripsi real lebih pendek)
+- **Token reduction**: deskripsi terse pemilik → median ~20 token setelah
+  stemming (real owner text jauh lebih ringkas dari synthetic padded)
+
+**Benchmark MAP per pipeline variant** (notebook 02_preprocessing.ipynb):
+
+![Preprocessing Stage Impact](notebooks/charts/02_preprocessing_impact.png)
+
+Setiap stage contribution incremental dari BASELINE → FULL pipeline.
+
+### 4.4 Real-World Insight
+
+1. **"gedong meneng" → "gedong neng"**: Sastrawi treats "meneng" punya
+   prefix "me-" yang invalid (false positive stemming)
+2. **"kamar mandi dalam" → "mandi"**: "kamar" jadi custom domain stopword,
+   sehingga sebagian besar signal hilang
+3. **Jargon substitution sukses**: "kmd" → "kamar mandi dalam" → "mandi"
+   setelah stemming (signal preserved walau partial)
+
+---
+
+## 5. Model Comparison
+
+### 5.1 Implementasi
+
+| Model | Library | Hyperparameter |
+|-------|---------|----------------|
+| TF-IDF | scikit-learn TfidfVectorizer | ngram_range=(1,2), min_df=2, max_features=10k, sublinear_tf=True |
+| BM25 | rank_bm25 BM25Okapi | k1=1.5, b=0.75 (literature standard) |
+| Neural "IndoBERT" | fastembed (ONNX) + FAISS | paraphrase-multilingual-MiniLM-L12-v2, IndexFlatIP, normalize_embeddings. Catatan kejujuran: label kode `indobert` historis; model sebenarnya MiniLM multilingual |
+| Hybrid | BM25 top-50 → MiniLM rerank | min-max normalize + alpha=0.9 (grid search pool-restricted, lihat §8.3; default lama 0.3 terbukti merugikan) |
+| **Smart (live)** | parser rule-based + gazetteer + BM25 | fusion 0.2 teks + 0.4 geo (1/(1+km), anchor terverifikasi OSM/Wikipedia) + 0.4 fasilitas (grid search CS@5, lihat §8.3b); hard filter gender/harga dengan relaxation bertahap; geo-augment kandidat radius 3 km; filter UI = constraint eksplisit yang tidak pernah di-relax |
+
+### 5.2 Index Build Performance
+
+| Index | Build Time | Disk Size |
+|-------|-----------|-----------|
+| TF-IDF | 0.2s | 125 KB |
+| BM25 | 0.1s | 53 KB |
+| IndoBERT (227 docs encode) | ~10s | ~0.7 MB (embeddings + FAISS + meta) |
+| Hybrid | runtime composition (no build) | — |
+
+**Total**: <1 MB indexes committed ke repo. Corpus real yang lebih kecil
+(227 docs) membuat footprint memory ringan — cocok untuk Render free tier
+512MB RAM (encoding via `fastembed` ONNX ~150MB, bukan torch ~500MB).
+
+---
+
+## 6. Evaluation Protocol
+
+### 6.1 Query Set (v1.1 — 30 queries)
+
+**30 queries** (eval/queries.json): q01-q15 dari v1.0 (didesain untuk data
+real), q16-q30 ditambah untuk menaikkan power statistik (n=15 underpowered
+untuk Wilcoxon) + memperluas coverage: kampus Darmajaya/UBL, kecamatan
+Tanjung Senang/Gedong Meneng/Labuhan Ratu, fasilitas kipas angin/tv/kasur,
+landmark Mall Boemi Kedaton. Setiap query diverifikasi *answerable* oleh
+corpus real (avg 14.4 relevant/query; 2 draft q16-q30 yang mati diganti
+sebelum dipakai). Berbasis vocabulary yang benar-benar muncul di deskripsi
+pemilik. Sampel pola v1.0:
+
+- **Per-universitas**: q01 (UNILA+putra+wifi+ac), q04 (ITERA+sukarame),
+  q05 (Teknokrat+kedaton), q06 (Polinela+rajabasa), q11 (UNILA+putra+rajabasa),
+  q12 (UNILA+putri), q13 (UIN raden intan)
+- **Per-tipe + fasilitas**: q02 (putri+kedaton), q03 (campur+kmd+parkir),
+  q07 (eksklusif+ac+wifi+shower), q14 (putri+way halim)
+- **Kualitas / use-case**: q08 (putri+aman+cctv), q09 (dapur+parkir mobil),
+  q10 (transmart+way halim), q15 (campur+nyaman+strategis)
+
+**Catatan desain penting**: BM25/IR meng-index **teks deskripsi**;
+filter kecamatan/harga/tipe dilayani **metadata** terpisah di `/search`
+endpoint. Karena itu query eval fokus pada konsep yang muncul di teks
+(universitas, fasilitas, kualitas). Dari versi awal (18 query untuk corpus
+synthetic), **8 query di-drop karena mati di data real** — yaitu yang
+menguji karakteristik khas synthetic: `pasutri` (corpus real 0 pasutri),
+"jam malam aturan ketat", "kerja dari rumah", "warung makan", "tahunan
+bayar di muka", street-name spesifik. Deskripsi pemilik asli yang terse
+tidak memuat frasa-frasa template tersebut.
+
+### 6.2 Ground Truth — AI-Assisted 3-Annotator Simulation
+
+**Pool**: BM25 top-30 per query = **900 (query, doc) pairs** (30 × 30).
+
+**Annotator simulation** (rule-based, `scripts/generate_ground_truth.py`):
+- A (strict): tipe match +1.5 / mismatch -2.0; kecamatan +1.5;
+  fasilitas +0.5; harga "murah" +1.0
+- B (lenient): A + 0.5 score boost
+- C (noisy): A + uniform(-0.5, 0.5) jitter (seed 42)
+
+Score → relevance: ≥2.5 → 2, ≥1.0 → 1, else 0. Consensus majority vote
+→ `eval/ground_truth.csv`.
+
+**Disclosure**: AI-assisted bootstrap GT, BUKAN human annotation — di-pool
+dari BM25 (lihat implikasi *pooling bias* di §7/§8). Untuk production-grade,
+3-human annotator pass tetap recommended (`docs/ground_truth_protocol.md`).
+
+### 6.3 Inter-Annotator Agreement
+
+| Pair | Cohen's Kappa | Interpretation |
+|------|---------------|----------------|
+| A vs B | 0.700 | substantial |
+| A vs C | 0.871 | almost perfect |
+| B vs C | 0.578 | moderate |
+
+Kappa pada data real sedikit lebih rendah dari iterasi synthetic karena
+deskripsi terse memberi sinyal lebih ambigu untuk heuristik. Pasangan B-C
+(lenient vs noisy) turun ke *moderate* (0.572); A-B dan A-C tetap
+substantial–almost perfect.
+
+### 6.4 Distribusi Consensus
+
+| Relevance | Count | % |
+|-----------|-------|---|
+| 0 (not relevant) | 467 | 51.9% |
+| 1 (somewhat) | 361 | 40.1% |
+| 2 (highly) | 72 | 8.0% |
+
+Proporsi rel=2 lebih kecil dari iterasi synthetic (template membuat banyak
+"highly relevant" semu); data real lebih realistis — sedikit yang benar-benar
+sangat relevan per query.
+
+---
+
+## 7. Results
+
+Tiga lensa evaluasi dilaporkan karena GT di-pool secara lexical (BM25):
+**(A) Standard top-K** — IR query atas seluruh corpus 227; **(B)
+Pool-restricted** — model hanya me-ranking ulang dokumen yang sudah
+ter-judge (fair untuk model non-lexical); **(C) Constraint
+Satisfaction@5** — proporsi top-5 yang memenuhi semua kebutuhan eksplisit
+user, tanpa qrels (bebas pooling bias). Eval Smart memakai fungsi
+`smart_rank()` yang persis sama dengan serving (`scripts/eval_smart.py`).
+
+### 7.1a Standard Top-K (30 queries × 5 models)
+
+| Model | P@5 | P@10 | MAP | NDCG@10 | MRR |
+|-------|-----|------|-----|---------|-----|
+| **Smart (live)** | **0.580** | 0.480 | **0.301** | 0.548 | **0.868** |
+| BM25 | 0.553 | **0.530** | 0.270 | **0.553** | 0.802 |
+| Hybrid (α=0.9) | 0.553 | 0.527 | 0.267 | 0.551 | 0.804 |
+| TF-IDF | 0.547 | 0.510 | 0.239 | 0.523 | 0.727 |
+| Neural MiniLM | 0.160 | 0.153 | 0.057 | 0.155 | 0.280 |
+
+**Ranking by MAP**: Smart > BM25 > Hybrid > TF-IDF > Neural. Catatan
+pembacaan jujur: Smart unggul di MAP, P@5, dan terutama MRR (0.868:
+hasil pertama hampir selalu relevan), tapi P@10-nya DI BAWAH BM25.
+Penyebabnya struktural: hard filter memangkas kandidat yang melanggar
+constraint dan dokumen hasil geo-augment yang belum ter-annotate dihitung
+rel=0 (pooling bias juga menekan Smart). Selisih MAP Smart vs BM25 belum
+signifikan (p=0.69, lihat §7.2); klaim utama Smart ada di lensa CS@5
+(§7.4) yang bebas qrels.
+
+### 7.1b Pool-Restricted (fair across models)
+
+| Model | P@5 | P@10 | MAP | NDCG@10 | MRR |
+|-------|-----|------|-----|---------|-----|
+| **Smart (live)** | **0.927** | **0.800** | **0.865** | **0.857** | **1.000** |
+| Hybrid (α=0.9) | 0.553 | 0.530 | 0.591 | 0.553 | 0.804 |
+| BM25 | 0.553 | 0.520 | 0.583 | 0.546 | 0.802 |
+| TF-IDF | 0.560 | 0.530 | 0.576 | 0.541 | 0.751 |
+| Neural MiniLM | 0.487 | 0.493 | 0.557 | 0.487 | 0.708 |
+
+**Insight kunci #1 (pooling bias)**: Neural melompat dari MAP 0.057
+(standard) → **0.557** (pool-restricted). Gap besar ini adalah artefak
+*pooling bias*: GT di-pool dari BM25 top-30, jadi dokumen yang dipilih
+model semantic tapi tidak masuk pool otomatis dihitung tidak-relevan
+(rel=0) di standard eval — bukan karena modelnya buruk, melainkan karena
+belum ter-judge. Pada perbandingan fair, neural kompetitif (~0.03 MAP di
+bawah BM25).
+
+**Insight kunci #2 (smart di pool)**: MAP Smart 0.865 (MRR 1.000) di
+pool-restricted sebagian besar datang dari **alignment constraint dengan
+judgment**: annotator (simulasi) menghukum gender/harga mismatch, dan hard
+filter Smart membuang dokumen yang sama sehingga dokumen relevan naik ke
+atas. Ini by-design (sistem mengoptimalkan kebutuhan user yang sama), tapi
+karena heuristik annotator dan filter Smart melihat sinyal yang mirip,
+angka ini dibaca sebagai *upper bound* yang optimistic, bukan bukti
+mutlak superioritas; karena itu lensa CS@5 yang bebas qrels dilaporkan
+juga (§7.4).
+
+![MAP per Model](notebooks/charts/04_map_bars.png)
+
+![All Metrics Comparison](notebooks/charts/04_all_metrics.png)
+
+### 7.2 Pairwise Statistical Significance (Wilcoxon Signed-Rank + Holm, n=30, α=0.05)
+
+Pada AP standard top-K, 5 model = 10 pasangan diuji sekaligus. Menguji
+banyak hipotesis pada α=0.05 tanpa koreksi menggelembungkan peluang false
+positive (10 uji: ~40%), jadi signifikansi final memakai koreksi
+**Holm-Bonferroni** (kontrol family-wise error rate); p-value mentah tetap
+dilaporkan untuk transparansi. (Pada iterasi n=15 sebelumnya, BM25 vs
+Hybrid lolos raw tapi gugur setelah Holm; pelajaran kenapa koreksi wajib.)
+
+| Pair | Statistic | p-value | p-Holm | Sig (raw) | Sig (Holm) |
+|------|-----------|---------|--------|-----------|------------|
+| BM25 vs Hybrid | 108.0 | 0.5481 | 1.0000 | no | no |
+| BM25 vs Neural | 31.0 | <0.0001 | <0.001 | yes | **YES** |
+| BM25 vs Smart | 199.0 | 0.6891 | 1.0000 | no | no |
+| BM25 vs TF-IDF | 176.0 | 0.2534 | 1.0000 | no | no |
+| Hybrid vs Neural | 29.0 | <0.0001 | <0.001 | yes | **YES** |
+| Hybrid vs Smart | 208.0 | 0.6263 | 1.0000 | no | no |
+| Hybrid vs TF-IDF | 190.0 | 0.3931 | 1.0000 | no | no |
+| Neural vs Smart | 17.0 | <0.0001 | 0.0001 | yes | **YES** |
+| Neural vs TF-IDF | 24.0 | <0.0001 | <0.001 | yes | **YES** |
+| Smart vs TF-IDF | 159.0 | 0.2059 | 1.0000 | no | no |
+
+**4/10 signifikan, dan keempatnya BERTAHAN setelah Holm** (p sangat kecil,
+semuanya pasangan vs Neural pada standard eval) — konsisten dengan cerita
+pooling bias, bukan ranking model. Empat model non-neural (Smart, BM25,
+Hybrid, TF-IDF) tidak saling berbeda signifikan pada n=30; hasil
+non-signifikan dibaca *inconclusive*, bukan bukti dua model setara.
+Detail: `eval/significance_map.csv`.
+
+### 7.3 Per-Query Analysis Insights
+
+- **BM25 dominan di queries lexical-heavy**: q01 (unila+putra+wifi+ac),
+  q03 (campur+kmd+parkir), q13 (uin raden intan) — exact-match nama
+  universitas + fasilitas yang muncul di deskripsi.
+- **TF-IDF kompetitif**: pada corpus kecil 227 dengan term yang jarang,
+  TF-IDF sublinear_tf cukup dekat dengan BM25 (selisih tidak signifikan).
+- **Query landmark lemah**: q10 (transmart way halim) hanya ~2 relevant —
+  sedikit deskripsi pemilik yang menyebut landmark eksplisit.
+- **Neural MiniLM**: kuat di pool-restricted (semantic mengelompokkan kos
+  sejenis) tapi tertekan di standard karena GT lexical-pooled.
+
+### 7.4 Constraint Satisfaction @5 (lensa bebas qrels)
+
+CS@5 = proporsi top-5 yang memenuhi **semua** constraint query: gender
+benar + harga ≤ budget + fasilitas ada + jarak ≤ 3 km dari anchor kampus/
+landmark (30 query di `eval/queries_constraints.json`, koordinat anchor
+terverifikasi OSM/Wikipedia). Metric ini mengukur langsung kebutuhan user
+dan tidak bergantung ground truth (bebas pooling bias).
+
+| Model | mean CS@5 (n=30) |
+|-------|------------------|
+| **Smart (live)** | **0.863** |
+| BM25 | 0.447 |
+
+Wilcoxon signed-rank smart vs BM25: **p<0.001 (signifikan)** — satu uji
+terencana, tidak butuh koreksi keluarga. Interpretasi praktis: dari 5
+hasil teratas, rata-rata 4.3 memenuhi semua kebutuhan user di Smart vs
+2.2 di BM25. Inilah justifikasi kuantitatif Smart sebagai model live.
+Detail per-query: `eval/results_constraints.csv`.
+
+---
+
+## 8. Discussion
+
+### 8.1 Mengapa BM25 Dominan?
+
+1. **Lexical-heavy query**: kebanyakan query mengandung exact terminology
+   (nama kecamatan, tipe kos, nama universitas, fasilitas). BM25 term
+   saturation handling lebih appropriate dari TF-IDF (sublinear_tf).
+2. **Short documents**: deskripsi pemilik sangat pendek (median ~23 kata,
+   ~20 token setelah stemming) — BM25 length normalization (b=0.75) efektif
+   untuk dokumen pendek.
+3. **Domain-specific corpus**: minim ambiguity semantic — tidak ada
+   keuntungan dari embedding-based representation.
+4. **GT lexical-pooled**: ground truth di-pool dari BM25, sehingga metric
+   standar memang menguntungkan model lexical (lihat §8.2 pooling bias).
+
+### 8.2 Mengapa IndoBERT "Underperform" — Pooling Bias, Bukan Model
+
+Temuan paling penting dari iterasi real-data: model neural **tidak**
+benar-benar buruk. Gap MAP 0.057 (standard) vs 0.557 (pool-restricted)
+hampir seluruhnya disebabkan **pooling bias**:
+
+1. **Ground truth di-pool dari BM25**: hanya BM25 top-30 yang di-judge.
+   Dokumen yang IndoBERT ranking tinggi tapi tidak masuk pool BM25 otomatis
+   dihitung rel=0 — *unjudged ≠ irrelevant*. Ini bias klasik TREC-style
+   pooling ketika pool dibangun dari satu jenis model.
+2. **Bukti pool-restricted**: saat semua model hanya me-ranking dokumen
+   yang sama-sama ter-judge, neural kompetitif (MAP 0.557, hanya ~0.03
+   di bawah BM25).
+3. **Faktor sekunder** (tetap relevan): deskripsi pemilik sangat pendek
+   (median 23 kata) memberi sedikit konteks untuk embedding; MiniLM
+   multilingual kurang terspesialisasi untuk nuansa Indonesia.
+
+**Implikasi metodologis**: untuk corpus yang di-pool lexical, melaporkan
+HANYA standard top-K akan menyesatkan tentang kemampuan model semantic.
+Karena itu kedua angka dilaporkan (§7.1a vs §7.1b).
+
+### 8.3 Hybrid: Grid Search Alpha (0.3 → 0.9)
+
+Iterasi awal memakai `alpha=0.3` (70% weight MiniLM) dan hybrid terseret
+jatuh di standard eval (MAP 0.188 di n=30) karena skor neural tertekan
+pooling bias. Grid search alpha ∈ [0, 1] step 0.1 dijalankan di dua lensa
+(`eval/hybrid_alpha_grid.csv` + `hybrid_alpha_grid_pool.csv`):
+
+- **Standard**: monoton naik sampai alpha=1.0 (murni BM25). Ini bukan
+  bukti MiniLM tidak berguna, melainkan artefak GT lexical-pooled: bobot
+  neural berapa pun "merusak" metric yang di-pool dari BM25.
+- **Pool-restricted (fair, dasar keputusan)**: kurva nyaris datar di
+  0.6-0.9 dengan puncak **alpha=0.9** (MAP 0.5909 vs BM25 murni 0.5836,
+  alpha lama 0.3 = 0.5836). Selisihnya kecil dan tidak signifikan
+  (inconclusive), tapi alpha tinggi juga menghapus kerugian besar di
+  standard (MAP 0.267 vs 0.188).
+
+Default produksi kini `alpha=0.9`: hybrid = BM25 + tie-breaking semantic
+tipis. Interpretasi jujur: pada corpus 227 dengan deskripsi terse, sinyal
+MiniLM hanya membantu di margin; nilai utamanya teredam pooling bias
+(lihat pool-restricted: hybrid MAP 0.591, tertinggi di antara model
+non-smart).
+
+Mitigasi eval yang tersisa (future work):
+- Pool GT dari union multi-model (BM25 ∪ MiniLM ∪ TF-IDF ∪ Smart) untuk
+  hilangkan pooling bias — perbaikan eval paling berdampak
+- Human annotation pass untuk GT yang truly model-agnostic
+
+### 8.3b Mengapa Smart Dipilih sebagai Model Live?
+
+Smart menjawab kelemahan praktis semua model murni-teks: query kos riil
+("kos putri deket unila max 800rb ada wifi") membawa **constraint
+terstruktur** yang tidak terbaca BM25/TF-IDF sebagai constraint, hanya
+sebagai token. Smart memecah query: parser menarik gender/harga/fasilitas/
+anchor; BM25 menangani sisa teks (setelah preprocessing yang sama dengan
+corpus); skor akhir = fusi teks + kedekatan geografis + kecocokan
+fasilitas; hard filter dengan relaxation bertahap menjamin hasil tidak
+kosong dan pelonggaran dilaporkan jujur ke user (chip "filter
+dilonggarkan").
+
+Tiga detail desain penting:
+1. **Geo-augment kandidat**: saat anchor terdeteksi, kandidat BM25
+   di-union dengan semua listing radius 3 km. Tanpa ini, listing dekat
+   ITERA yang deskripsinya tidak menulis "itera" mustahil muncul (skor geo
+   hanya bisa menata ulang kandidat yang ada, tidak bisa menambah).
+2. **Filter UI = constraint eksplisit**: tidak pernah di-relax. Relaxation
+   hanya untuk constraint hasil inferensi teks.
+3. **Bobot fusion dari grid search, bukan tebakan**: simplex step 0.1
+   (55 kombinasi, `eval/smart_weights_grid.csv`) memilih **0.2 teks /
+   0.4 geo / 0.4 atribut**, menang atas default awal 0.4/0.4/0.2 di kedua
+   lensa (CS@5 0.863 vs 0.810, Wilcoxon p=0.023; MAP standard 0.301 vs
+   0.287). Kombinasi ber-CS tertinggi (mis. teks 0.1 / atribut 0.7)
+   ditolak sadar: CS@5 memang menghitung atribut, jadi bobot atribut
+   ekstrem = overfit metric; teks tetap dibutuhkan untuk query kualitas
+   ("nyaman", "bersih") di luar pola constraint.
+
+Bukti kuantitatif: CS@5 0.86 vs 0.45 (p<0.001, §7.4); standard MAP juga
+tertinggi (0.301) meski selisihnya terhadap BM25 belum signifikan (n=30).
+Biaya: nol model neural di runtime — aman untuk free tier 512MB.
+
+### 8.4 Karakteristik Data Real vs Coverage
+
+- **Deskripsi terse**: median 23 kata (vs synthetic 110+). Menurunkan
+  absolute IR metric tapi mencerminkan realita — pemilik kos menulis singkat
+  ("dekat UNILA, jalan kaki 5 menit"). Sistem production harus handle ini.
+- **Distribusi kecamatan tidak merata**: Sukarame/Kedaton/Rajabasa dominan
+  (dekat UNILA/UIN/ITERA/Polinela) karena memang di situ density kos Mamikos
+  tertinggi. Mencerminkan pasar nyata, bukan artefak sampling.
+- **Koordinat real**: 226/227 listing punya lat/lng asli Mamikos →
+  langsung dipakai untuk pin Google Maps di web app. 1 listing dengan
+  koordinat korup (geocoding fallback ke Jakarta) di-null secara eksplisit.
+
+### 8.5 Limitations
+
+1. **Corpus size 227**: dibatasi ketersediaan listing Mamikos Bandar Lampung
+   yang ter-index Google (~300-400 total) + success rate ekstraksi ~76%.
+2. **Pooling bias**: GT di-pool dari BM25 → bias terhadap model lexical
+   (dimitigasi sebagian via pool-restricted eval, belum via human/multi-model pool).
+3. **AI-assisted GT**: heuristic scoring, bukan true human judgment.
+4. **Sample size n=30 query** (dinaikkan dari 15): power lebih baik tapi
+   tetap moderat; semua hasil non-signifikan dibaca inconclusive, bukan
+   "setara". Hyperparameter (alpha, bobot fusion) dipilih pada data eval
+   yang sama dengan yang dilaporkan (tanpa held-out set) — di-disclose
+   sebagai mild selection bias.
+5. **Single embedding model**: hanya MiniLM multilingual (fastembed ONNX);
+   `indobert-base-p2` belum diuji (RAM constraint Render free tier;
+   deploy HF Spaces 16GB membuka jalan untuk menguji ini).
+6. **Sinyal annotator vs filter Smart tumpang tindih**: heuristik
+   annotator simulasi menghukum mismatch gender/harga, yang juga
+   merupakan hard filter Smart — angka pool-restricted Smart (MAP 0.865)
+   karenanya optimistic; lensa CS@5 yang bebas qrels dilaporkan sebagai
+   pengimbang.
+
+---
+
+## 9. System Architecture
+
+### 9.1 Stack
+
+| Layer | Tech |
+|-------|------|
+| Frontend | React 18 + Vite 5 + TypeScript — 4 tab: Pencarian, Evaluasi Model, Preprocessing (visualisasi 9-stage), Statistik |
+| Backend | Python 3.11 + FastAPI + Uvicorn |
+| Database | PostgreSQL 16 (asyncpg + SQLAlchemy 2.0 + Alembic) |
+| IR Libraries | Sastrawi, rank_bm25, fastembed (ONNX MiniLM), faiss-cpu, scikit-learn |
+| Deploy | Docker single-container: HF Spaces (16GB, neural ON) + Render.com backup (512MB, smart-only) |
+
+### 9.2 API Endpoints
+
+| Method | Path | Description |
+|--------|------|-------------|
+| GET | `/health` | Liveness probe |
+| GET | `/api/search` | IR search dengan model & filter (`q`, `model`, `top_k`, `harga_min`, `harga_max`, `tipe`, `kecamatan`); filter berlaku untuk SEMUA model termasuk smart |
+| GET | `/api/listings/{id}` | Listing detail |
+| GET | `/api/eval/summary` | Aggregate eval (standard + pool-restricted + CS@5 + Wilcoxon/Holm), file-based dari CSV hasil eval |
+| GET | `/api/eval/query/{query_id}` | Per-query breakdown |
+| GET | `/api/preprocess` | Trace pipeline 9-stage per langkah (tab Preprocessing) |
+| GET | `/api/stats` | Statistik corpus live (distribusi kecamatan/tipe, harga, vocab) |
+| GET | `/api/status` | Diagnostics lengkap (index loaded, RAM, DB) |
+| GET | `/api/docs` | Swagger UI |
+| GET | `/` (catch-all) | React SPA index.html |
+
+### 9.3 Deployment
+
+Satu Dockerfile untuk dua platform (neural di-gate env `ENABLE_NEURAL`):
+
+- **HF Spaces** (primary demo) → `https://dymazeh-kozynear.hf.space` —
+  CPU basic free 16GB RAM, `ENABLE_NEURAL=true`: kelima model live
+  termasuk Neural MiniLM + Hybrid. Model ONNX di-pre-download saat build.
+- **Render.com** (backup) → `https://kozynear.onrender.com` — free 512MB,
+  `ENABLE_NEURAL=false`: smart/BM25/TF-IDF. `kozynear-db` PostgreSQL
+  managed dipakai kedua deploy.
+- Cold start: `seed_db --truncate --skip-if-synced` (skip reseed kalau DB
+  sudah sinkron); GitHub Actions `keepalive.yml` ping /health tiap 12
+  menit supaya tidak spin-down saat demo.
+
+---
+
+## 10. Conclusion & Future Work
+
+### 10.1 Conclusion
+
+KozyNear berhasil membangun end-to-end IR system khusus kos-kosan
+**seluruh Bandar Lampung** dengan corpus **227 listing REAL Mamikos**
+(deskripsi pemilik + koordinat asli) dan 5 model retrieval. Finding utama:
+
+1. **Smart (query understanding + geo + BM25 fusion) sebagai model live**:
+   MAP standard tertinggi (0.301, MRR 0.868) dan CS@5 0.86 vs BM25 0.45
+   (Wilcoxon p<0.001, n=30). Memenuhi kebutuhan terstruktur user (gender/
+   budget/fasilitas/jarak kampus) yang tidak terbaca model murni-teks,
+   tanpa model neural di runtime. Bobot fusion 0.2/0.4/0.4 hasil grid
+   search dua-lensa.
+2. **BM25 best lexical baseline** (MAP 0.270, P@5 0.553) — setelah koreksi
+   Holm-Bonferroni, unggul signifikan hanya vs Neural (p-Holm<0.001);
+   selisih antar model non-neural (vs TF-IDF/Hybrid/Smart) inconclusive
+   pada n=30.
+3. **Neural MiniLM bukan buruk — pooling bias**: MAP melompat 0.057 →
+   0.557 saat pool-restricted. Pelajaran metodologis tentang pelaporan
+   multi-lensa (standard + pool-restricted + CS@5).
+4. **Hybrid butuh alpha tinggi** (grid search → 0.9): pada GT
+   lexical-pooled, bobot neural besar hanya merugikan; dengan alpha 0.9
+   hybrid jadi model non-smart terbaik di pool-restricted (MAP 0.591).
+5. **Data real menuntut kejujuran**: deskripsi terse menurunkan absolute
+   metric, tapi sistem siap production (koordinat Maps terverifikasi,
+   traceable url_source, model identity disclosed).
+
+### 10.2 Rubric Coverage
+
+| Aspect | Weight | Coverage |
+|--------|--------|----------|
+| Theme | 15% | Kos search Bandar Lampung multi-universitas + integrasi Google Maps, under-represented di IR Indonesia |
+| Dataset | 15% | **227 listing REAL Mamikos** (scrape detail-page, 18 kecamatan, koordinat asli), methodology + pipeline disclosed |
+| Preprocessing | 15% | 9-stage pipeline + ~110 jargon entries (>100 req) + benchmark + **visualisasi live per-stage di tab Preprocessing** |
+| Model | 20% | 5 model unified (TF-IDF/BM25/Neural MiniLM/Hybrid/**Smart live**) + honest comparison 3 lensa + hyperparameter via grid search (alpha 0.9, fusion 0.2/0.4/0.4) |
+| Evaluation | 10% | n=30 query, 900 annotations; P@K + MAP + NDCG + MRR + Cohen's Kappa + Wilcoxon **dengan koreksi Holm-Bonferroni** (10 pasangan, 4 signifikan) + Constraint-Satisfaction@5 + dashboard eval live di UI |
+| System | 25% | FastAPI + React 4-tab + PG + Docker; deploy ganda HF Spaces (neural ON) + Render backup + keepalive |
+
+### 10.3 Future Work
+
+1. **Perluas corpus real**: scrape kompetitor (Cove, Rukita, sewakost.com)
+   atau Google Places untuk listing di luar Mamikos → tembus >300 real.
+2. **Hilangkan pooling bias**: pool GT dari union multi-model (BM25 ∪
+   MiniLM ∪ TF-IDF ∪ Smart) atau human annotation — perbaikan eval paling
+   berdampak.
+3. **Indonesian-specialized embedding**: comparison `indobenchmark/indobert-base-p2`
+   (kini feasible di HF Spaces 16GB).
+4. **Human annotation pass**: replace AI-assisted GT (protocol di docs).
+5. **Held-out query set**: hyperparameter (alpha, bobot fusion) saat ini
+   dipilih di data eval yang sama; pisahkan dev/test set di iterasi
+   berikutnya. (Selesai di iterasi ini: query 15→30, grid search alpha
+   dan bobot fusion.)
+
+---
+
+## 11. Repository Structure
+
+```
+KozyNear/
+├── LAPORAN.md                          # ini, single-doc report
+├── README.md                           # quick start + project overview
+├── Dockerfile                          # multi-stage build (Node + Python)
+├── render.yaml                         # Render infra-as-code
+├── docker-compose.yml                  # local PG dev
+├── docs/
+│   ├── architecture.md
+│   ├── deploy_guide.md
+│   └── ground_truth_protocol.md
+├── data/
+│   ├── raw/mamikos_real_v2.jsonl       # 227 listing REAL Mamikos (committed)
+│   ├── raw/_discovered_slugs.txt        # 312 URL discovery (committed)
+│   ├── processed/corpus.json           # post-preprocessing (committed)
+│   └── indexes/                        # TF-IDF + BM25 + IndoBERT (committed)
+├── eval/
+│   ├── queries.json                    # 15 test queries (v1.0 real-data)
+│   ├── ground_truth.csv                # 900 consensus annotations
+│   ├── kappa_report.md                 # inter-annotator agreement
+│   ├── annotations_annotator_{A,B,C}.csv
+│   ├── results.csv                     # per-model per-query (standard)
+│   ├── results_pool_restricted.csv     # pool-restricted (fair) eval
+│   ├── results.withsynth.csv           # arsip: eval era synthetic (perbandingan)
+│   ├── _audit_dataset.py               # data quality audit (reusable)
+│   └── eval_summary.md                 # aggregate + Wilcoxon + analysis
+├── backend/
+│   ├── app/                            # FastAPI app
+│   ├── alembic/                        # DB migrations
+│   ├── scripts/                        # generate, preprocess, gen_GT, eval_report
+│   └── tests/                          # pytest suite
+└── frontend/
+    └── src/
+        ├── api/client.ts               # typed API client
+        ├── components/                 # FilterPanel, ResultCard
+        └── App.tsx                     # main SPA
+```
+
+---
+
+## 12. Acknowledgments
+
+Project ini didevelop dengan AI agent assistance (Claude) sebagai
+implementation executor, dengan user sebagai project manager / reviewer.
+Methodology disclosure jujur tentang scrape data real Mamikos, pooling
+bias pada GT yang AI-assisted, dan keputusan drop synthetic demi keaslian
+data — untuk maintain integrity research findings. Source code + data
+public di GitHub untuk reproducibility.
+
+---
+
+**Submitted**: 17 Juni 2026 (vClass) — *placeholder, submission setelah final review*
+**License**: MIT (lihat `LICENSE`)

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 TKI-KOS Team — Universitas Lampung
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,11 +1,237 @@
 ---
-title: Kozynear
-emoji: 👀
-colorFrom: indigo
-colorTo: green
+title: KozyNear
+emoji: 🏠
+colorFrom: blue
+colorTo: indigo
 sdk: docker
+app_port: 10000
 pinned: false
 license: mit
+short_description: Indonesian Kos-Kosan Search Engine Bandar Lampung
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# KozyNear — Indonesian Kos-Kosan Search Engine
+
+> Final project Mata Kuliah **Temu Kembali Informasi** (COM620321, 3 SKS) — Universitas Lampung (UNILA)
+> Search engine kos-kosan **seluruh Bandar Lampung** — 20 kecamatan, 9 universitas (UNILA, ITERA, Darmajaya, UBL, UIN, Teknokrat, Polinela, Malahayati, Saburai)
+
+🌐 **Live**: deploy via [HuggingFace Spaces](https://huggingface.co/spaces) (lihat [docs/deploy_huggingface.md](docs/deploy_huggingface.md))
+📖 **Laporan**: [LAPORAN.md](LAPORAN.md)
+
+![Status](https://img.shields.io/badge/status-deployed-green)
+![Deadline](https://img.shields.io/badge/deadline-17%20Jun%202026-red)
+![License](https://img.shields.io/badge/license-MIT-blue)
+![Python](https://img.shields.io/badge/python-3.11-blue)
+![React](https://img.shields.io/badge/react-18-61dafb)
+
+## Overview
+
+KozyNear adalah Information Retrieval system untuk listing kos-kosan di **seluruh Bandar Lampung** — berguna untuk mahasiswa dari **9 universitas** (UNILA, ITERA, Darmajaya, UBL, UIN Raden Intan, Teknokrat, Polinela, Malahayati, Saburai). User cari kos dengan natural language query bahasa Indonesia, dengan **5 model retrieval**:
+
+1. **Lexical baseline** — TF-IDF + cosine similarity (scikit-learn)
+2. **Probabilistic lexical** — BM25 (`rank_bm25`)
+3. **Neural / semantic** — MiniLM multilingual sentence embeddings (via fastembed ONNX; label kode `indobert` historis) + FAISS
+4. **Hybrid** — BM25 top-50 candidate → MiniLM rerank top-10
+5. **Smart (default live)** — query understanding (parser gender/harga/fasilitas/anchor) + geo ranking (gazetteer kampus terverifikasi + haversine) + BM25 fusion; hard filter dengan relaxation jujur
+
+UI punya 4 tab demo: **Pencarian**, **Evaluasi Model** (metrik live + signifikansi), **Preprocessing** (visualisasi 9-stage per langkah), **Statistik** (distribusi corpus).
+
+## Tech Stack
+
+| Layer | Tech |
+|-------|------|
+| Frontend | React 18 + Vite 5 + TypeScript |
+| Backend | Python 3.11 + FastAPI + Uvicorn |
+| Database | PostgreSQL 16 (asyncpg + SQLAlchemy 2.0) |
+| Migrations | Alembic |
+| IR Libraries | Sastrawi, rank_bm25, fastembed (ONNX MiniLM), faiss-cpu, scikit-learn |
+| Eval | scipy (Wilcoxon + Holm-Bonferroni), custom Kappa + IR metrics + CS@K |
+| Deploy | HF Spaces (16GB, neural ON) + Render.com backup; satu Dockerfile, gated `ENABLE_NEURAL` |
+
+## Quick Start (Local Dev)
+
+### Prerequisites
+
+- Python 3.11+, Node 20+, Docker Desktop, Git
+
+### Setup
+
+```bash
+# 1. Clone repo
+git clone https://github.com/DYmazeh/TKI-KOS.git
+cd TKI-KOS
+
+# 2. Start local PostgreSQL via Docker
+docker compose up -d
+
+# 3. Backend setup
+cd backend
+python -m venv .venv
+.venv\Scripts\activate          # Windows
+# source .venv/bin/activate      # macOS/Linux
+pip install -r requirements.txt
+cp .env.example .env
+
+# 4. Run migrations
+alembic upgrade head
+
+# 5. (Optional, kalau scrape data sudah ada) Seed DB + preprocess
+python -m scripts.seed_db --input ../data/raw/mamikos.jsonl \
+    --preprocess --corpus-output ../data/processed/corpus.json
+
+# 6. (Optional, kalau corpus.json sudah ada) Build indexes
+python -m app.indexing.build --corpus ../data/processed/corpus.json \
+    --output-dir ../data/indexes
+
+# 7. Run backend
+uvicorn app.main:app --reload --port 8000
+# Backend: http://localhost:8000 -- docs: http://localhost:8000/docs
+
+# 8. Frontend (terminal baru)
+cd frontend
+npm install
+cp .env.example .env
+npm run dev
+# Frontend: http://localhost:5173
+```
+
+## API Endpoints
+
+| Method | Path | Description |
+|--------|------|-------------|
+| GET | `/` | Root info |
+| GET | `/health` | Liveness probe (Render uses this) |
+| GET | `/api/search` | IR search dengan filter (`q`, `model`, `top_k`, `harga_min`, `harga_max`, `tipe`, `kecamatan`); filter berlaku untuk semua model termasuk smart |
+| GET | `/api/listings/{id}` | Detail kos by ID |
+| GET | `/api/eval/summary` | Aggregate metrics (standard + pool-restricted + CS@5 + Wilcoxon/Holm), file-based |
+| GET | `/api/eval/query/{query_id}` | Per-query metric breakdown |
+| GET | `/api/preprocess` | Trace pipeline 9-stage per langkah |
+| GET | `/api/stats` | Statistik corpus live |
+| GET | `/api/status` | Diagnostics (index loaded, RAM, DB) |
+| GET | `/api/docs` | Swagger UI (auto) |
+
+## Project Structure
+
+```
+TKI-KOS/
+├── frontend/                # React Vite SPA (TypeScript)
+│   ├── src/
+│   │   ├── api/             # API client
+│   │   ├── components/      # FilterPanel, ResultCard
+│   │   └── App.tsx
+│   └── package.json
+├── backend/                 # Python FastAPI
+│   ��── app/
+│   │   ├── api/             # Routes: health, search, listings, eval
+│   │   ├── core/            # config, db
+│   │   ├── models/          # ORM + Pydantic schemas
+│   │   ├── scraper/         # Mamikos + OLX + base + utils
+│   │   ├── preprocessing/   # 9-stage pipeline + 108 jargon dict
+│   │   ├── indexing/        # TF-IDF + BM25 + IndoBERT + Hybrid
+│   │   ├── search/          # query handling
+│   │   ├── evaluation/      # P@K, MAP, NDCG, Kappa, stat tests
+│   │   └── main.py          # FastAPI entry + lifespan
+│   ├── alembic/             # Database migrations
+│   ├── scripts/             # seed_db.py
+│   ├── tests/               # pytest suite
+│   └── requirements.txt
+├── data/
+│   ├── raw/                 # Scraped JSONL (gitignored)
+│   ├── processed/           # corpus.json setelah preprocessing
+│   └── indexes/             # TF-IDF/BM25/FAISS serialized
+├── notebooks/               # EDA, preprocessing experiment, eval
+├── eval/                    # queries.json, ground_truth.csv, results.csv
+├── docs/                    # architecture, deploy guide, GT protocol
+├── docker-compose.yml       # Local PG
+├── render.yaml              # Render infrastructure-as-code
+└── README.md
+```
+
+## Roles & Team (5 anggota)
+
+| Role | Bidang | Module |
+|------|--------|--------|
+| A — Lead / Scraper | Koordinasi tim + Mamikos scraper + data quality | `backend/app/scraper/` |
+| B — Preprocessing | Custom jargon dictionary + pipeline tuning | `backend/app/preprocessing/` |
+| C — IR Baseline | TF-IDF + BM25 hyperparameter tuning | `backend/app/indexing/` |
+| D — IR Neural | IndoBERT + FAISS + hybrid tuning | `backend/app/indexing/` |
+| E — Frontend + Eval | React UI + ground truth coord + metrics | `frontend/` + `backend/app/evaluation/` |
+
+## Timeline (4-week sprint)
+
+| Week | Dates | Milestone |
+|------|-------|-----------|
+| 1 | 20–26 Mei 2026 | Repo setup, Mamikos scraper, ≥1500 listings |
+| 2 | 27 Mei–2 Jun | Preprocessing pipeline, 3 IR indexes, FastAPI scaffold |
+| 3 | 3–9 Jun | React UI live, ground truth annotation (Kappa ≥0.7), deploy Render |
+| 4 | 10–16 Jun | Evaluation, statistical tests, report finalization |
+| **Submit** | **17 Jun 23:59 WIB** | Upload ke vClass UNILA |
+
+## Documentation
+
+| Topic | File |
+|-------|------|
+| Architecture (komponen, data flow, deploy strategy) | [docs/architecture.md](docs/architecture.md) |
+| Deploy guide step-by-step (Render.com) | [docs/deploy_guide.md](docs/deploy_guide.md) |
+| Ground truth annotation protocol (3-annotator + Kappa) | [docs/ground_truth_protocol.md](docs/ground_truth_protocol.md) |
+| Scraper usage + TODOs | [backend/app/scraper/README.md](backend/app/scraper/README.md) |
+| Preprocessing pipeline + jargon dict | [backend/app/preprocessing/README.md](backend/app/preprocessing/README.md) |
+| IR indexes (TF-IDF/BM25/IndoBERT/Hybrid) | [backend/app/indexing/README.md](backend/app/indexing/README.md) |
+| Evaluation metrics + Kappa + stat tests | [backend/app/evaluation/README.md](backend/app/evaluation/README.md) |
+| Data schema + acquisition methodology | [data/README.md](data/README.md) |
+| Notebooks convention | [notebooks/README.md](notebooks/README.md) |
+
+## Course Rubric Coverage
+
+| Aspect | Weight | Status |
+|--------|--------|--------|
+| Theme | 15% | Kos-kosan **seluruh Bandar Lampung**, multi-university (UNILA, ITERA, Darmajaya, UBL, UIN, Teknokrat, Polinela, Malahayati, Saburai) |
+| Dataset | 15% | **227 listing REAL Mamikos** (scrape detail-page, deskripsi pemilik + koordinat asli, 18 kecamatan), methodology + pipeline disclosed |
+| Preprocessing | 15% | 9-stage pipeline + ~110 domain jargon entries (>=100 rubric req) + visualisasi live per-stage |
+| Model | 20% | 5 model (TF-IDF/BM25/Neural MiniLM/Hybrid/**Smart live**) — Smart teratas standard MAP 0.301 + CS@5 0.86 vs BM25 0.45 (p<0.001, n=30); hyperparameter via grid search; pooling-bias dibahas jujur |
+| Evaluation | 10% | 30 query, 900 annotations; P@K + MAP + NDCG + MRR + Cohen's Kappa (all pair >=0.58) + Wilcoxon + **Holm-Bonferroni** + Constraint-Satisfaction@5 |
+| System | 25% | FastAPI + React 4-tab + Postgres single Docker; HF Spaces (neural ON) + Render backup |
+
+## Deploy URLs
+
+- **Live App (primary)**: https://dymazeh-kozynear.hf.space (HF Spaces 16GB, `ENABLE_NEURAL=true`: kelima model live) &mdash; lihat [docs/deploy_huggingface.md](docs/deploy_huggingface.md)
+- **Backup**: https://kozynear.onrender.com (Render free 512MB, smart/BM25/TF-IDF)
+- **API Docs**: `<base-url>/api/docs` (Swagger UI)
+- **Health**: `<base-url>/health`
+
+⚠️ Render free spin down 15 menit idle (cold start ~30-60s); HF Spaces sleep setelah 48 jam idle. Workflow [keepalive.yml](.github/workflows/keepalive.yml) ping keduanya tiap 12 menit.
+
+## Live Results Summary (n=30 queries × 5 models, corpus real 227)
+
+**Standard top-K** (IR atas seluruh corpus):
+
+| Model | P@5 | P@10 | MAP | NDCG@10 | MRR |
+|-------|-----|------|-----|---------|-----|
+| **Smart (live)** | **0.580** | 0.480 | **0.301** | 0.548 | **0.868** |
+| BM25 | 0.553 | **0.530** | 0.270 | **0.553** | 0.802 |
+| Hybrid (α=0.9) | 0.553 | 0.527 | 0.267 | 0.551 | 0.804 |
+| TF-IDF | 0.547 | 0.510 | 0.239 | 0.523 | 0.727 |
+| Neural MiniLM | 0.160 | 0.153 | 0.057 | 0.155 | 0.280 |
+
+**Pool-restricted** (fair &mdash; Neural MAP melompat 0.057 → **0.557**, membuktikan skor rendah di atas adalah *pooling bias*, bukan model buruk; angka Smart di lensa ini optimistic karena sinyal filter overlap dengan heuristik annotator, lihat LAPORAN §7.1b):
+
+| Model | P@5 | MAP |
+|-------|-----|-----|
+| Smart (live) | 0.927 | 0.865 |
+| Hybrid (α=0.9) | 0.553 | 0.591 |
+| **BM25** | 0.553 | 0.583 |
+| TF-IDF | 0.560 | 0.576 |
+| Neural MiniLM | 0.487 | 0.557 |
+
+**Constraint Satisfaction @5** (lensa bebas qrels: % top-5 yang memenuhi SEMUA kebutuhan user &mdash; gender, budget, fasilitas, radius 3 km kampus):
+
+| Model | mean CS@5 (n=30) |
+|-------|------------------|
+| **Smart (live)** | **0.863** |
+| BM25 | 0.447 |
+
+Smart vs BM25 signifikan (Wilcoxon p&lt;0.001). Signifikansi pairwise MAP memakai koreksi **Holm-Bonferroni**: **4/10 signifikan** (semuanya pasangan vs Neural di standard eval; bertahan setelah Holm). Hyperparameter via grid search: alpha hybrid 0.9, bobot fusion smart 0.2/0.4/0.4. Detail di [LAPORAN.md](LAPORAN.md) + [eval/eval_summary.md](eval/eval_summary.md).
+
+## License
+
+MIT &mdash; lihat [LICENSE](LICENSE)

backend/.env.example

@@ -0,0 +1,6 @@
+ENVIRONMENT=development
+DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/tki_kos
+CORS_ORIGINS=["http://localhost:5173","http://localhost:3000"]
+EMBEDDING_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+INDEXES_DIR=../data/indexes
+LOG_LEVEL=INFO

backend/alembic.ini

@@ -0,0 +1,49 @@
+# Alembic config untuk TKI-KOS backend
+# URL di-override dari app.core.config.settings di env.py
+
+[alembic]
+script_location = alembic
+prepend_sys_path = .
+version_path_separator = os
+sqlalchemy.url = postgresql+asyncpg://postgres:postgres@localhost:5432/tki_kos
+
+[post_write_hooks]
+# Optional: auto-format migration files setelah generate
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = REVISION_SCRIPT_FILENAME
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

backend/alembic/env.py

@@ -0,0 +1,78 @@
+"""Alembic environment script untuk async SQLAlchemy 2.0.
+
+Migration commands (dari backend/):
+    alembic revision --autogenerate -m "create listings table"
+    alembic upgrade head
+    alembic downgrade -1
+    alembic history
+"""
+
+from __future__ import annotations
+
+import asyncio
+from logging.config import fileConfig
+
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+
+from alembic import context
+
+from app.core.config import settings
+from app.core.db import Base
+
+# Import ALL models di sini supaya Alembic detect untuk autogenerate
+from app.models import listing  # noqa: F401
+
+
+config = context.config
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# Override URL dari env-driven settings
+config.set_main_option("sqlalchemy.url", settings.database_url)
+
+target_metadata = Base.metadata
+
+
+def run_migrations_offline() -> None:
+    """Run migrations dalam offline mode (generate SQL ke stdout)."""
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+    context.configure(connection=connection, target_metadata=target_metadata)
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+    """Run migrations dengan async engine."""
+    connectable = async_engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+
+    await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+    asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

backend/alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

backend/alembic/versions/001_initial_schema.py

@@ -0,0 +1,129 @@
+"""initial schema: listings + queries + ground_truth + eval_results
+
+Revision ID: 001
+Revises:
+Create Date: 2026-05-26 00:00:00
+"""
+
+from __future__ import annotations
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic
+revision: str = "001"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ------------------------------------------------------------------
+    # listings: raw + processed kos data
+    # ------------------------------------------------------------------
+    op.create_table(
+        "listings",
+        sa.Column("id", sa.String(), primary_key=True),
+        sa.Column("judul", sa.Text(), nullable=False),
+        sa.Column("deskripsi", sa.Text(), nullable=False),
+        sa.Column("deskripsi_processed", sa.Text(), nullable=True),
+        sa.Column("harga_per_bulan", sa.Integer(), nullable=True),
+        sa.Column("tipe", sa.String(length=20), nullable=True),
+        sa.Column("fasilitas", sa.ARRAY(sa.String()), nullable=True),
+        sa.Column("alamat", sa.Text(), nullable=True),
+        sa.Column("kecamatan", sa.String(length=100), nullable=True),
+        sa.Column("koordinat_lat", sa.Numeric(precision=8, scale=5), nullable=True),
+        sa.Column("koordinat_lng", sa.Numeric(precision=8, scale=5), nullable=True),
+        sa.Column("jarak_kampus_km", sa.Numeric(precision=5, scale=2), nullable=True),
+        sa.Column("url_source", sa.Text(), nullable=True),
+        sa.Column("scrape_date", sa.Date(), nullable=True),
+        sa.Column("source", sa.String(length=20), nullable=True),
+        sa.Column(
+            "inserted_at",
+            sa.DateTime(),
+            server_default=sa.func.now(),
+            nullable=True,
+        ),
+    )
+
+    # Indexes umum
+    op.create_index("ix_listings_tipe", "listings", ["tipe"])
+    op.create_index("ix_listings_kecamatan", "listings", ["kecamatan"])
+    op.create_index("ix_listings_harga", "listings", ["harga_per_bulan"])
+
+    # ------------------------------------------------------------------
+    # queries: query set untuk eval
+    # ------------------------------------------------------------------
+    op.create_table(
+        "queries",
+        sa.Column("id", sa.String(), primary_key=True),
+        sa.Column("query_text", sa.Text(), nullable=False),
+        sa.Column("context", sa.Text(), nullable=True),
+        sa.Column("expected_tipe", sa.String(length=20), nullable=True),
+    )
+
+    # ------------------------------------------------------------------
+    # ground_truth: raw annotation per annotator (3-annotator setup)
+    # ------------------------------------------------------------------
+    op.create_table(
+        "ground_truth",
+        sa.Column("query_id", sa.String(), nullable=False),
+        sa.Column("listing_id", sa.String(), nullable=False),
+        sa.Column("annotator", sa.String(length=50), nullable=False),
+        sa.Column("relevance", sa.SmallInteger(), nullable=False),
+        sa.Column("notes", sa.Text(), nullable=True),
+        sa.CheckConstraint("relevance IN (0, 1, 2)", name="ck_relevance_0_1_2"),
+        sa.ForeignKeyConstraint(["query_id"], ["queries.id"]),
+        sa.ForeignKeyConstraint(["listing_id"], ["listings.id"]),
+        sa.PrimaryKeyConstraint("query_id", "listing_id", "annotator"),
+    )
+
+    # ------------------------------------------------------------------
+    # ground_truth_consensus: hasil consensus 3 annotator (input ke eval runner)
+    # ------------------------------------------------------------------
+    op.create_table(
+        "ground_truth_consensus",
+        sa.Column("query_id", sa.String(), nullable=False),
+        sa.Column("listing_id", sa.String(), nullable=False),
+        sa.Column("relevance", sa.SmallInteger(), nullable=False),
+        sa.CheckConstraint("relevance IN (0, 1, 2)", name="ck_consensus_relevance"),
+        sa.ForeignKeyConstraint(["query_id"], ["queries.id"]),
+        sa.ForeignKeyConstraint(["listing_id"], ["listings.id"]),
+        sa.PrimaryKeyConstraint("query_id", "listing_id"),
+    )
+
+    # ------------------------------------------------------------------
+    # eval_results: per-model per-query metrics
+    # ------------------------------------------------------------------
+    op.create_table(
+        "eval_results",
+        sa.Column("model", sa.String(length=20), nullable=False),
+        sa.Column("query_id", sa.String(), nullable=False),
+        sa.Column("precision_at_5", sa.Numeric(precision=5, scale=4), nullable=True),
+        sa.Column("precision_at_10", sa.Numeric(precision=5, scale=4), nullable=True),
+        sa.Column("average_precision", sa.Numeric(precision=5, scale=4), nullable=True),
+        sa.Column("ndcg_at_10", sa.Numeric(precision=5, scale=4), nullable=True),
+        sa.Column("reciprocal_rank", sa.Numeric(precision=5, scale=4), nullable=True),
+        sa.Column(
+            "computed_at",
+            sa.DateTime(),
+            server_default=sa.func.now(),
+            nullable=True,
+        ),
+        sa.ForeignKeyConstraint(["query_id"], ["queries.id"]),
+        sa.PrimaryKeyConstraint("model", "query_id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("eval_results")
+    op.drop_table("ground_truth_consensus")
+    op.drop_table("ground_truth")
+    op.drop_table("queries")
+    op.drop_index("ix_listings_harga", table_name="listings")
+    op.drop_index("ix_listings_kecamatan", table_name="listings")
+    op.drop_index("ix_listings_tipe", table_name="listings")
+    op.drop_table("listings")

backend/app/__init__.py

@@ -0,0 +1 @@
+"""TKI-KOS application package."""

backend/app/api/__init__.py

@@ -0,0 +1 @@
+"""HTTP API routes."""

backend/app/api/eval.py

@@ -0,0 +1,170 @@
+"""Evaluation dashboard endpoints (file-based).
+
+Sumber data: CSV hasil eval yang di-COMMIT di repo (eval/results.csv,
+results_pool_restricted.csv, results_constraints.csv, significance_map.csv).
+Sebelumnya endpoint ini membaca tabel `eval_results` yang TIDAK PERNAH diisi
+oleh siapa pun (runner hanya menulis CSV) sehingga selalu kosong di production.
+File-based = nol langkah deploy ekstra dan hasil identik dengan laporan.
+"""
+
+from __future__ import annotations
+
+import csv
+from collections import defaultdict
+from pathlib import Path
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+router = APIRouter(prefix="/eval", tags=["evaluation"])
+
+# repo root: backend/app/api/eval.py -> parents[3]
+_EVAL_DIR = Path(__file__).resolve().parents[3] / "eval"
+
+
+class ModelMetrics(BaseModel):
+    model: str
+    p_at_5: float
+    p_at_10: float
+    map: float
+    ndcg_at_10: float
+    mrr: float
+    n_queries: int
+
+
+class ConstraintSummary(BaseModel):
+    n_queries: int
+    mean_cs_at_5: dict[str, float]
+
+
+class SignificanceRow(BaseModel):
+    pair: str
+    p_value: float
+    p_holm: float
+    significant_raw: bool
+    significant_holm: bool
+
+
+class EvalSummary(BaseModel):
+    standard: list[ModelMetrics]
+    pool_restricted: list[ModelMetrics]
+    constraints: ConstraintSummary | None
+    significance: list[SignificanceRow]
+    total_queries: int
+    note: str
+
+
+class QueryResult(BaseModel):
+    query_id: str
+    query_text: str
+    model: str
+    p_at_5: float
+    p_at_10: float
+    average_precision: float
+    ndcg_at_10: float
+    reciprocal_rank: float
+
+
+def _aggregate_results_csv(path: Path) -> tuple[list[ModelMetrics], int]:
+    """results.csv -> aggregate per model + jumlah query."""
+    if not path.exists():
+        return [], 0
+    per_model: dict[str, list[dict]] = defaultdict(list)
+    with open(path, encoding="utf-8") as f:
+        for row in csv.DictReader(f):
+            per_model[row["model"]].append(row)
+    out: list[ModelMetrics] = []
+    n_queries = 0
+    for model, rows in sorted(per_model.items()):
+        n = len(rows)
+        n_queries = max(n_queries, n)
+        out.append(ModelMetrics(
+            model=model,
+            p_at_5=sum(float(r["p_at_5"]) for r in rows) / n,
+            p_at_10=sum(float(r["p_at_10"]) for r in rows) / n,
+            map=sum(float(r["ap"]) for r in rows) / n,
+            ndcg_at_10=sum(float(r["ndcg_at_10"]) for r in rows) / n,
+            mrr=sum(float(r["rr"]) for r in rows) / n,
+            n_queries=n,
+        ))
+    out.sort(key=lambda m: -m.map)
+    return out, n_queries
+
+
+def _load_constraints(path: Path) -> ConstraintSummary | None:
+    if not path.exists():
+        return None
+    with open(path, encoding="utf-8") as f:
+        rows = list(csv.DictReader(f))
+    if not rows:
+        return None
+    return ConstraintSummary(
+        n_queries=len(rows),
+        mean_cs_at_5={
+            "smart": sum(float(r["cs_at_5_smart"]) for r in rows) / len(rows),
+            "bm25": sum(float(r["cs_at_5_bm25"]) for r in rows) / len(rows),
+        },
+    )
+
+
+def _load_significance(path: Path) -> list[SignificanceRow]:
+    if not path.exists():
+        return []
+    with open(path, encoding="utf-8") as f:
+        return [
+            SignificanceRow(
+                pair=r["pair"],
+                p_value=float(r["p_value"]),
+                p_holm=float(r["p_holm"]),
+                significant_raw=r["significant_raw"] == "yes",
+                significant_holm=r["significant_holm"] == "yes",
+            )
+            for r in csv.DictReader(f)
+        ]
+
+
+@router.get("/summary", response_model=EvalSummary)
+async def eval_summary() -> EvalSummary:
+    """Aggregate metrics per IR model: standard, pool-restricted, CS@5, Wilcoxon+Holm."""
+    standard, n_queries = _aggregate_results_csv(_EVAL_DIR / "results.csv")
+    pool, _ = _aggregate_results_csv(_EVAL_DIR / "results_pool_restricted.csv")
+    return EvalSummary(
+        standard=standard,
+        pool_restricted=pool,
+        constraints=_load_constraints(_EVAL_DIR / "results_constraints.csv"),
+        significance=_load_significance(_EVAL_DIR / "significance_map.csv"),
+        total_queries=n_queries,
+        note=(
+            "Standard top-K memakai qrels yang di-pool dari kandidat lexical "
+            "(pooling bias menekan model semantic & smart geo-augment); "
+            "pool-restricted = ranking di dalam pool annotated (fair); "
+            "CS@5 = % top-5 yang memenuhi semua constraint user (bebas qrels). "
+            "Signifikansi: Wilcoxon signed-rank + koreksi Holm-Bonferroni."
+        ),
+    )
+
+
+@router.get("/query/{query_id}", response_model=list[QueryResult])
+async def eval_per_query(query_id: str) -> list[QueryResult]:
+    """Per-model metric untuk satu query (drill-down dari results.csv)."""
+    path = _EVAL_DIR / "results.csv"
+    if not path.exists():
+        raise HTTPException(404, "results.csv belum ada — jalankan evaluasi dulu")
+    out: list[QueryResult] = []
+    with open(path, encoding="utf-8") as f:
+        for r in csv.DictReader(f):
+            if r["query_id"] == query_id:
+                out.append(QueryResult(
+                    query_id=r["query_id"],
+                    query_text=r["query"],
+                    model=r["model"],
+                    p_at_5=float(r["p_at_5"]),
+                    p_at_10=float(r["p_at_10"]),
+                    average_precision=float(r["ap"]),
+                    ndcg_at_10=float(r["ndcg_at_10"]),
+                    reciprocal_rank=float(r["rr"]),
+                ))
+    if not out:
+        raise HTTPException(404, f"Query '{query_id}' tidak ada di results.csv")
+    out.sort(key=lambda x: x.model)
+    return out

backend/app/api/health.py

@@ -0,0 +1,138 @@
+"""Health check + status diagnostics endpoints.
+
+- /health: simple liveness probe untuk Render
+- /api/info: basic version info
+- /api/status: comprehensive diagnostics (indexes, DB, model warm state)
+  Penting untuk debug user-reported issues karena user copy-paste output ini.
+"""
+
+from __future__ import annotations
+
+import platform
+import sys
+from datetime import datetime, timezone
+
+from fastapi import APIRouter, Depends, Request
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.db import get_session
+
+router = APIRouter()
+
+
+@router.get("/health")
+async def health():
+    """Liveness probe untuk Render.com."""
+    return {"status": "ok", "service": "kozynear"}
+
+
+@router.get("/api/info")
+async def api_info():
+    """Basic API info."""
+    return {
+        "name": "KozyNear API",
+        "version": "0.2.0",
+        "scope": "Bandar Lampung full coverage (20 kecamatan, 9 universitas)",
+        "docs": "/api/docs",
+        "health": "/health",
+        "status": "/api/status",
+    }
+
+
+@router.get("/api/status")
+async def status(
+    request: Request,
+    session: AsyncSession = Depends(get_session),
+):
+    """Comprehensive diagnostics — paste output ini saat report bug.
+
+    Returns:
+    - Service: name, version, environment, uptime
+    - Indexes loaded (tfidf, bm25, indobert, hybrid)
+    - Preprocessing pipeline loaded
+    - Database: listings count, connection ok
+    - Runtime: Python version, platform
+    """
+    from app.core.config import settings
+
+    # Index availability checks
+    state = request.app.state
+    indexes_status = {
+        "tfidf": state.tfidf is not None,
+        "bm25": state.bm25 is not None,
+        "indobert": state.indobert is not None,
+        "hybrid": state.hybrid is not None,
+    }
+    indobert_ready = getattr(state, "indobert_ready", False)
+
+    # Index sizes (kalau loaded)
+    indexes_size = {}
+    for name in ("tfidf", "bm25", "indobert"):
+        idx = getattr(state, name, None)
+        if idx is not None and hasattr(idx, "size"):
+            try:
+                indexes_size[name] = idx.size()
+            except Exception:
+                indexes_size[name] = None
+
+    # DB row counts
+    db_ok = True
+    db_error = None
+    listings_count = None
+    try:
+        from app.models.listing import Listing
+
+        result = await session.execute(select(func.count(Listing.id)))
+        listings_count = int(result.scalar() or 0)
+    except Exception as e:
+        db_ok = False
+        db_error = str(e)
+
+    # Memory usage (kalau psutil ada)
+    memory_info = None
+    try:
+        import psutil
+
+        process = psutil.Process()
+        mem = process.memory_info()
+        memory_info = {
+            "rss_mb": round(mem.rss / 1024 / 1024, 1),
+            "vms_mb": round(mem.vms / 1024 / 1024, 1),
+        }
+    except ImportError:
+        pass
+
+    return {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "service": {
+            "name": "kozynear",
+            "version": "0.2.0",
+            "environment": settings.environment,
+        },
+        "preprocessing": {
+            "loaded": state.preprocessing_pipeline is not None,
+        },
+        "indexes": {
+            "loaded": indexes_status,
+            "sizes": indexes_size,
+            "indexes_dir": settings.indexes_dir,
+            "indobert_model_ready": indobert_ready,
+        },
+        "database": {
+            "connected": db_ok,
+            "listings_count": listings_count,
+            "error": db_error,
+        },
+        "runtime": {
+            "python": sys.version.split()[0],
+            "platform": platform.platform(),
+        },
+        "memory": memory_info,
+        "settings": {
+            "embedding_model": settings.embedding_model,
+            "default_top_k": settings.default_top_k,
+            "bm25_k1": settings.bm25_k1,
+            "bm25_b": settings.bm25_b,
+        },
+    }

backend/app/api/insights.py

@@ -0,0 +1,126 @@
+"""Endpoint demo/insight: visualisasi preprocessing + statistik corpus.
+
+Diporting dari konsep prototype Flask `proyek stki` (tab Preprocessing +
+/api/stats) — fitur paling berguna untuk demo rubric Preprocessing & Sistem.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.db import get_session
+from app.models.listing import Listing
+
+router = APIRouter()
+
+# Fallback statistik kalau DB down: corpus.json ikut ter-copy di image.
+_CORPUS_JSON = Path(__file__).resolve().parents[3] / "data" / "processed" / "corpus.json"
+
+
+@router.get("/preprocess", tags=["insights"])
+async def preprocess_trace(
+    request: Request,
+    text: str = Query(..., min_length=1, max_length=1000,
+                      description="Teks untuk dilewatkan pipeline 9-stage"),
+):
+    """Jalankan pipeline preprocessing dengan trace per-stage (untuk UI demo)."""
+    pipeline = getattr(request.app.state, "preprocessing_pipeline", None)
+    if pipeline is None:
+        raise HTTPException(503, detail={"error": "preprocessing pipeline belum loaded"})
+    result = pipeline.process(text, trace=True)
+    return {
+        "raw": result.raw,
+        "processed": result.processed,
+        "tokens": result.tokens,
+        "extracted_prices": result.extracted_prices,
+        "stages": result.trace,
+    }
+
+
+def _corpus_fallback_stats() -> dict:
+    """Statistik dari corpus.json saat DB tidak tersedia (mis. test lokal)."""
+    docs = json.loads(_CORPUS_JSON.read_text(encoding="utf-8"))
+    kecamatan: dict[str, int] = {}
+    tipe: dict[str, int] = {}
+    hargas: list[int] = []
+    for d in docs:
+        meta = d.get("metadata", d)
+        kec = meta.get("kecamatan") or "?"
+        kecamatan[kec] = kecamatan.get(kec, 0) + 1
+        t = meta.get("tipe") or "?"
+        tipe[t] = tipe.get(t, 0) + 1
+        h = meta.get("harga_per_bulan")
+        if isinstance(h, int):
+            hargas.append(h)
+    return {
+        "total_listings": len(docs),
+        "kecamatan": dict(sorted(kecamatan.items(), key=lambda kv: -kv[1])),
+        "tipe": tipe,
+        "harga_min": min(hargas) if hargas else None,
+        "harga_max": max(hargas) if hargas else None,
+        "harga_avg": int(sum(hargas) / len(hargas)) if hargas else None,
+        "source": "corpus.json (DB tidak tersedia)",
+    }
+
+
+@router.get("/stats", tags=["insights"])
+async def corpus_stats(
+    request: Request,
+    session: AsyncSession = Depends(get_session),
+):
+    """Statistik corpus live: jumlah listing, distribusi kecamatan/tipe, harga,
+    ukuran vocabulary index. Untuk tab Statistik + bahan slide."""
+    stats: dict
+    try:
+        total = (await session.execute(select(func.count(Listing.id)))).scalar() or 0
+        kec_rows = (await session.execute(
+            select(Listing.kecamatan, func.count())
+            .group_by(Listing.kecamatan)
+            .order_by(func.count().desc())
+        )).all()
+        tipe_rows = (await session.execute(
+            select(Listing.tipe, func.count()).group_by(Listing.tipe)
+        )).all()
+        harga_row = (await session.execute(
+            select(
+                func.min(Listing.harga_per_bulan),
+                func.max(Listing.harga_per_bulan),
+                func.avg(Listing.harga_per_bulan),
+            )
+        )).one()
+        stats = {
+            "total_listings": int(total),
+            "kecamatan": {k or "?": int(c) for k, c in kec_rows},
+            "tipe": {t or "?": int(c) for t, c in tipe_rows},
+            "harga_min": int(harga_row[0]) if harga_row[0] is not None else None,
+            "harga_max": int(harga_row[1]) if harga_row[1] is not None else None,
+            "harga_avg": int(harga_row[2]) if harga_row[2] is not None else None,
+            "source": "database",
+        }
+    except Exception:
+        # DB down / belum seed -> fallback corpus committed (read-only stats)
+        try:
+            stats = _corpus_fallback_stats()
+        except Exception as e:  # corpus juga tidak ada
+            raise HTTPException(503, detail={"error": f"stats tidak tersedia: {e}"})
+
+    # Vocabulary size dari index yang loaded (BM25 idf vocab)
+    bm25 = getattr(request.app.state, "bm25", None)
+    vocab_size = None
+    if bm25 is not None and getattr(bm25, "bm25", None) is not None:
+        try:
+            vocab_size = len(bm25.bm25.idf)
+        except Exception:
+            vocab_size = None
+    stats["vocab_size"] = vocab_size
+    stats["models_loaded"] = [
+        m for m in ("tfidf", "bm25", "indobert", "hybrid")
+        if getattr(request.app.state, m, None) is not None
+    ] + (["smart"] if getattr(request.app.state, "bm25", None) is not None
+         and getattr(request.app.state, "gazetteer", None) is not None else [])
+    return stats

backend/app/api/listings.py

@@ -0,0 +1,62 @@
+"""Listing detail endpoint: GET /listings/{id}."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.db import get_session
+from app.models.listing import Listing, ListingDetail
+
+
+router = APIRouter()
+
+
+@router.get(
+    "/listings/{listing_id}",
+    response_model=ListingDetail,
+    tags=["listings"],
+)
+async def get_listing(
+    listing_id: str,
+    session: AsyncSession = Depends(get_session),
+) -> ListingDetail:
+    """Get detail listing by ID. Return 404 kalau gak ditemukan."""
+    result = await session.execute(
+        select(Listing).where(Listing.id == listing_id)
+    )
+    listing = result.scalar_one_or_none()
+    if not listing:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Listing '{listing_id}' not found",
+        )
+
+    coordinates = None
+    if listing.koordinat_lat is not None and listing.koordinat_lng is not None:
+        coordinates = [
+            float(listing.koordinat_lat),
+            float(listing.koordinat_lng),
+        ]
+
+    return ListingDetail(
+        id=listing.id,
+        judul=listing.judul,
+        deskripsi=listing.deskripsi,
+        deskripsi_processed=listing.deskripsi_processed,
+        harga_per_bulan=listing.harga_per_bulan,
+        tipe=listing.tipe,
+        fasilitas=listing.fasilitas,
+        alamat=listing.alamat,
+        kecamatan=listing.kecamatan,
+        koordinat=coordinates,
+        jarak_kampus_km=(
+            float(listing.jarak_kampus_km)
+            if listing.jarak_kampus_km is not None
+            else None
+        ),
+        url_source=listing.url_source,
+        scrape_date=listing.scrape_date,
+        source=listing.source,
+    )

backend/app/api/search.py

@@ -0,0 +1,232 @@
+"""Search endpoint: GET /search?q=X&model=Y&top_k=N + optional filters.
+
+Pipeline:
+1. Pilih IR index dari app.state berdasar param `model`
+2. Preprocess query (same as corpus preprocessing saat indexing)
+3. IR index query top-K * overshoot (3x) — supaya filter setelahnya
+   gak return < top_k
+4. Hydrate dari DB dengan filter (harga, tipe, kecamatan)
+5. Take top-K filtered, preserve ranking, return SearchResponse
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Literal, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from loguru import logger
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.db import get_session
+from app.models.listing import Listing, ListingRead
+from app.models.search import SearchResponse
+from app.search.pipeline import SmartFilters, smart_search
+
+
+router = APIRouter()
+
+
+# Overshoot factor: index query ambil top_k * X supaya filter gak terlalu
+# memotong hasil. 3 cukup untuk filter ringan; bisa naik kalau filter ketat.
+OVERSHOOT_FACTOR = 3
+
+
+@router.get("/search", response_model=SearchResponse, tags=["search"])
+async def search(
+    request: Request,
+    q: str = Query(..., min_length=1, max_length=500, description="Query string"),
+    model: Literal["tfidf", "bm25", "indobert", "hybrid", "smart"] = Query(
+        "smart", description="IR model pilihan (default: smart = query understanding + geo + BM25)"
+    ),
+    top_k: int = Query(10, ge=1, le=50, description="Jumlah hasil"),
+    # ---- Optional filters (metadata-based) ----
+    harga_min: Optional[int] = Query(
+        None, ge=0, description="Min harga per bulan (IDR)"
+    ),
+    harga_max: Optional[int] = Query(
+        None, ge=0, description="Max harga per bulan (IDR)"
+    ),
+    tipe: Optional[Literal["putra", "putri", "campur"]] = Query(
+        None, description="Filter tipe kos"
+    ),
+    kecamatan: Optional[str] = Query(
+        None, description="Filter kecamatan (substring match)"
+    ),
+    session: AsyncSession = Depends(get_session),
+) -> SearchResponse:
+    """Cari kos dengan IR model + optional metadata filters.
+
+    Returns 503 kalau index belum di-load di backend.
+    """
+    # 0. Smart pipeline (default): query understanding + geo + BM25 (no neural).
+    if model == "smart":
+        gz = getattr(request.app.state, "gazetteer", None)
+        bm25_index = getattr(request.app.state, "bm25", None)
+        if bm25_index is None or gz is None:
+            raise HTTPException(
+                status_code=503,
+                detail={"error": "smart pipeline belum siap (butuh bm25 + gazetteer)"},
+            )
+        # Residual query harus lewat preprocessing yang sama dengan corpus
+        # saat indexing (stem + jargon), kalau tidak token gak match index.
+        pipeline = getattr(request.app.state, "preprocessing_pipeline", None)
+        preprocess = (
+            (lambda s: pipeline.process(s).processed) if pipeline else None
+        )
+        # Filter UI eksplisit ikut ke smart sebagai hard constraint
+        # (sebelumnya diabaikan diam-diam di mode smart).
+        smart_filters = SmartFilters(
+            harga_min=harga_min, harga_max=harga_max,
+            tipe=tipe, kecamatan=kecamatan,
+        )
+        t0 = time.perf_counter()
+        results, understood, relaxed = await smart_search(
+            q, bm25_index, session, gz, top_k=top_k,
+            preprocess=preprocess, filters=smart_filters,
+        )
+        return SearchResponse(
+            query=q, model=model, top_k=top_k,
+            took_ms=int((time.perf_counter() - t0) * 1000),
+            results=results, understood=understood, relaxed=relaxed,
+        )
+
+    # 1. Get index dari app.state
+    index = getattr(request.app.state, model, None)
+    if index is None:
+        available = [
+            m for m in ("tfidf", "bm25", "indobert", "hybrid")
+            if getattr(request.app.state, m, None) is not None
+        ]
+        raise HTTPException(
+            status_code=503,
+            detail={
+                "error": f"Index '{model}' belum di-load di backend",
+                "model_requested": model,
+                "available_models": available,
+                "preprocessing_loaded": getattr(
+                    request.app.state, "preprocessing_pipeline", None
+                ) is not None,
+                "hint": (
+                    "Kemungkinan OOM saat prewarm IndoBERT di Render free tier "
+                    "(512MB RAM). Check Render logs untuk detail. Untuk "
+                    "diagnostics lengkap GET /api/status."
+                ),
+            },
+        )
+
+    # 1b. Check IndoBERT/Hybrid model warm state (background prewarm)
+    if model in ("indobert", "hybrid"):
+        if not getattr(request.app.state, "indobert_ready", False):
+            failed = getattr(request.app.state, "indobert_failed", False)
+            if failed:
+                raise HTTPException(
+                    status_code=503,
+                    detail={
+                        "error": "IndoBERT model gagal load (prewarm permanent failure)",
+                        "model_requested": model,
+                        "ready_models": [
+                            m for m in ("tfidf", "bm25")
+                            if getattr(request.app.state, m, None) is not None
+                        ],
+                        "hint": (
+                            "IndoBERT prewarm gagal permanen (kemungkinan OOM di "
+                            "Render free tier 512MB). Restart deployment atau gunakan "
+                            "TF-IDF/BM25. Retry TIDAK akan membantu tanpa restart."
+                        ),
+                    },
+                )
+            raise HTTPException(
+                status_code=503,
+                detail={
+                    "error": "IndoBERT model masih loading (background prewarm)",
+                    "model_requested": model,
+                    "ready_models": [
+                        m for m in ("tfidf", "bm25")
+                        if getattr(request.app.state, m, None) is not None
+                    ],
+                    "hint": (
+                        "Background prewarm jalan di startup, butuh ~10-30 detik "
+                        "di Render free tier. Coba TF-IDF/BM25 dulu yang sudah "
+                        "ready, atau retry IndoBERT/Hybrid setelah ~30 detik."
+                    ),
+                    "retry_after_sec": 30,
+                },
+                headers={"Retry-After": "30"},
+            )
+
+    t0 = time.perf_counter()
+
+    # 2. Preprocess query (untuk BM25/TF-IDF). IndoBERT & Hybrid pakai raw
+    # query karena sentence-transformers ekspektasi natural language.
+    pipeline = getattr(request.app.state, "preprocessing_pipeline", None)
+    processed_q = pipeline.process(q).processed if pipeline else q
+    search_q = q if model in ("indobert", "hybrid") else processed_q
+    logger.debug(
+        f"[search] q='{q}' processed='{processed_q}' model={model} "
+        f"using={'raw' if model in ('indobert', 'hybrid') else 'processed'}"
+    )
+
+    # 3. IR query — overshoot kalau ada filter (mitigate filter cutting hits)
+    has_filter = any([harga_min is not None, harga_max is not None, tipe is not None, kecamatan is not None])
+    fetch_k = top_k * OVERSHOOT_FACTOR if has_filter else top_k
+    hits = index.query(search_q, top_k=fetch_k)
+    if not hits:
+        return SearchResponse(
+            query=q, model=model, top_k=top_k, took_ms=0, results=[]
+        )
+
+    doc_ids = [h.doc_id for h in hits]
+    score_map = {h.doc_id: h.score for h in hits}
+
+    # 4. Hydrate from DB dengan filters
+    stmt = select(Listing).where(Listing.id.in_(doc_ids))
+    if harga_min is not None:
+        stmt = stmt.where(Listing.harga_per_bulan >= harga_min)
+    if harga_max is not None:
+        stmt = stmt.where(Listing.harga_per_bulan <= harga_max)
+    if tipe:
+        stmt = stmt.where(Listing.tipe == tipe)
+    if kecamatan:
+        stmt = stmt.where(Listing.kecamatan.ilike(f"%{kecamatan}%"))
+
+    db_result = await session.execute(stmt)
+    listings_map = {row.id: row for row in db_result.scalars().all()}
+
+    # 5. Preserve IR ranking, take top_k filtered
+    results: list[ListingRead] = []
+    for doc_id in doc_ids:
+        if len(results) >= top_k:
+            break
+        listing = listings_map.get(doc_id)
+        if not listing:
+            continue  # filtered out atau gak ada di DB
+        results.append(
+            ListingRead(
+                id=listing.id,
+                judul=listing.judul,
+                deskripsi=listing.deskripsi,
+                harga_per_bulan=listing.harga_per_bulan,
+                tipe=listing.tipe,
+                fasilitas=listing.fasilitas,
+                alamat=listing.alamat,
+                kecamatan=listing.kecamatan,
+                score=score_map[doc_id],
+                koordinat=(
+                    [float(listing.koordinat_lat), float(listing.koordinat_lng)]
+                    if listing.koordinat_lat is not None
+                    and listing.koordinat_lng is not None
+                    else None
+                ),
+            )
+        )
+
+    took_ms = int((time.perf_counter() - t0) * 1000)
+    logger.info(
+        f"[search] q='{q}' model={model} filter={has_filter} "
+        f"hits={len(results)}/{top_k} took_ms={took_ms}"
+    )
+    return SearchResponse(
+        query=q, model=model, top_k=top_k, took_ms=took_ms, results=results
+    )

backend/app/core/__init__.py

@@ -0,0 +1 @@
+"""Core utilities: config, DB connection, logging."""

backend/app/core/config.py

@@ -0,0 +1,65 @@
+"""Application configuration via Pydantic Settings (env-driven).
+
+Analogi Laravel: ini setara dengan `config/app.php` + `config/database.php`
+yang baca dari `.env`. Pydantic Settings otomatis cast type (mis. CORS_ORIGINS
+JSON array → Python list).
+"""
+
+from pydantic import field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    # General
+    environment: str = "development"
+    log_level: str = "INFO"
+
+    # Database
+    database_url: str = "postgresql+asyncpg://postgres:postgres@localhost:5432/tki_kos"
+
+    # CORS — list domain frontend yang boleh akses
+    cors_origins: list[str] = [
+        "http://localhost:5173",
+        "http://localhost:3000",
+    ]
+
+    # IR config
+    indexes_dir: str = "../data/indexes"
+    embedding_model: str = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
+    default_top_k: int = 10
+    bm25_k1: float = 1.5
+    bm25_b: float = 0.75
+
+    # Neural (IndoBERT) di-OFF di production (hemat RAM Render free 512MB).
+    # Hidup di notebook/Colab untuk eval. Set ENABLE_NEURAL=true untuk aktif.
+    enable_neural: bool = False
+
+    @field_validator("database_url", mode="after")
+    @classmethod
+    def normalize_database_url(cls, v: str) -> str:
+        """Normalize URL Postgres ke `postgresql+asyncpg://` (SQLAlchemy 2.0 async).
+
+        Render kasih env var DATABASE_URL dengan scheme `postgres://`, sementara
+        SQLAlchemy 2.0 minta `postgresql://` minimum, dan untuk async pakai
+        `postgresql+asyncpg://`. Validator ini handle 3 case:
+
+        - `postgres://...`            -> `postgresql+asyncpg://...`  (Render default)
+        - `postgresql://...`          -> `postgresql+asyncpg://...`  (bare)
+        - `postgresql+asyncpg://...`  -> unchanged                   (sudah benar)
+        - `postgresql+<other>://...`  -> unchanged                   (explicit driver)
+        """
+        if v.startswith("postgres://"):
+            return v.replace("postgres://", "postgresql+asyncpg://", 1)
+        # Match bare `postgresql://` (no driver suffix)
+        if v.startswith("postgresql://"):
+            return v.replace("postgresql://", "postgresql+asyncpg://", 1)
+        return v
+
+
+settings = Settings()

backend/app/core/db.py

@@ -0,0 +1,76 @@
+"""Async SQLAlchemy engine + session factory.
+
+Pattern: yield AsyncSession per-request via FastAPI dependency.
+
+Analogi Laravel: ini setara dengan Database Manager + DB facade.
+- Eloquent ORM = SQLAlchemy ORM
+- DB::transaction = async with session.begin()
+- DB::table('x') = select(X)
+"""
+
+from __future__ import annotations
+
+from typing import AsyncGenerator
+
+from sqlalchemy.ext.asyncio import (
+    AsyncSession,
+    async_sessionmaker,
+    create_async_engine,
+)
+from sqlalchemy.orm import DeclarativeBase
+
+from app.core.config import settings
+
+
+class Base(DeclarativeBase):
+    """SQLAlchemy declarative base untuk semua ORM models.
+
+    Subclass ini di setiap model file:
+        from app.core.db import Base
+        class Listing(Base): ...
+    """
+
+
+engine = create_async_engine(
+    settings.database_url,
+    echo=settings.environment == "development",
+    pool_pre_ping=True,  # validate connection before use (avoid stale conns)
+)
+
+
+async_session_factory = async_sessionmaker(
+    engine,
+    class_=AsyncSession,
+    expire_on_commit=False,  # objects accessible after commit
+)
+
+
+async def get_session() -> AsyncGenerator[AsyncSession, None]:
+    """FastAPI dependency: yield session per request, auto cleanup + rollback.
+
+    Usage di route:
+        @router.get("/x")
+        async def handler(session: AsyncSession = Depends(get_session)):
+            ...
+    """
+    async with async_session_factory() as session:
+        try:
+            yield session
+        except Exception:
+            await session.rollback()
+            raise
+
+
+async def init_db() -> None:
+    """Create tables dari metadata. Untuk DEV ONLY.
+
+    Production: pakai Alembic migrations (lihat backend/alembic/).
+    Tim TODO: setup Alembic Week 2:
+        cd backend
+        alembic init alembic
+        # edit alembic/env.py untuk import Base
+        alembic revision --autogenerate -m "initial schema"
+        alembic upgrade head
+    """
+    async with engine.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)

backend/app/evaluation/README.md

@@ -0,0 +1,172 @@
+# Evaluation Module
+
+IR evaluation metrics, inter-annotator agreement (Cohen's Kappa), dan
+statistical significance tests untuk rubric Evaluation 10%.
+
+## Metrics
+
+| Metric | Definisi | Range | Best |
+|--------|----------|-------|------|
+| P@K (K=5,10) | proportion of top-K predicted yang relevant | [0,1] | higher |
+| MAP | Mean Average Precision across queries | [0,1] | higher |
+| NDCG@10 | Normalized Discounted Cumulative Gain (graded relevance) | [0,1] | higher |
+| MRR | Mean Reciprocal Rank (1/rank of first relevant) | [0,1] | higher |
+
+## Inter-Annotator Agreement
+
+- **`cohen_kappa`**: untuk 2 annotator, nominal labels
+- **`weighted_kappa`**: untuk ordinal labels (0/1/2 relevance) — direkomendasikan
+
+**Target rubric**: Kappa >= 0.7 across all annotator pairs. Kalau di bawah,
+**dokumentasikan + lakuin consensus resolution** sebelum compute IR metrics.
+
+Interpretation (Landis & Koch 1977):
+- < 0.20: slight | 0.21-0.40: fair | 0.41-0.60: moderate
+- 0.61-0.80: substantial | 0.81-1.00: almost perfect
+
+## Statistical Significance
+
+Untuk compare 2 model (e.g., BM25 vs IndoBERT) berdasar per-query metric:
+
+- **`paired_ttest`**: kalau distribusi roughly normal (lebih powerful)
+- **`wilcoxon_signed_rank`**: non-parametric, **direkomendasikan** untuk IR
+  (metric in [0,1] sering skewed)
+
+H0: "Model A == Model B". alpha = 0.05. p < alpha => significant difference.
+
+## Workflow Penuh
+
+### 1. Annotation Phase (Week 3)
+
+Persiapan:
+1. Tim Anggota E koordinasi 3 annotator (anggota tim, bukan AI)
+2. Pre-annotation calibration: discuss 3-5 sample queries bareng, agree on
+   what "relevant" / "somewhat relevant" / "not relevant" means
+3. Distribute task: tiap annotator label SEMUA top-30 hits dari BM25 untuk
+   setiap query (300+ judgments per annotator, ~12 queries x 30 docs)
+
+Output: `eval/annotations_annotator_{A,B,C}.csv`
+```csv
+query_id,doc_id,relevance
+q01,d12345,2
+q01,d12346,1
+...
+```
+
+### 2. Compute Kappa + Consensus
+
+```python
+import csv
+from app.evaluation import cohen_kappa, weighted_kappa, interpret_kappa
+
+# Load annotations
+def load(path):
+    with open(path) as f:
+        return {(row["query_id"], row["doc_id"]): int(row["relevance"])
+                for row in csv.DictReader(f)}
+
+ann_a = load("eval/annotations_annotator_A.csv")
+ann_b = load("eval/annotations_annotator_B.csv")
+ann_c = load("eval/annotations_annotator_C.csv")
+
+# Align by shared (query, doc) keys
+shared = set(ann_a.keys()) & set(ann_b.keys()) & set(ann_c.keys())
+
+for pair_name, (pa, pb) in [("A-B", (ann_a, ann_b)), ("A-C", (ann_a, ann_c)),
+                             ("B-C", (ann_b, ann_c))]:
+    a_vals = [pa[k] for k in shared]
+    b_vals = [pb[k] for k in shared]
+    k = weighted_kappa(a_vals, b_vals)
+    print(f"{pair_name}: kappa={k:.3f} ({interpret_kappa(k)})")
+```
+
+Target: semua pair >= 0.7. Kalau ada yang <0.7:
+1. Identify queries dengan disagreement tertinggi
+2. Discussion session: 3 annotator bareng, agree pada labeling
+3. Re-annotate problematic queries
+4. Recompute kappa
+5. Generate consensus: majority vote, atau (kalau 2-2 split) tertinggi/median
+
+Output: `eval/ground_truth.csv` (consensus labels)
+```csv
+query_id,doc_id,relevance
+q01,d12345,2
+...
+```
+
+### 3. Run Evaluation (Week 4)
+
+```bash
+cd backend
+python -m app.evaluation.runner \
+    --queries ../eval/queries.json \
+    --ground-truth ../eval/ground_truth.csv \
+    --indexes-dir ../data/indexes \
+    --output ../eval/results.csv
+```
+
+Output:
+- `eval/results.csv`: per-model per-query metrics
+- Console: aggregate MAP / P@5 / P@10 / NDCG@10 / MRR per model + pairwise
+  Wilcoxon significance
+
+### 4. Generate Report (Notebook)
+
+File: `notebooks/04_evaluation.ipynb` (Anggota E buat).
+
+Wajib content untuk laporan:
+- Table aggregate metrics (4 model x 5 metrics)
+- Per-query analysis: queries mana yang hard untuk each model + kenapa
+- Statistical significance table (pairwise Wilcoxon p-values)
+- Visualizations:
+  - Bar chart MAP per model
+  - Scatter plot: query difficulty (avg performance) vs query length/complexity
+  - Heatmap: model x query (cell = MAP per query)
+- Interpretation: kenapa Model X menang/kalah di query Y
+
+## Files in Module
+
+| File | Purpose |
+|------|---------|
+| `metrics.py` | P@K, MAP, NDCG@K, MRR |
+| `kappa.py` | Cohen's & Weighted Kappa untuk inter-annotator |
+| `statistical.py` | Paired t-test, Wilcoxon signed-rank |
+| `runner.py` | CLI: run all-models x all-queries, output CSV |
+| `README.md` | this file |
+
+## What Tim Anggota E (Frontend + Eval Lead) WAJIB Do
+
+### Priority 1 — Coordinate annotation (Week 3 start)
+
+- Kick off 3-annotator process minggu pertama Week 3
+- Use BM25 top-30 sebagai pool kandidat (cepat, lexical baseline)
+- Provide annotators dengan: query text, listing snippet (judul + 200 char
+  deskripsi), full deskripsi link
+- Calibration session sebelum mulai (1 jam, walk through 5 sample queries)
+
+### Priority 2 — Build evaluation notebook (Week 4)
+
+File: `notebooks/04_evaluation.ipynb`. Template structure di README di atas.
+
+### Priority 3 — Build /eval dashboard (optional, kalau ada budget waktu)
+
+FastAPI route `/eval/summary` yang return aggregate metrics + per-query
+breakdown sebagai JSON. Frontend tab di React app untuk dosen lihat eval
+langsung di web (bonus poin System 25%).
+
+## Anti-Patterns
+
+- [BAD] Compute MAP tanpa Kappa check dulu — kalau Kappa < 0.7, ground truth
+  unreliable, MAP angka gak meaningful
+- [BAD] Annotator labels berbeda interpretation "relevant" — selalu kalibrasi
+  dulu
+- [BAD] Cuma 1 annotator — gak ada agreement check, gak reproducible
+- [BAD] Ground truth dari AI/dosen tanpa human annotator independen
+- [BAD] Report MAP without significance test — perbedaan 0.05 bisa noise
+
+## Testing
+
+```bash
+cd backend
+pytest tests/test_evaluation.py -v
+```

backend/app/evaluation/__init__.py

@@ -0,0 +1,37 @@
+"""Evaluation metrics + statistical tests untuk IR experiment.
+
+Public API:
+    from app.evaluation import (
+        precision_at_k, average_precision, mean_average_precision,
+        ndcg_at_k, reciprocal_rank, mean_reciprocal_rank,
+        cohen_kappa, weighted_kappa,
+        paired_ttest, wilcoxon_signed_rank,
+    )
+"""
+
+from .kappa import cohen_kappa, weighted_kappa
+from .metrics import (
+    average_precision,
+    mean_average_precision,
+    mean_reciprocal_rank,
+    ndcg_at_k,
+    precision_at_k,
+    reciprocal_rank,
+)
+from .statistical import paired_ttest, wilcoxon_signed_rank
+
+__all__ = [
+    # Metrics
+    "precision_at_k",
+    "average_precision",
+    "mean_average_precision",
+    "ndcg_at_k",
+    "reciprocal_rank",
+    "mean_reciprocal_rank",
+    # Inter-annotator agreement
+    "cohen_kappa",
+    "weighted_kappa",
+    # Statistical significance
+    "paired_ttest",
+    "wilcoxon_signed_rank",
+]

backend/app/evaluation/kappa.py

@@ -0,0 +1,135 @@
+"""Cohen's Kappa untuk inter-annotator agreement.
+
+Course requirement (rubric Evaluation 10%): target Kappa >= 0.7 across
+all annotator pairs. Kalau di bawah 0.7, dokumentasikan + lakuin
+consensus resolution sebelum compute metric IR.
+
+Interpretation (Landis & Koch 1977):
+- < 0.20: slight agreement
+- 0.21 - 0.40: fair
+- 0.41 - 0.60: moderate
+- 0.61 - 0.80: substantial
+- 0.81 - 1.00: almost perfect
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def cohen_kappa(
+    annotator_a: list[int],
+    annotator_b: list[int],
+    labels: list[int] | None = None,
+) -> float:
+    """Cohen's Kappa untuk 2 annotator pada same items.
+
+    Args:
+        annotator_a: labels dari annotator A (urutan harus match B)
+        annotator_b: labels dari annotator B
+        labels: list of possible label values (default: {0, 1, 2})
+
+    Returns:
+        Kappa value in [-1, 1]. 1 = perfect agreement, 0 = chance level.
+    """
+    if len(annotator_a) != len(annotator_b):
+        raise ValueError(
+            f"Length mismatch: A={len(annotator_a)}, B={len(annotator_b)}"
+        )
+    n = len(annotator_a)
+    if n == 0:
+        return 0.0
+
+    if labels is None:
+        labels = sorted(set(annotator_a) | set(annotator_b))
+
+    # Confusion matrix
+    label_to_idx = {label: i for i, label in enumerate(labels)}
+    k = len(labels)
+    matrix = np.zeros((k, k))
+    for a, b in zip(annotator_a, annotator_b):
+        matrix[label_to_idx[a]][label_to_idx[b]] += 1
+
+    # Observed agreement (Po)
+    po = np.trace(matrix) / n
+
+    # Expected agreement by chance (Pe)
+    row_marginals = matrix.sum(axis=1) / n
+    col_marginals = matrix.sum(axis=0) / n
+    pe = float(np.sum(row_marginals * col_marginals))
+
+    if pe >= 1.0:
+        return 1.0 if po >= 1.0 else 0.0
+
+    return float((po - pe) / (1 - pe))
+
+
+def weighted_kappa(
+    annotator_a: list[int],
+    annotator_b: list[int],
+    labels: list[int] | None = None,
+    weight_type: str = "linear",
+) -> float:
+    """Weighted Cohen's Kappa untuk ordinal labels (0/1/2 relevance).
+
+    Lebih appropriate dari plain kappa untuk graded relevance: disagreement
+    1 vs 2 di-penalize lebih ringan dari 0 vs 2.
+
+    Args:
+        weight_type: 'linear' atau 'quadratic' (lebih tegas)
+    """
+    if len(annotator_a) != len(annotator_b):
+        raise ValueError("Length mismatch")
+    n = len(annotator_a)
+    if n == 0:
+        return 0.0
+
+    if labels is None:
+        labels = sorted(set(annotator_a) | set(annotator_b))
+    k = len(labels)
+    label_to_idx = {label: i for i, label in enumerate(labels)}
+
+    # Confusion matrix
+    matrix = np.zeros((k, k))
+    for a, b in zip(annotator_a, annotator_b):
+        matrix[label_to_idx[a]][label_to_idx[b]] += 1
+
+    # Weight matrix
+    weights = np.zeros((k, k))
+    for i in range(k):
+        for j in range(k):
+            diff = abs(i - j)
+            if weight_type == "linear":
+                weights[i][j] = diff / (k - 1) if k > 1 else 0
+            elif weight_type == "quadratic":
+                weights[i][j] = (diff / (k - 1)) ** 2 if k > 1 else 0
+            else:
+                raise ValueError(f"Unknown weight_type: {weight_type}")
+
+    # Expected matrix (outer product of marginals)
+    row_sum = matrix.sum(axis=1)
+    col_sum = matrix.sum(axis=0)
+    expected = np.outer(row_sum, col_sum) / n
+
+    weighted_observed = np.sum(weights * matrix)
+    weighted_expected = np.sum(weights * expected)
+
+    if weighted_expected == 0:
+        return 1.0 if weighted_observed == 0 else 0.0
+    return float(1 - weighted_observed / weighted_expected)
+
+
+def interpret_kappa(kappa: float) -> str:
+    """Kategorisasi kappa per Landis & Koch (1977)."""
+    if kappa < 0.0:
+        return "less than chance"
+    elif kappa < 0.20:
+        return "slight"
+    elif kappa < 0.40:
+        return "fair"
+    elif kappa < 0.60:
+        return "moderate"
+    elif kappa < 0.80:
+        return "substantial"
+    else:
+        return "almost perfect"

backend/app/evaluation/metrics.py

@@ -0,0 +1,197 @@
+"""IR evaluation metrics: P@K, MAP, NDCG@K, MRR.
+
+Convention:
+- `predicted` = ordered list of doc_ids dari index query (rank 1 = first)
+- `relevant` = set of doc_ids yang TRULY relevant (dari ground truth)
+- `relevance_scores` = dict {doc_id: int} untuk graded relevance (0/1/2)
+
+Semua metric in [0, 1] dengan 1 = perfect.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterable
+
+
+# =============================================================================
+# Precision @ K
+# =============================================================================
+def precision_at_k(
+    predicted: list[str], relevant: Iterable[str], k: int
+) -> float:
+    """P@K = proportion of top-K predicted yang relevant.
+
+    Example:
+        predicted = ['a', 'b', 'c', 'd', 'e']
+        relevant = {'a', 'c', 'x'}
+        k = 5  -> 2/5 = 0.4
+    """
+    if k <= 0:
+        return 0.0
+    relevant_set = set(relevant)
+    top_k = predicted[:k]
+    if not top_k:
+        return 0.0
+    hits = sum(1 for doc_id in top_k if doc_id in relevant_set)
+    return hits / k
+
+
+# =============================================================================
+# Average Precision (AP) per query
+# =============================================================================
+def average_precision(
+    predicted: list[str], relevant: Iterable[str]
+) -> float:
+    """Average Precision per query.
+
+    AP = mean of P@k untuk setiap relevant doc yang muncul di predicted.
+
+    Example:
+        predicted = ['a', 'b', 'c', 'd']
+        relevant = {'a', 'c'}
+        -> P@1 = 1/1 (a hit), P@3 = 2/3 (c hit)
+        -> AP = (1.0 + 0.667) / 2 = 0.833
+    """
+    relevant_set = set(relevant)
+    if not relevant_set:
+        return 0.0
+
+    precisions: list[float] = []
+    hits = 0
+    for i, doc_id in enumerate(predicted, start=1):
+        if doc_id in relevant_set:
+            hits += 1
+            precisions.append(hits / i)
+
+    if not precisions:
+        return 0.0
+    # Divide by number of relevant docs (kalau gak semua relevant ada di predicted,
+    # bagi tetap pake count of relevant — penalti untuk missed relevant)
+    return sum(precisions) / len(relevant_set)
+
+
+def mean_average_precision(
+    predicted_per_query: dict[str, list[str]],
+    relevant_per_query: dict[str, Iterable[str]],
+) -> float:
+    """MAP = mean of AP across queries."""
+    if not predicted_per_query:
+        return 0.0
+    aps: list[float] = []
+    for query_id, predicted in predicted_per_query.items():
+        relevant = relevant_per_query.get(query_id, [])
+        aps.append(average_precision(predicted, relevant))
+    return sum(aps) / len(aps)
+
+
+# =============================================================================
+# NDCG @ K (Normalized Discounted Cumulative Gain)
+# =============================================================================
+def _dcg_at_k(gains: list[float], k: int) -> float:
+    """DCG@K = sum gain_i / log2(i+1), 1-indexed rank."""
+    return sum(g / math.log2(i + 1) for i, g in enumerate(gains[:k], start=1))
+
+
+def ndcg_at_k(
+    predicted: list[str],
+    relevance_scores: dict[str, int],
+    k: int = 10,
+) -> float:
+    """NDCG@K dengan graded relevance.
+
+    Args:
+        predicted: ordered doc_ids dari IR model
+        relevance_scores: {doc_id: int} -- typical 0/1/2 atau 0-3 scale
+        k: cutoff rank
+
+    Score in [0, 1]. Penalize relevant docs di posisi rendah.
+    """
+    if k <= 0 or not predicted:
+        return 0.0
+
+    # Actual DCG: pakai relevance_scores
+    gains = [relevance_scores.get(doc_id, 0) for doc_id in predicted[:k]]
+    dcg = _dcg_at_k(gains, k)
+
+    # Ideal DCG: sort all relevant docs by score desc
+    ideal_gains = sorted(relevance_scores.values(), reverse=True)
+    idcg = _dcg_at_k(ideal_gains, k)
+
+    if idcg == 0:
+        return 0.0
+    return dcg / idcg
+
+
+# =============================================================================
+# Mean Reciprocal Rank (MRR)
+# =============================================================================
+def reciprocal_rank(
+    predicted: list[str], relevant: Iterable[str]
+) -> float:
+    """1 / rank dari relevant pertama di predicted. 0 kalau gak ada relevant."""
+    relevant_set = set(relevant)
+    for i, doc_id in enumerate(predicted, start=1):
+        if doc_id in relevant_set:
+            return 1.0 / i
+    return 0.0
+
+
+def mean_reciprocal_rank(
+    predicted_per_query: dict[str, list[str]],
+    relevant_per_query: dict[str, Iterable[str]],
+) -> float:
+    """MRR = mean of reciprocal_rank across queries."""
+    if not predicted_per_query:
+        return 0.0
+    rrs: list[float] = []
+    for query_id, predicted in predicted_per_query.items():
+        relevant = relevant_per_query.get(query_id, [])
+        rrs.append(reciprocal_rank(predicted, relevant))
+    return sum(rrs) / len(rrs)
+
+
+# =============================================================================
+# Constraint Satisfaction @ K (lensa kedua: ukur kebutuhan user, bukan teks)
+# =============================================================================
+def constraint_satisfaction_at_k(
+    results: list[dict],
+    constraints: dict,
+    k: int = 5,
+    max_km: float = 3.0,
+) -> float:
+    """Rasio top-K hasil yang memenuhi SEMUA konstrain query yang ada.
+
+    Berbeda dari P@K/MAP (yang mengukur relevansi teks dari qrels), metric ini
+    mengukur hal yang dioptimalkan smart pipeline: gender benar + harga sesuai +
+    fasilitas ada + dalam radius dari anchor. Tidak butuh qrels.
+
+    Args:
+        results: list dict listing TERURUT, key: tipe, harga_per_bulan,
+                 fasilitas (list[str]), lat, lng.
+        constraints: {gender, harga_max, fasilitas: list[str], anchor: (lat, lng)|None}.
+        k: cutoff. max_km: ambang jarak "dekat" dari anchor.
+    """
+    from app.search.gazetteer import haversine_km
+
+    topk = results[:k]
+    if not topk:
+        return 0.0
+
+    ok = 0
+    for r in topk:
+        good = True
+        if constraints.get("gender") and r.get("tipe") != constraints["gender"]:
+            good = False
+        if constraints.get("harga_max") and (r.get("harga_per_bulan") or 0) > constraints["harga_max"]:
+            good = False
+        have = [str(f).lower() for f in (r.get("fasilitas") or [])]
+        for kw in constraints.get("fasilitas", []):
+            if not any(kw in f for f in have):
+                good = False
+        anchor = constraints.get("anchor")
+        if anchor and r.get("lat") is not None and r.get("lng") is not None:
+            if haversine_km(float(r["lat"]), float(r["lng"]), anchor[0], anchor[1]) > max_km:
+                good = False
+        ok += int(good)
+    return ok / len(topk)

backend/app/evaluation/runner.py

@@ -0,0 +1,195 @@
+"""CLI runner untuk evaluation: run semua model x query, compute metrics.
+
+Workflow:
+    # 1. Pastikan: corpus + indexes + ground truth siap
+    #    - data/processed/corpus.json
+    #    - data/indexes/{tfidf.pkl, bm25.pkl, indobert/}
+    #    - eval/queries.json
+    #    - eval/ground_truth.csv
+
+    # 2. Run evaluation
+    python -m app.evaluation.runner \\
+        --queries ../eval/queries.json \\
+        --ground-truth ../eval/ground_truth.csv \\
+        --indexes-dir ../data/indexes \\
+        --output ../eval/results.csv
+
+    # 3. Output:
+    #    eval/results.csv -- per-model per-query metrics
+    #    Console summary: aggregate MAP, P@5/10, NDCG@10, MRR per model
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+from .kappa import interpret_kappa
+from .metrics import (
+    mean_average_precision,
+    mean_reciprocal_rank,
+    ndcg_at_k,
+    precision_at_k,
+)
+from .statistical import paired_ttest, wilcoxon_signed_rank
+
+
+def load_queries(path: Path) -> list[dict[str, Any]]:
+    """Load eval/queries.json -> list of {id, query, context, ...}."""
+    with open(path, encoding="utf-8") as f:
+        data = json.load(f)
+    return data.get("queries", data)  # support raw list atau {queries: [...]}
+
+
+def load_ground_truth(path: Path) -> dict[str, dict[str, int]]:
+    """Load ground_truth.csv -> {query_id: {doc_id: consensus_relevance}}.
+
+    Expected columns: query_id, doc_id, relevance (consensus)
+    """
+    gt: dict[str, dict[str, int]] = {}
+    with open(path, encoding="utf-8") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            qid = row["query_id"]
+            did = row["doc_id"]
+            rel = int(row["relevance"])
+            gt.setdefault(qid, {})[did] = rel
+    return gt
+
+
+def relevant_set(rel_dict: dict[str, int], threshold: int = 1) -> set[str]:
+    """Convert graded relevance ke binary relevant set (rel >= threshold)."""
+    return {doc_id for doc_id, rel in rel_dict.items() if rel >= threshold}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Run IR evaluation")
+    parser.add_argument("--queries", type=Path, required=True)
+    parser.add_argument("--ground-truth", type=Path, required=True)
+    parser.add_argument("--indexes-dir", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--top-k", type=int, default=10)
+    parser.add_argument(
+        "--models",
+        nargs="+",
+        default=["tfidf", "bm25", "indobert", "hybrid"],
+    )
+    args = parser.parse_args()
+
+    # Load eval data
+    queries = load_queries(args.queries)
+    ground_truth = load_ground_truth(args.ground_truth)
+    logger.info(
+        f"[eval] {len(queries)} queries, "
+        f"{sum(len(v) for v in ground_truth.values())} GT annotations"
+    )
+
+    # Load preprocessing pipeline (untuk preprocess queries)
+    from app.preprocessing import PreprocessingPipeline
+
+    pipeline = PreprocessingPipeline()
+
+    # Load indexes
+    from app.indexing.hybrid import HybridIndex
+    from app.indexing.loader import load_all_indexes
+
+    indexes = load_all_indexes(args.indexes_dir)
+    if "bm25" in indexes and "indobert" in indexes and "hybrid" in args.models:
+        indexes["hybrid"] = HybridIndex(
+            indexes["bm25"],
+            indexes["indobert"],
+            query_preprocessor=lambda q: pipeline.process(q).processed,
+        )
+
+    # Run all model x all queries
+    # Structure: results[model][query_id] = {p5, p10, ap, ndcg10, rr}
+    results: dict[str, dict[str, dict[str, float]]] = {}
+
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with open(args.output, "w", encoding="utf-8", newline="") as out_f:
+        writer = csv.writer(out_f)
+        writer.writerow(
+            ["model", "query_id", "query", "p_at_5", "p_at_10", "ap",
+             "ndcg_at_10", "rr"]
+        )
+
+        for model_name in args.models:
+            if model_name not in indexes:
+                logger.warning(f"[skip] index {model_name} gak ditemukan")
+                continue
+            index = indexes[model_name]
+            results[model_name] = {}
+
+            for q in queries:
+                qid = q["id"]
+                q_text = q["query"]
+                processed = pipeline.process(q_text).processed
+                # BM25/TF-IDF: lexical, butuh ter-stem. IndoBERT/Hybrid:
+                # semantic, perlu natural language (raw query).
+                if model_name in ("indobert", "hybrid"):
+                    search_q = q_text
+                else:
+                    search_q = processed
+                hits = index.query(search_q, top_k=args.top_k)
+                predicted = [h.doc_id for h in hits]
+
+                rel_dict = ground_truth.get(qid, {})
+                rel_set = relevant_set(rel_dict, threshold=1)
+
+                p5 = precision_at_k(predicted, rel_set, 5)
+                p10 = precision_at_k(predicted, rel_set, 10)
+                from .metrics import average_precision, reciprocal_rank
+
+                ap = average_precision(predicted, rel_set)
+                nd10 = ndcg_at_k(predicted, rel_dict, 10)
+                rr = reciprocal_rank(predicted, rel_set)
+
+                results[model_name][qid] = {
+                    "p5": p5, "p10": p10, "ap": ap, "ndcg10": nd10, "rr": rr,
+                }
+                writer.writerow([model_name, qid, q_text, p5, p10, ap, nd10, rr])
+
+            logger.info(f"[done] model={model_name}")
+
+    # Console summary
+    logger.info("\n=== AGGREGATE METRICS ===")
+    for model_name, per_query in results.items():
+        n = len(per_query)
+        if n == 0:
+            continue
+        avg_p5 = sum(v["p5"] for v in per_query.values()) / n
+        avg_p10 = sum(v["p10"] for v in per_query.values()) / n
+        avg_ap = sum(v["ap"] for v in per_query.values()) / n
+        avg_ndcg = sum(v["ndcg10"] for v in per_query.values()) / n
+        avg_rr = sum(v["rr"] for v in per_query.values()) / n
+        logger.info(
+            f"{model_name:10s} P@5={avg_p5:.4f} P@10={avg_p10:.4f} "
+            f"MAP={avg_ap:.4f} NDCG@10={avg_ndcg:.4f} MRR={avg_rr:.4f}"
+        )
+
+    # Pairwise statistical significance (Wilcoxon)
+    model_names = [m for m in args.models if m in results]
+    if len(model_names) >= 2:
+        logger.info("\n=== PAIRWISE WILCOXON (MAP) ===")
+        for i, ma in enumerate(model_names):
+            for mb in model_names[i + 1 :]:
+                a_aps = [results[ma][qid]["ap"] for qid in results[ma]]
+                b_aps = [results[mb][qid]["ap"] for qid in results[mb]]
+                try:
+                    test = wilcoxon_signed_rank(a_aps, b_aps)
+                    logger.info(f"{ma} vs {mb}: {test}")
+                except Exception as e:
+                    logger.warning(f"{ma} vs {mb}: {e}")
+
+    logger.info(f"\n[done] per-query metrics saved to {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

backend/app/evaluation/statistical.py

@@ -0,0 +1,171 @@
+"""Statistical significance tests untuk compare IR models.
+
+Untuk per-query metric (e.g., AP per query untuk Model A vs Model B):
+- **paired_ttest**: kalau distribusi roughly normal
+- **wilcoxon_signed_rank**: non-parametric, lebih robust untuk metric bounded [0,1]
+
+Convention: H0 = "Model A = Model B", alpha = 0.05.
+Kalau p-value < 0.05, reject H0 => significant difference.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+from scipy import stats
+
+
+@dataclass
+class StatTestResult:
+    """Hasil statistical test."""
+    test_name: str
+    statistic: float
+    p_value: float
+    n: int  # jumlah pair
+    significant: bool  # p < alpha
+    alpha: float = 0.05
+    effect_size: float | None = None  # cohen's d untuk t-test
+
+    def __str__(self) -> str:
+        sig = "SIGNIFICANT" if self.significant else "not significant"
+        return (
+            f"{self.test_name}: stat={self.statistic:.4f}, "
+            f"p={self.p_value:.4f}, n={self.n}, {sig} (alpha={self.alpha})"
+        )
+
+
+def paired_ttest(
+    model_a_scores: list[float],
+    model_b_scores: list[float],
+    alpha: float = 0.05,
+    alternative: Literal["two-sided", "less", "greater"] = "two-sided",
+) -> StatTestResult:
+    """Paired t-test untuk per-query metric.
+
+    Args:
+        model_a_scores: metric per query untuk Model A (e.g., AP@10)
+        model_b_scores: same query order, Model B
+        alpha: significance threshold (default 0.05)
+        alternative: 'two-sided' / 'less' / 'greater' (one-sided)
+
+    Returns StatTestResult dengan Cohen's d effect size.
+    """
+    if len(model_a_scores) != len(model_b_scores):
+        raise ValueError(
+            f"Length mismatch: A={len(model_a_scores)}, B={len(model_b_scores)}"
+        )
+    if len(model_a_scores) < 2:
+        raise ValueError("Butuh minimal 2 query untuk t-test")
+
+    result = stats.ttest_rel(
+        model_a_scores, model_b_scores, alternative=alternative
+    )
+
+    # Cohen's d effect size (paired)
+    import numpy as np
+
+    diffs = np.array(model_a_scores) - np.array(model_b_scores)
+    effect_size = float(np.mean(diffs) / np.std(diffs, ddof=1)) if np.std(diffs, ddof=1) > 0 else 0.0
+
+    return StatTestResult(
+        test_name="paired_ttest",
+        statistic=float(result.statistic),
+        p_value=float(result.pvalue),
+        n=len(model_a_scores),
+        significant=result.pvalue < alpha,
+        alpha=alpha,
+        effect_size=effect_size,
+    )
+
+
+@dataclass
+class HolmEntry:
+    """Hasil koreksi Holm-Bonferroni untuk satu uji dalam keluarga uji."""
+    label: str
+    p_value: float
+    p_adjusted: float       # Holm step-down adjusted p-value (monotone)
+    threshold: float        # ambang alpha/(m - rank) untuk uji ini
+    significant: bool       # p_adjusted < alpha
+
+
+def holm_bonferroni(
+    tests: list[tuple[str, float]],
+    alpha: float = 0.05,
+) -> list[HolmEntry]:
+    """Koreksi multiple comparison Holm-Bonferroni (step-down).
+
+    Menjalankan m uji sekaligus pada alpha=0.05 menggelembungkan peluang
+    false positive (m=6 -> ~26%). Holm: urutkan p ascending, bandingkan
+    p_i dengan alpha/(m-i); berhenti di kegagalan pertama, sisanya tidak
+    signifikan. Lebih powerful dari Bonferroni murni, tetap kontrol FWER.
+
+    Args:
+        tests: list (label, p_value), urutan bebas.
+    Returns:
+        list HolmEntry dengan urutan input dipertahankan.
+    """
+    m = len(tests)
+    if m == 0:
+        return []
+    order = sorted(range(m), key=lambda i: tests[i][1])
+
+    # Adjusted p: max kumulatif dari (m - rank) * p, di-cap 1.0 (monotone)
+    adjusted = [0.0] * m
+    running_max = 0.0
+    rejecting = True
+    thresholds = [0.0] * m
+    significant = [False] * m
+    for rank, idx in enumerate(order):
+        p = tests[idx][1]
+        adj = min(1.0, (m - rank) * p)
+        running_max = max(running_max, adj)
+        adjusted[idx] = running_max
+        thresholds[idx] = alpha / (m - rank)
+        # step-down: begitu satu gagal, semua setelahnya gagal
+        if rejecting and p <= thresholds[idx]:
+            significant[idx] = True
+        else:
+            rejecting = False
+
+    return [
+        HolmEntry(
+            label=tests[i][0],
+            p_value=tests[i][1],
+            p_adjusted=adjusted[i],
+            threshold=thresholds[i],
+            significant=significant[i],
+        )
+        for i in range(m)
+    ]
+
+
+def wilcoxon_signed_rank(
+    model_a_scores: list[float],
+    model_b_scores: list[float],
+    alpha: float = 0.05,
+    alternative: Literal["two-sided", "less", "greater"] = "two-sided",
+) -> StatTestResult:
+    """Wilcoxon signed-rank test — non-parametric paired comparison.
+
+    Lebih robust untuk metric bounded [0, 1] yang sering tidak normal.
+    Direkomendasikan untuk IR metric comparison.
+    """
+    if len(model_a_scores) != len(model_b_scores):
+        raise ValueError("Length mismatch")
+    if len(model_a_scores) < 5:
+        raise ValueError("Butuh minimal 5 query untuk Wilcoxon (statistical power)")
+
+    result = stats.wilcoxon(
+        model_a_scores, model_b_scores, alternative=alternative
+    )
+
+    return StatTestResult(
+        test_name="wilcoxon_signed_rank",
+        statistic=float(result.statistic),
+        p_value=float(result.pvalue),
+        n=len(model_a_scores),
+        significant=result.pvalue < alpha,
+        alpha=alpha,
+        effect_size=None,  # Wilcoxon gak punya standard effect size
+    )

backend/app/indexing/README.md

@@ -0,0 +1,189 @@
+# Indexing Module
+
+Tiga paradigma IR + optional hybrid:
+
+| Index | Library | Type | Strength | Weakness |
+|-------|---------|------|----------|----------|
+| `TFIDFIndex` | scikit-learn `TfidfVectorizer` + cosine | Lexical | Fast, interpretable | Synonym/paraphrase miss |
+| `BM25Index` | `rank_bm25` `BM25Okapi` | Lexical (probabilistic) | Term saturation | Still lexical |
+| `IndoBERTIndex` | `sentence-transformers` + FAISS | Neural / semantic | Semantic, paraphrase-aware | Heavy, slower |
+| `HybridIndex` | BM25 candidates -> IndoBERT rerank | Composite | Best of both | Latency tertinggi |
+
+## Workflow
+
+```
+data/raw/mamikos.jsonl              (output Anggota A scraper)
+  -> preprocessing pipeline          (Anggota B)
+  -> data/processed/corpus.json      (input ke build.py)
+  -> python -m app.indexing.build    (Anggota C + D)
+  -> data/indexes/
+      tfidf.pkl
+      bm25.pkl
+      indobert/
+        embeddings.npy
+        faiss.index
+        meta.json
+```
+
+## Quick Start
+
+```bash
+cd backend
+.venv\Scripts\activate  # Windows
+
+# 1. Pastikan corpus.json sudah ada
+ls ../data/processed/corpus.json
+
+# 2. Build semua indexes
+python -m app.indexing.build \
+    --corpus ../data/processed/corpus.json \
+    --output-dir ../data/indexes
+
+# 3. Smoke test load (di Python REPL atau notebook)
+python -c "
+from pathlib import Path
+from app.indexing.loader import load_all_indexes
+indexes = load_all_indexes(Path('../data/indexes'))
+hits = indexes['bm25'].query('kos putra dekat unila', top_k=5)
+for h in hits:
+    print(h)
+"
+```
+
+## Corpus Format
+
+`data/processed/corpus.json` = list of objects:
+
+```json
+[
+  {
+    "id": "kos-abc-123",
+    "text": "kos putra eksklusif gedong meneng air conditioner kamar mandi dalam ...",
+    "raw_text": "Kos Putra Exclusive Gedong Meneng dengan AC dan KMD ...",
+    "metadata": {
+      "judul": "Kos Putra Exclusive Gedong Meneng",
+      "harga_per_bulan": 850000,
+      "tipe": "putra",
+      "fasilitas": ["ac", "wifi", "kamar mandi dalam"],
+      "alamat": "Jl. Sumantri Brojonegoro No. 1",
+      "kecamatan": "Rajabasa"
+    }
+  },
+  ...
+]
+```
+
+- `text` = output preprocessing pipeline (lowercased, stemmed, stopword-free)
+- `raw_text` = original deskripsi untuk display di UI
+- `metadata` = field tambahan dari schema scraper
+
+## What's Done (oleh mentor — scaffold)
+
+- [x] `base.py` — `Document`, `SearchHit`, `IndexBase` abstract
+- [x] `tfidf.py` — `TFIDFIndex` dengan default ngram=(1,2), min_df=2,
+  max_features=10000, sublinear_tf
+- [x] `bm25.py` — `BM25Index` dengan k1=1.5, b=0.75 (rank_bm25)
+- [x] `indobert.py` — `IndoBERTIndex` dengan MiniLM multilingual default,
+  FAISS IndexFlatIP, lazy model load, `score_docs()` untuk hybrid rerun
+- [x] `hybrid.py` — `HybridIndex` (BM25 top-50 -> IndoBERT rerank, min-max
+  normalize + weighted combination, configurable alpha)
+- [x] `build.py` — CLI builder untuk semua indexes
+- [x] `loader.py` — FastAPI lifespan helper
+- [x] `tests/test_indexing.py` — unit test dengan small fixture corpus
+
+## What Tim Anggota C (IR Baseline) WAJIB Do
+
+### Priority 1 — Tune TF-IDF + BM25 hyperparameters
+
+File: `notebooks/03_model_comparison.ipynb`.
+
+Experiment:
+- TF-IDF: `ngram_range` in [(1,1), (1,2), (1,3)], `min_df` in [1, 2, 5],
+  `max_features` in [5000, 10000, 20000, None]
+- BM25: `k1` in [1.0, 1.2, 1.5, 2.0], `b` in [0.5, 0.75, 1.0]
+
+Plot heatmap MAP per kombinasi. Pilih best, dokumentasikan kenapa di laporan.
+
+### Priority 2 — Edge case query testing
+
+Setelah index ready, test queries:
+- 1-word query: "kos" — apakah TF-IDF/BM25 robust?
+- Stopword-heavy query: "yang ada di dekat kampus" — apakah preprocessing
+  handle dengan baik?
+- Out-of-vocabulary: query dengan term yang gak muncul di corpus
+
+## What Tim Anggota D (IR Neural) WAJIB Do
+
+### Priority 1 — Compare embedding models
+
+Default: `paraphrase-multilingual-MiniLM-L12-v2` (~118MB).
+Alternative: `indobenchmark/indobert-base-p2` (~440MB, lebih bagus tapi heavy).
+
+Test both di sample queries, compare MAP. Trade-off: model size vs accuracy
+vs Render free tier RAM (512MB).
+
+### Priority 2 — Tune Hybrid alpha
+
+File: `hybrid.py` di constructor `alpha` parameter.
+
+Experiment alpha in [0.0, 0.3, 0.5, 0.7] di ground truth queries. Pure rerank
+(alpha=0) sering kompetitif untuk Indonesian.
+
+### Priority 3 — Pooling strategy comparison
+
+`sentence-transformers` default mean-pool. Bisa explore:
+- CLS token pooling
+- Max pooling
+- Concat (CLS + mean)
+
+Pakai `model.encode(texts, output_value="token_embeddings")` untuk custom pool.
+
+## Anti-Patterns
+
+- [BAD] `FAISS IndexIVFFlat` untuk corpus <=5000 docs — overhead lebih besar
+  dari benefit. Pakai `IndexFlatIP` (exhaustive) sampai 10K+ docs.
+- [BAD] Re-encode docs di tiap hybrid query — slow. Pakai `score_docs()`
+  yang reuse embeddings.
+- [BAD] Pickle FAISS index langsung — fragile across versions. Pakai
+  `faiss.write_index()` + `faiss.read_index()`.
+- [BAD] Cache embeddings di memory tanpa save — restart loss. Selalu save
+  ke disk.
+
+## Performance Expectations
+
+Corpus 3000 listings @ 384-dim embeddings:
+
+| Operation | Time |
+|-----------|------|
+| TF-IDF build | 1-2s |
+| BM25 build | <1s |
+| IndoBERT encode all 3K docs (MiniLM, CPU) | 30-60s |
+| IndoBERT encode all 3K docs (IndoBERT base, CPU) | 2-3 min |
+| TF-IDF query | <5ms |
+| BM25 query | <10ms |
+| IndoBERT query (encode + FAISS Flat) | 50-100ms (mostly encode) |
+| Hybrid query | 80-150ms |
+
+Acceptable untuk demo. Optimization optional kalau ada budget waktu di Week 4.
+
+## Testing
+
+```bash
+cd backend
+pytest tests/test_indexing.py -v
+```
+
+Test cover:
+- Each index: build, query, save, load roundtrip
+- Edge cases: empty corpus, top_k > corpus_size
+- Hybrid: combination math, alpha=0/0.5/1
+- Performance smoke: 50-doc fixture builds < 5s
+
+## Save Format Reference
+
+| Index | File | Format |
+|-------|------|--------|
+| TF-IDF | `tfidf.pkl` | pickle: `{vectorizer, doc_matrix (sparse), doc_ids, size}` |
+| BM25 | `bm25.pkl` | pickle: `{k1, b, doc_ids, tokenized_corpus, size}` (BM25Okapi rebuilt on load) |
+| IndoBERT | `indobert/` folder | `embeddings.npy` + `faiss.index` + `meta.json` |
+| Hybrid | _none_ | Composition only — load sub-indexes, instantiate HybridIndex |

backend/app/indexing/__init__.py

@@ -0,0 +1,30 @@
+"""IR index builders: TF-IDF, BM25, IndoBERT+FAISS, Hybrid.
+
+Public API:
+    from app.indexing import (
+        Document, SearchHit, IndexBase,
+        TFIDFIndex, BM25Index, IndoBERTIndex, HybridIndex,
+    )
+
+Workflow:
+    1. Load corpus dari data/processed/corpus.json (after preprocessing)
+    2. Build setiap index, save ke data/indexes/
+    3. FastAPI lifespan load semua indexes saat startup
+    4. Search route /search?model=X panggil index.query(q)
+"""
+
+from .base import Document, IndexBase, SearchHit
+from .bm25 import BM25Index
+from .hybrid import HybridIndex
+from .indobert import IndoBERTIndex
+from .tfidf import TFIDFIndex
+
+__all__ = [
+    "Document",
+    "SearchHit",
+    "IndexBase",
+    "TFIDFIndex",
+    "BM25Index",
+    "IndoBERTIndex",
+    "HybridIndex",
+]

backend/app/indexing/base.py

@@ -0,0 +1,64 @@
+"""Abstract IndexBase + shared dataclasses.
+
+Setiap index subclass implement: build, query, save, load.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Optional
+
+
+@dataclass
+class Document:
+    """Document representation untuk indexing.
+
+    `text` = processed text (output preprocessing pipeline), siap di-index.
+    `raw_text` = original deskripsi (untuk display di UI).
+    `metadata` = field tambahan (judul, harga, alamat, kecamatan, dll).
+    """
+    id: str
+    text: str
+    raw_text: Optional[str] = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class SearchHit:
+    """Single result dari index query."""
+    doc_id: str
+    score: float
+    rank: int
+
+
+class IndexBase(ABC):
+    """Abstract base class untuk semua IR index.
+
+    Convention: setiap subclass punya class attribute `name` (string identifier
+    untuk routing di /search?model=<name>).
+    """
+
+    name: str = "base"  # override per subclass
+
+    @abstractmethod
+    def build(self, corpus: list[Document]) -> None:
+        """Build index dari corpus. Overwrite existing state."""
+
+    @abstractmethod
+    def query(self, q: str, top_k: int = 10) -> list[SearchHit]:
+        """Run query, return top-K hits sorted by score DESC."""
+
+    @abstractmethod
+    def save(self, path: Path) -> None:
+        """Serialize index ke disk."""
+
+    @classmethod
+    @abstractmethod
+    def load(cls, path: Path) -> "IndexBase":
+        """Deserialize index dari disk."""
+
+    def size(self) -> int:
+        """Jumlah dokumen yang sudah di-index."""
+        return getattr(self, "_size", 0)

backend/app/indexing/bm25.py

@@ -0,0 +1,84 @@
+"""BM25 (Best Matching 25) index via rank_bm25.
+
+Hyperparameter default sesuai standar literature:
+- k1=1.5: term frequency saturation parameter
+- b=0.75: length normalization parameter
+
+Tim Anggota C: experiment k1 in [1.2, 2.0] dan b in [0.5, 1.0] untuk lihat
+sensitivity. Biasanya BM25 beat TF-IDF tipis (~5-10% MAP) di Indonesian corpus.
+"""
+
+from __future__ import annotations
+
+import pickle
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+from rank_bm25 import BM25Okapi
+
+from .base import Document, IndexBase, SearchHit
+
+
+class BM25Index(IndexBase):
+    name = "bm25"
+
+    def __init__(self, k1: float = 1.5, b: float = 0.75):
+        self.k1 = k1
+        self.b = b
+        self.bm25: Optional[BM25Okapi] = None
+        self.doc_ids: list[str] = []
+        self.tokenized_corpus: list[list[str]] = []
+        self._size = 0
+
+    def build(self, corpus: list[Document]) -> None:
+        self.doc_ids = [doc.id for doc in corpus]
+        # rank_bm25 minta list of token lists
+        self.tokenized_corpus = [doc.text.split() for doc in corpus]
+        self.bm25 = BM25Okapi(self.tokenized_corpus, k1=self.k1, b=self.b)
+        self._size = len(corpus)
+
+    def query(self, q: str, top_k: int = 10) -> list[SearchHit]:
+        if self.bm25 is None or self._size == 0:
+            return []
+        q_tokens = q.split()
+        scores = self.bm25.get_scores(q_tokens)
+        top_k = min(top_k, self._size)
+        top_indices = np.argpartition(-scores, top_k - 1)[:top_k]
+        top_indices = top_indices[np.argsort(-scores[top_indices])]
+        return [
+            SearchHit(
+                doc_id=self.doc_ids[idx],
+                score=float(scores[idx]),
+                rank=rank + 1,
+            )
+            for rank, idx in enumerate(top_indices)
+        ]
+
+    def save(self, path: Path) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(path, "wb") as f:
+            pickle.dump(
+                {
+                    "k1": self.k1,
+                    "b": self.b,
+                    "doc_ids": self.doc_ids,
+                    "tokenized_corpus": self.tokenized_corpus,
+                    "size": self._size,
+                },
+                f,
+            )
+
+    @classmethod
+    def load(cls, path: Path) -> "BM25Index":
+        with open(path, "rb") as f:
+            data = pickle.load(f)
+        instance = cls(k1=data["k1"], b=data["b"])
+        instance.doc_ids = data["doc_ids"]
+        instance.tokenized_corpus = data["tokenized_corpus"]
+        instance._size = data["size"]
+        # Rebuild BM25Okapi (gak bisa di-pickle reliably across versions)
+        instance.bm25 = BM25Okapi(
+            instance.tokenized_corpus, k1=instance.k1, b=instance.b
+        )
+        return instance

backend/app/indexing/build.py

@@ -0,0 +1,119 @@
+"""CLI builder untuk semua IR indexes.
+
+Workflow:
+    # 1. Setelah scraping selesai dan preprocessing dijalankan:
+    #    data/processed/corpus.json berisi list of {id, text, raw_text, metadata}
+
+    # 2. Build semua index sekaligus
+    python -m app.indexing.build \\
+        --corpus ../data/processed/corpus.json \\
+        --output-dir ../data/indexes
+
+    # 3. Output struktur:
+    #    data/indexes/tfidf.pkl
+    #    data/indexes/bm25.pkl
+    #    data/indexes/indobert/embeddings.npy
+    #    data/indexes/indobert/faiss.index
+    #    data/indexes/indobert/meta.json
+
+Tiap index loadable independen di FastAPI lifespan startup (lihat loader.py).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+from loguru import logger
+
+from .base import Document
+from .bm25 import BM25Index
+from .indobert import IndoBERTIndex
+from .tfidf import TFIDFIndex
+
+
+def load_corpus(path: Path) -> list[Document]:
+    """Load corpus JSON: expect list of {id, text, raw_text?, metadata?}."""
+    with open(path, encoding="utf-8") as f:
+        raw = json.load(f)
+    return [
+        Document(
+            id=item["id"],
+            text=item["text"],
+            raw_text=item.get("raw_text"),
+            metadata=item.get("metadata", {}),
+        )
+        for item in raw
+    ]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Build IR indexes dari corpus")
+    parser.add_argument(
+        "--corpus", type=Path, required=True,
+        help="Path ke corpus.json (output preprocessing)",
+    )
+    parser.add_argument(
+        "--output-dir", type=Path, required=True,
+        help="Folder output indexes (e.g., ../data/indexes)",
+    )
+    parser.add_argument(
+        "--skip", nargs="*", default=[],
+        choices=["tfidf", "bm25", "indobert"],
+        help="Index yang di-skip (untuk debug)",
+    )
+    parser.add_argument(
+        "--indobert-model",
+        default="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
+        help="Sentence-transformers model name (default: MiniLM lightweight)",
+    )
+    parser.add_argument(
+        "--tfidf-ngram", nargs=2, type=int, default=[1, 2],
+        help="TF-IDF ngram range (default: 1 2)",
+    )
+    parser.add_argument("--bm25-k1", type=float, default=1.5)
+    parser.add_argument("--bm25-b", type=float, default=0.75)
+    args = parser.parse_args()
+
+    logger.info(f"[load corpus] {args.corpus}")
+    corpus = load_corpus(args.corpus)
+    logger.info(f"[corpus] {len(corpus)} documents")
+
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+
+    # ---- TF-IDF ----
+    if "tfidf" not in args.skip:
+        logger.info("[build TF-IDF]")
+        t0 = time.time()
+        tfidf = TFIDFIndex(ngram_range=tuple(args.tfidf_ngram))
+        tfidf.build(corpus)
+        tfidf.save(args.output_dir / "tfidf.pkl")
+        logger.info(f"[TF-IDF] saved ({time.time() - t0:.1f}s)")
+
+    # ---- BM25 ----
+    if "bm25" not in args.skip:
+        logger.info("[build BM25]")
+        t0 = time.time()
+        bm25 = BM25Index(k1=args.bm25_k1, b=args.bm25_b)
+        bm25.build(corpus)
+        bm25.save(args.output_dir / "bm25.pkl")
+        logger.info(f"[BM25] saved ({time.time() - t0:.1f}s)")
+
+    # ---- IndoBERT + FAISS ----
+    if "indobert" not in args.skip:
+        logger.info(f"[build IndoBERT] model={args.indobert_model}")
+        t0 = time.time()
+        indobert = IndoBERTIndex(model_name=args.indobert_model)
+        indobert.build(corpus)
+        indobert.save(args.output_dir / "indobert")
+        logger.info(f"[IndoBERT] saved ({time.time() - t0:.1f}s)")
+
+    logger.info(f"[done] all indexes saved ke {args.output_dir}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

backend/app/indexing/hybrid.py

@@ -0,0 +1,127 @@
+"""Hybrid retrieval: BM25 candidates -> IndoBERT rerank.
+
+Strategy:
+1. BM25 fetch top-N candidates (default N=50). BM25 cepat dan lexical-aware,
+   bagus buat narrow down dari 1500-3000 docs ke 50.
+2. IndoBERT score 50 candidates dengan pre-computed embeddings (no re-encode).
+3. Combine score: `final = alpha * bm25_norm + (1 - alpha) * indobert_norm`,
+   atau pure rerank by IndoBERT (alpha=0).
+4. Return top-K (default K=10).
+
+Tim Anggota D: experiment alpha in [0.0, 0.3, 0.5, 0.7] untuk lihat balance
+yang optimal. Pure rerank (alpha=0) sering kompetitif untuk Indonesian
+queries yang semantik.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Callable, Optional
+
+import numpy as np
+
+from .base import Document, IndexBase, SearchHit
+from .bm25 import BM25Index
+from .indobert import IndoBERTIndex
+
+
+class HybridIndex(IndexBase):
+    name = "hybrid"
+
+    def __init__(
+        self,
+        bm25: BM25Index,
+        indobert: IndoBERTIndex,
+        bm25_top_k: int = 50,
+        # alpha 0.9 dari grid search pool-restricted n=30 (eval/
+        # hybrid_alpha_grid_pool.csv): kurva flat 0.6-0.9, puncak 0.9
+        # (MAP 0.5909 vs BM25 murni 0.5836; selisih kecil, inconclusive).
+        # Default lama 0.3 menyeret hybrid jatuh di standard eval
+        # (MAP 0.188 vs 0.267 di alpha 0.9) karena GT lexical-pooled.
+        alpha: float = 0.9,
+        query_preprocessor: Optional[Callable[[str], str]] = None,
+    ):
+        """Args:
+        bm25: BM25Index yang sudah di-build
+        indobert: IndoBERTIndex yang sudah di-build
+        bm25_top_k: jumlah candidate dari BM25 untuk re-rank (default 50)
+        alpha: weight untuk BM25 score di final combination.
+               alpha=0 -> pure IndoBERT rerank, alpha=1 -> pure BM25.
+        query_preprocessor: callable(q_raw) -> q_processed; dipakai untuk BM25
+            karena BM25 perlu teks ter-preprocess (stemmed). IndoBERT pakai
+            raw query (natural language). Kalau None, q dipakai untuk
+            keduanya (backwards-compat untuk pemanggil yang sudah preprocess).
+        """
+        self.bm25 = bm25
+        self.indobert = indobert
+        self.bm25_top_k = bm25_top_k
+        self.alpha = alpha
+        self.query_preprocessor = query_preprocessor
+        self._size = bm25.size()
+
+    def build(self, corpus: list[Document]) -> None:
+        """Hybrid = composition; sub-indexes di-build separately."""
+        # Tetap track size kalau dipanggil
+        self._size = len(corpus)
+        # NOTE: caller harus pastikan bm25 & indobert sudah di-build sebelum query.
+
+    def query(self, q: str, top_k: int = 10) -> list[SearchHit]:
+        # 1. BM25 candidates — pakai processed query (stemmed lexical match)
+        q_for_bm25 = self.query_preprocessor(q) if self.query_preprocessor else q
+        bm25_hits = self.bm25.query(q_for_bm25, top_k=self.bm25_top_k)
+        if not bm25_hits:
+            return []
+        candidate_ids = [h.doc_id for h in bm25_hits]
+        bm25_scores_raw = np.array([h.score for h in bm25_hits])
+
+        # 2. IndoBERT score candidates pakai RAW query (semantic embedding)
+        q_emb = self.indobert.encode_query(q)
+        indobert_scores_pairs = self.indobert.score_docs(q_emb, candidate_ids)
+        indobert_scores_map = dict(indobert_scores_pairs)
+
+        # Align IndoBERT scores dengan order BM25 candidates
+        indobert_scores_raw = np.array(
+            [indobert_scores_map.get(doc_id, 0.0) for doc_id in candidate_ids]
+        )
+
+        # 3. Min-max normalize tiap score list (skala [0, 1])
+        bm25_norm = self._minmax_normalize(bm25_scores_raw)
+        indobert_norm = self._minmax_normalize(indobert_scores_raw)
+
+        # 4. Combine
+        combined = self.alpha * bm25_norm + (1 - self.alpha) * indobert_norm
+
+        # 5. Sort + top-K
+        sorted_idx = np.argsort(-combined)[:top_k]
+        return [
+            SearchHit(
+                doc_id=candidate_ids[idx],
+                score=float(combined[idx]),
+                rank=rank + 1,
+            )
+            for rank, idx in enumerate(sorted_idx)
+        ]
+
+    @staticmethod
+    def _minmax_normalize(scores: np.ndarray) -> np.ndarray:
+        """Min-max normalize ke [0, 1]. Kalau semua sama, return zeros."""
+        if len(scores) == 0:
+            return scores
+        s_min, s_max = scores.min(), scores.max()
+        if s_max - s_min < 1e-9:
+            return np.zeros_like(scores)
+        return (scores - s_min) / (s_max - s_min)
+
+    def save(self, path: Path) -> None:
+        """Hybrid bukan index murni — sub-indexes save separately."""
+        raise NotImplementedError(
+            "HybridIndex composition; save bm25 & indobert separately. "
+            "Hybrid restore via constructor dari sub-indexes yang sudah di-load."
+        )
+
+    @classmethod
+    def load(cls, path: Path) -> "HybridIndex":
+        raise NotImplementedError(
+            "HybridIndex composition; load bm25 & indobert separately, "
+            "lalu instantiate HybridIndex(bm25, indobert)."
+        )

backend/app/indexing/indobert.py

@@ -0,0 +1,178 @@
+"""Neural / semantic index pakai sentence-transformers + FAISS.
+
+Default model: `paraphrase-multilingual-MiniLM-L12-v2` (~118 MB, multilingual,
+inferensi cepat di CPU). Untuk hasil lebih bagus tapi heavy: ganti ke
+`indobenchmark/indobert-base-p2` (~440 MB).
+
+FAISS IndexFlatIP = inner product. Karena embeddings sudah L2-normalized
+(via `normalize_embeddings=True`), inner product == cosine similarity.
+
+Tim Anggota D: experiment di notebook 03_model_comparison.ipynb:
+- Compare MiniLM (small/fast) vs IndoBERT-base (better quality)
+- Pooling strategy: mean-pool (default) vs CLS token
+- FAISS Flat (exhaustive) vs IVF (clustered, faster tapi approximate)
+
+Untuk corpus <= 5K docs, Flat sudah cukup (sub-ms query).
+"""
+
+from __future__ import annotations
+
+import json
+import pickle
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+
+from .base import Document, IndexBase, SearchHit
+
+
+class IndoBERTIndex(IndexBase):
+    name = "indobert"
+
+    def __init__(
+        self,
+        model_name: str = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
+        normalize_embeddings: bool = True,
+        batch_size: int = 32,
+    ):
+        self.model_name = model_name
+        self.normalize_embeddings = normalize_embeddings
+        self.batch_size = batch_size
+        self.embeddings: Optional[np.ndarray] = None  # (N, dim) float32
+        self.index = None  # faiss.IndexFlatIP
+        self.doc_ids: list[str] = []
+        self.id_to_idx: dict[str, int] = {}
+        self._size = 0
+        self._model = None  # Lazy load
+
+    @property
+    def model(self):
+        """Lazy load fastembed TextEmbedding (ONNX runtime).
+
+        Trade-off vs sentence-transformers + torch:
+        - fastembed: ~150MB total RAM (Render free tier compatible)
+        - sentence-transformers + torch: ~500MB total (OOM di 512MB Render)
+        - Same underlying MiniLM model, numerical results virtually identical
+        - Faster init: ~1-2s vs 10-30s untuk first encode
+        """
+        if self._model is None:
+            import os
+
+            from fastembed import TextEmbedding
+
+            # cache_dir eksplisit (FASTEMBED_CACHE_PATH) supaya model ONNX yang
+            # di-pre-download saat docker build ke-reuse di runtime — tanpa ini
+            # tiap cold start download ulang ~120MB.
+            cache_dir = os.getenv("FASTEMBED_CACHE_PATH")
+            self._model = TextEmbedding(self.model_name, cache_dir=cache_dir)
+        return self._model
+
+    def _encode(self, texts: list[str]) -> "np.ndarray":
+        """Encode texts via fastembed + L2-normalize untuk FAISS IndexFlatIP."""
+        import numpy as np
+
+        embeddings = list(self.model.embed(texts))
+        arr = np.vstack(embeddings).astype("float32")
+        if self.normalize_embeddings:
+            norms = np.linalg.norm(arr, axis=1, keepdims=True)
+            norms = np.where(norms > 0, norms, 1.0)
+            arr = arr / norms
+        return arr
+
+    def build(self, corpus: list[Document]) -> None:
+        import faiss
+
+        texts = [(doc.raw_text or doc.text) for doc in corpus]
+        embeddings = self._encode(texts)
+        self.embeddings = embeddings
+        # FAISS IndexFlatIP — exhaustive cosine via inner product
+        dim = embeddings.shape[1]
+        self.index = faiss.IndexFlatIP(dim)
+        self.index.add(embeddings)
+        self.doc_ids = [doc.id for doc in corpus]
+        self.id_to_idx = {doc_id: i for i, doc_id in enumerate(self.doc_ids)}
+        self._size = len(corpus)
+
+    def encode_query(self, q: str) -> np.ndarray:
+        """Encode query string ke embedding (via fastembed ONNX).
+
+        Returns shape (1, dim) — satu baris, bukan flat (dim,).
+        FAISS index.search dan score_docs keduanya mengharapkan (1, dim).
+        Jangan lakukan dot-product langsung tanpa .T transpose.
+        """
+        return self._encode([q])
+
+    def score_docs(
+        self, q_emb: np.ndarray, doc_ids: list[str]
+    ) -> list[tuple[str, float]]:
+        """Score subset doc_ids vs query embedding using pre-computed embeddings.
+
+        Dipakai oleh HybridIndex untuk rerank candidate dari BM25 tanpa
+        re-encode dokumen.
+        """
+        if self.embeddings is None:
+            return []
+        indices = [self.id_to_idx[d] for d in doc_ids if d in self.id_to_idx]
+        subset = self.embeddings[indices]
+        # Inner product (cosine kalau normalized)
+        scores = (subset @ q_emb.T).flatten()
+        return [
+            (self.doc_ids[idx], float(score))
+            for idx, score in zip(indices, scores)
+        ]
+
+    def query(self, q: str, top_k: int = 10) -> list[SearchHit]:
+        if self.index is None or self._size == 0:
+            return []
+        q_emb = self.encode_query(q)
+        top_k = min(top_k, self._size)
+        scores, indices = self.index.search(q_emb, top_k)
+        return [
+            SearchHit(
+                doc_id=self.doc_ids[idx],
+                score=float(scores[0][rank]),
+                rank=rank + 1,
+            )
+            for rank, idx in enumerate(indices[0])
+            if idx >= 0  # FAISS pakai -1 untuk padding kalau result < top_k
+        ]
+
+    def save(self, path: Path) -> None:
+        """Save: embeddings.npy + faiss.index + meta.json di folder `path`."""
+        import faiss
+
+        path.mkdir(parents=True, exist_ok=True)
+        np.save(path / "embeddings.npy", self.embeddings)
+        faiss.write_index(self.index, str(path / "faiss.index"))
+        with open(path / "meta.json", "w", encoding="utf-8") as f:
+            json.dump(
+                {
+                    "model_name": self.model_name,
+                    "normalize_embeddings": self.normalize_embeddings,
+                    "batch_size": self.batch_size,
+                    "doc_ids": self.doc_ids,
+                    "size": self._size,
+                },
+                f,
+                ensure_ascii=False,
+            )
+
+    @classmethod
+    def load(cls, path: Path) -> "IndoBERTIndex":
+        """Load dari folder yang berisi embeddings.npy + faiss.index + meta.json."""
+        import faiss
+
+        with open(path / "meta.json", encoding="utf-8") as f:
+            meta = json.load(f)
+        instance = cls(
+            model_name=meta["model_name"],
+            normalize_embeddings=meta["normalize_embeddings"],
+            batch_size=meta["batch_size"],
+        )
+        instance.embeddings = np.load(path / "embeddings.npy")
+        instance.index = faiss.read_index(str(path / "faiss.index"))
+        instance.doc_ids = meta["doc_ids"]
+        instance.id_to_idx = {d: i for i, d in enumerate(instance.doc_ids)}
+        instance._size = meta["size"]
+        return instance

backend/app/indexing/loader.py

@@ -0,0 +1,68 @@
+"""Loader helper untuk FastAPI lifespan startup.
+
+Load semua indexes dari disk sekali saat backend startup, store di
+`app.state` untuk reuse di route handlers.
+
+Usage di app/main.py:
+
+    from app.indexing.loader import load_all_indexes
+
+    @asynccontextmanager
+    async def lifespan(app: FastAPI):
+        indexes = load_all_indexes(Path(settings.indexes_dir))
+        app.state.tfidf = indexes["tfidf"]
+        app.state.bm25 = indexes["bm25"]
+        app.state.indobert = indexes["indobert"]
+        app.state.hybrid = HybridIndex(indexes["bm25"], indexes["indobert"])
+        yield
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+from loguru import logger
+
+from .bm25 import BM25Index
+from .tfidf import TFIDFIndex
+
+
+def load_all_indexes(
+    indexes_dir: Path, include_neural: bool = True
+) -> dict[str, object]:
+    """Load semua indexes dari folder. Return dict {name: index_instance}.
+
+    Missing index logged sebagai warning, tidak raise (supaya partial deploy OK).
+    `include_neural=False` melewati IndoBERT supaya runtime production tidak
+    perlu import faiss/fastembed (hemat RAM).
+    """
+    result: dict[str, object] = {}
+
+    tfidf_path = indexes_dir / "tfidf.pkl"
+    if tfidf_path.exists():
+        logger.info(f"[load] TF-IDF dari {tfidf_path}")
+        result["tfidf"] = TFIDFIndex.load(tfidf_path)
+    else:
+        logger.warning(f"[skip] TF-IDF tidak ditemukan di {tfidf_path}")
+
+    bm25_path = indexes_dir / "bm25.pkl"
+    if bm25_path.exists():
+        logger.info(f"[load] BM25 dari {bm25_path}")
+        result["bm25"] = BM25Index.load(bm25_path)
+    else:
+        logger.warning(f"[skip] BM25 tidak ditemukan di {bm25_path}")
+
+    indobert_path = indexes_dir / "indobert"
+    if not include_neural:
+        logger.info("[skip] IndoBERT dilewati (enable_neural=False)")
+    elif indobert_path.exists() and indobert_path.is_dir():
+        from .indobert import IndoBERTIndex  # lazy: hindari import faiss saat off
+
+        logger.info(f"[load] IndoBERT dari {indobert_path}")
+        result["indobert"] = IndoBERTIndex.load(indobert_path)
+    else:
+        logger.warning(f"[skip] IndoBERT tidak ditemukan di {indobert_path}")
+
+    logger.info(f"[load] {len(result)} indexes loaded: {list(result.keys())}")
+    return result

backend/app/indexing/tfidf.py

@@ -0,0 +1,94 @@
+"""TF-IDF + cosine similarity index (scikit-learn).
+
+Hyperparameter default:
+- ngram_range=(1, 2): unigram + bigram (capture compound terms seperti
+  "kamar mandi", "air panas")
+- min_df=2: drop term yang muncul cuma sekali (likely typo/noise)
+- max_features=10000: limit vocab size untuk memory + speed
+
+Tim Anggota C: experiment dengan hyperparameter di
+notebooks/03_model_comparison.ipynb. Pada corpus real 227 listing, TF-IDF
+MAP ~0.24 (standard) / ~0.59 (pool-restricted) — kompetitif dengan BM25.
+"""
+
+from __future__ import annotations
+
+import pickle
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+from .base import Document, IndexBase, SearchHit
+
+
+class TFIDFIndex(IndexBase):
+    name = "tfidf"
+
+    def __init__(
+        self,
+        ngram_range: tuple[int, int] = (1, 2),
+        min_df: int = 2,
+        max_features: int = 10000,
+    ):
+        self.vectorizer = TfidfVectorizer(
+            ngram_range=ngram_range,
+            min_df=min_df,
+            max_features=max_features,
+            sublinear_tf=True,  # log(1+tf) — handle high-tf terms
+        )
+        self.doc_matrix = None  # sparse matrix (N, vocab_size)
+        self.doc_ids: list[str] = []
+        self._size = 0
+
+    def build(self, corpus: list[Document]) -> None:
+        texts = [doc.text for doc in corpus]
+        self.doc_ids = [doc.id for doc in corpus]
+        self.doc_matrix = self.vectorizer.fit_transform(texts)
+        self._size = len(corpus)
+
+    def query(self, q: str, top_k: int = 10) -> list[SearchHit]:
+        if self.doc_matrix is None or self._size == 0:
+            return []
+        q_vec = self.vectorizer.transform([q])
+        # Cosine similarity (since TF-IDF vectors already L2-normalized by default)
+        scores = cosine_similarity(q_vec, self.doc_matrix).flatten()
+        # Top-K indices (descending)
+        top_k = min(top_k, self._size)
+        top_indices = np.argpartition(-scores, top_k - 1)[:top_k]
+        # Sort within top-K
+        top_indices = top_indices[np.argsort(-scores[top_indices])]
+        return [
+            SearchHit(
+                doc_id=self.doc_ids[idx],
+                score=float(scores[idx]),
+                rank=rank + 1,
+            )
+            for rank, idx in enumerate(top_indices)
+        ]
+
+    def save(self, path: Path) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(path, "wb") as f:
+            pickle.dump(
+                {
+                    "vectorizer": self.vectorizer,
+                    "doc_matrix": self.doc_matrix,
+                    "doc_ids": self.doc_ids,
+                    "size": self._size,
+                },
+                f,
+            )
+
+    @classmethod
+    def load(cls, path: Path) -> "TFIDFIndex":
+        with open(path, "rb") as f:
+            data = pickle.load(f)
+        instance = cls.__new__(cls)
+        instance.vectorizer = data["vectorizer"]
+        instance.doc_matrix = data["doc_matrix"]
+        instance.doc_ids = data["doc_ids"]
+        instance._size = data["size"]
+        return instance

backend/app/main.py

@@ -0,0 +1,226 @@
+"""FastAPI entry point untuk KozyNear search engine.
+
+Architecture: single-container Docker. FastAPI serve:
+- React SPA dari `/app/static/` (built dari frontend/)
+- API endpoints di `/api/*`
+- Health check di `/health` (Render uses this)
+- Swagger UI di `/api/docs`
+
+Routing order (FastAPI matches in registration order):
+1. Specific API routes: /health, /api/info, /api/search, /api/listings/{id}, /api/eval/*
+2. /api/docs, /api/openapi.json (FastAPI auto-generated)
+3. /assets/* (Vite build output)
+4. Catch-all /{path} -> serve index.html for SPA routing (React Router fallback)
+
+Lifespan:
+- Startup: load preprocessing pipeline + IR indexes dari disk
+- Shutdown: cleanup
+"""
+
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+from loguru import logger
+
+from app.api.eval import router as eval_router
+from app.api.health import router as health_router
+from app.api.insights import router as insights_router
+from app.api.listings import router as listings_router
+from app.api.search import router as search_router
+from app.core.config import settings
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Startup: load heavy resources. Shutdown: cleanup."""
+    logger.info(f"[startup] KozyNear backend (env={settings.environment})")
+
+    # Initialize app.state slots
+    app.state.tfidf = None
+    app.state.bm25 = None
+    app.state.indobert = None
+    app.state.hybrid = None
+    app.state.preprocessing_pipeline = None
+    app.state.gazetteer = None
+
+    # 1. Preprocessing pipeline (Sastrawi factory init ~1s)
+    try:
+        from app.preprocessing import PreprocessingPipeline
+
+        app.state.preprocessing_pipeline = PreprocessingPipeline()
+        logger.info("[startup] preprocessing pipeline loaded")
+    except Exception as e:
+        logger.error(f"[startup] preprocessing pipeline FAIL: {e}")
+
+    # 1b. Gazetteer (kampus + landmark) untuk smart search
+    try:
+        from app.search.gazetteer import Gazetteer
+
+        app.state.gazetteer = Gazetteer.load()
+        logger.info("[startup] gazetteer loaded")
+    except Exception as e:
+        app.state.gazetteer = None
+        logger.error(f"[startup] gazetteer FAIL: {e}")
+
+    # 2. IR indexes (graceful skip kalau missing)
+    try:
+        from app.indexing.loader import load_all_indexes
+
+        indexes_path = Path(settings.indexes_dir)
+        if indexes_path.exists():
+            indexes = load_all_indexes(
+                indexes_path, include_neural=settings.enable_neural
+            )
+            app.state.tfidf = indexes.get("tfidf")
+            app.state.bm25 = indexes.get("bm25")
+            app.state.indobert = indexes.get("indobert")
+
+            if app.state.bm25 is not None and app.state.indobert is not None:
+                from app.indexing.hybrid import HybridIndex
+
+                _pipeline = getattr(app.state, "preprocessing_pipeline", None)
+                app.state.hybrid = HybridIndex(
+                    app.state.bm25,
+                    app.state.indobert,
+                    query_preprocessor=(
+                        (lambda q: _pipeline.process(q).processed)
+                        if _pipeline is not None else None
+                    ),
+                )
+                logger.info("[startup] hybrid index assembled")
+
+            # BACKGROUND prewarm IndoBERT model:
+            # - Index sudah loaded (embeddings.npy + faiss.index dari disk)
+            # - SentenceTransformer model load butuh ~10-30s di Render free tier CPU
+            # - SYNCHRONOUS prewarm di lifespan = block port binding (deploy fail)
+            # - LAZY load di first request = Render proxy timeout 30s -> 502
+            # - SOLUTION: spawn async background task setelah port binds
+            # - Sambil model load di background, /search?model=indobert return
+            #   503 dengan flag indobert_ready=False sampai task complete
+            app.state.indobert_ready = False
+            app.state.indobert_failed = False
+            if app.state.indobert is not None:
+                async def _bg_prewarm():
+                    import asyncio
+                    import time
+                    try:
+                        logger.info("[bg] starting IndoBERT model prewarm...")
+                        t0 = time.perf_counter()
+                        # Run blocking encode_query di thread pool supaya
+                        # tidak block event loop
+                        await asyncio.to_thread(
+                            app.state.indobert.encode_query, "warmup",
+                        )
+                        elapsed = time.perf_counter() - t0
+                        app.state.indobert_ready = True
+                        logger.info(f"[bg] IndoBERT model warm ({elapsed:.1f}s)")
+                    except Exception as e:
+                        logger.error(f"[bg] IndoBERT prewarm FAIL: {e}")
+                        # Set ke None supaya graceful 503 di /search
+                        app.state.indobert = None
+                        app.state.hybrid = None
+                        app.state.indobert_failed = True
+
+                import asyncio
+                asyncio.create_task(_bg_prewarm())
+                logger.info(
+                    "[startup] IndoBERT prewarm scheduled di background; "
+                    "/search?model=indobert akan return 503 sampai ready"
+                )
+        else:
+            logger.warning(
+                f"[startup] indexes_dir tidak ada: {indexes_path}. "
+                "Build dulu via `python -m app.indexing.build`."
+            )
+    except Exception as e:
+        logger.error(f"[startup] index loading FAIL: {e}")
+
+    yield
+    logger.info("[shutdown] KozyNear backend stopping")
+
+
+app = FastAPI(
+    title="KozyNear API",
+    description=(
+        "Indonesian Kos-Kosan Search Engine -- UNILA STKI Final Project. "
+        "Compare 3 IR paradigms: TF-IDF, BM25, IndoBERT+FAISS, dan Hybrid."
+    ),
+    version="0.1.0",
+    docs_url="/api/docs",
+    redoc_url="/api/redoc",
+    openapi_url="/api/openapi.json",
+    lifespan=lifespan,
+)
+
+# CORS: tetap aktifkan untuk dev (frontend di :5173, backend di :8000).
+# Di production single-container, frontend & API same origin -> CORS no-op.
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.cors_origins,
+    allow_credentials=True,
+    allow_methods=["GET", "POST"],
+    allow_headers=["*"],
+)
+
+# ----------------------------------------------------------------------------
+# Routes order: API dulu, lalu static catch-all
+# ----------------------------------------------------------------------------
+
+# Health check at /health (Render uses this)
+app.include_router(health_router, tags=["health"])
+
+# API endpoints under /api prefix
+app.include_router(search_router, prefix="/api", tags=["search"])
+app.include_router(listings_router, prefix="/api", tags=["listings"])
+app.include_router(eval_router, prefix="/api")  # eval_router internal prefix /eval
+app.include_router(insights_router, prefix="/api", tags=["insights"])
+
+
+# ----------------------------------------------------------------------------
+# Static frontend (Docker build) -- serve dari /app/static
+# ----------------------------------------------------------------------------
+# Path discovery: di Docker container path absolute /app/static.
+# Untuk local dev tanpa build, skip mount (frontend run terpisah via vite dev).
+_DOCKER_STATIC = Path("/app/static")
+_LOCAL_STATIC = Path(__file__).resolve().parent.parent.parent / "static"
+
+STATIC_DIR: Path | None = None
+if _DOCKER_STATIC.exists():
+    STATIC_DIR = _DOCKER_STATIC
+elif _LOCAL_STATIC.exists():
+    STATIC_DIR = _LOCAL_STATIC
+
+if STATIC_DIR is not None:
+    # Mount /assets/ untuk JS/CSS chunks (Vite outputs ke /assets/)
+    assets_dir = STATIC_DIR / "assets"
+    if assets_dir.exists():
+        app.mount(
+            "/assets",
+            StaticFiles(directory=str(assets_dir)),
+            name="assets",
+        )
+
+    # Catch-all: serve real file kalau ada, else index.html (SPA fallback)
+    @app.get("/{full_path:path}", include_in_schema=False)
+    async def serve_spa(full_path: str):
+        # Security: prevent path traversal
+        candidate = (STATIC_DIR / full_path).resolve()
+        if not str(candidate).startswith(str(STATIC_DIR.resolve())):
+            raise HTTPException(403, "Forbidden")
+
+        if candidate.is_file():
+            return FileResponse(candidate)
+
+        # SPA fallback: serve index.html untuk unknown routes
+        index = STATIC_DIR / "index.html"
+        if index.exists():
+            return FileResponse(index)
+        raise HTTPException(404, "Not Found")
+
+    logger.info(f"[static] frontend mounted dari {STATIC_DIR}")
+else:
+    logger.info("[static] tidak ada static dir -- run frontend terpisah via vite dev")

backend/app/models/__init__.py

@@ -0,0 +1,11 @@
+"""Pydantic schemas (API request/response) + SQLAlchemy ORM models.
+
+Analogi Laravel:
+- Pydantic schemas = Form Request validation + API Resource
+- SQLAlchemy ORM = Eloquent Models
+
+Files (akan dibuat di Week 2):
+- listing.py — Listing ORM + Pydantic schemas (ListingRead, ListingDetail)
+- query.py — Query + Annotation + EvalResult ORM
+- search.py — SearchRequest, SearchResponse Pydantic schemas
+"""

backend/app/models/eval.py

@@ -0,0 +1,144 @@
+"""ORM + Pydantic schemas untuk eval data (queries, ground truth, results).
+
+Tables di-create via Alembic migration 001_initial_schema.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict
+from sqlalchemy import (
+    CheckConstraint,
+    DateTime,
+    ForeignKey,
+    Numeric,
+    SmallInteger,
+    String,
+    Text,
+    func,
+)
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.core.db import Base
+
+
+# =============================================================================
+# ORM models
+# =============================================================================
+class Query(Base):
+    """Query set untuk evaluation."""
+    __tablename__ = "queries"
+
+    id: Mapped[str] = mapped_column(String, primary_key=True)
+    query_text: Mapped[str] = mapped_column(Text, nullable=False)
+    context: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    expected_tipe: Mapped[Optional[str]] = mapped_column(
+        String(20), nullable=True
+    )
+
+
+class GroundTruth(Base):
+    """Raw annotation per (query, listing, annotator)."""
+    __tablename__ = "ground_truth"
+
+    query_id: Mapped[str] = mapped_column(
+        String, ForeignKey("queries.id"), primary_key=True
+    )
+    listing_id: Mapped[str] = mapped_column(
+        String, ForeignKey("listings.id"), primary_key=True
+    )
+    annotator: Mapped[str] = mapped_column(String(50), primary_key=True)
+    relevance: Mapped[int] = mapped_column(SmallInteger, nullable=False)
+    notes: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+
+    __table_args__ = (
+        CheckConstraint(
+            "relevance IN (0, 1, 2)", name="ck_relevance_0_1_2"
+        ),
+    )
+
+
+class GroundTruthConsensus(Base):
+    """Consensus labels (post-discussion). Input untuk evaluation runner."""
+    __tablename__ = "ground_truth_consensus"
+
+    query_id: Mapped[str] = mapped_column(
+        String, ForeignKey("queries.id"), primary_key=True
+    )
+    listing_id: Mapped[str] = mapped_column(
+        String, ForeignKey("listings.id"), primary_key=True
+    )
+    relevance: Mapped[int] = mapped_column(SmallInteger, nullable=False)
+
+    __table_args__ = (
+        CheckConstraint(
+            "relevance IN (0, 1, 2)", name="ck_consensus_relevance"
+        ),
+    )
+
+
+class EvalResult(Base):
+    """Per-model per-query evaluation metrics."""
+    __tablename__ = "eval_results"
+
+    model: Mapped[str] = mapped_column(String(20), primary_key=True)
+    query_id: Mapped[str] = mapped_column(
+        String, ForeignKey("queries.id"), primary_key=True
+    )
+    precision_at_5: Mapped[Optional[float]] = mapped_column(
+        Numeric(5, 4), nullable=True
+    )
+    precision_at_10: Mapped[Optional[float]] = mapped_column(
+        Numeric(5, 4), nullable=True
+    )
+    average_precision: Mapped[Optional[float]] = mapped_column(
+        Numeric(5, 4), nullable=True
+    )
+    ndcg_at_10: Mapped[Optional[float]] = mapped_column(
+        Numeric(5, 4), nullable=True
+    )
+    reciprocal_rank: Mapped[Optional[float]] = mapped_column(
+        Numeric(5, 4), nullable=True
+    )
+    computed_at: Mapped[Optional[datetime]] = mapped_column(
+        DateTime, server_default=func.now(), nullable=True
+    )
+
+
+# =============================================================================
+# Pydantic schemas (API)
+# =============================================================================
+class ModelMetrics(BaseModel):
+    """Aggregate metrics per IR model."""
+    model_config = ConfigDict(from_attributes=True, protected_namespaces=())
+
+    model: str
+    p_at_5: float
+    p_at_10: float
+    map: float
+    ndcg_at_10: float
+    mrr: float
+    n_queries: int
+
+
+class EvalSummary(BaseModel):
+    """Response untuk /eval/summary."""
+    models: list[ModelMetrics]
+    total_queries: int
+    total_listings_indexed: int
+
+
+class QueryResult(BaseModel):
+    """Per-query metric breakdown untuk /eval/query/{id}."""
+    model_config = ConfigDict(protected_namespaces=())
+
+    query_id: str
+    query_text: str
+    model: str
+    p_at_5: float
+    p_at_10: float
+    average_precision: float
+    ndcg_at_10: float
+    reciprocal_rank: float

backend/app/models/listing.py

@@ -0,0 +1,98 @@
+"""Listing ORM model + Pydantic schemas.
+
+Analogi Laravel:
+- Listing ORM = Eloquent Model (database mapping)
+- ListingRead/Detail Pydantic = Form Request / API Resource (serialization)
+
+ORM untuk persistence, Pydantic untuk API I/O. Konversi lewat
+`ListingRead.model_validate(listing_orm)` atau `from_attributes=True`.
+"""
+
+from __future__ import annotations
+
+from datetime import date
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+from sqlalchemy import ARRAY, Date, DateTime, Integer, Numeric, String, Text, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.core.db import Base
+
+
+# =============================================================================
+# ORM model
+# =============================================================================
+class Listing(Base):
+    """Table `listings` — raw scraped + processed kos data."""
+
+    __tablename__ = "listings"
+
+    id: Mapped[str] = mapped_column(String, primary_key=True)
+    judul: Mapped[str] = mapped_column(Text, nullable=False)
+    deskripsi: Mapped[str] = mapped_column(Text, nullable=False)
+    deskripsi_processed: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+
+    harga_per_bulan: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
+    tipe: Mapped[Optional[str]] = mapped_column(String(20), nullable=True)
+    fasilitas: Mapped[Optional[list[str]]] = mapped_column(
+        ARRAY(String), nullable=True
+    )
+
+    alamat: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    kecamatan: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
+    koordinat_lat: Mapped[Optional[float]] = mapped_column(
+        Numeric(8, 5), nullable=True
+    )
+    koordinat_lng: Mapped[Optional[float]] = mapped_column(
+        Numeric(8, 5), nullable=True
+    )
+    jarak_kampus_km: Mapped[Optional[float]] = mapped_column(
+        Numeric(5, 2), nullable=True
+    )
+
+    url_source: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    scrape_date: Mapped[Optional[date]] = mapped_column(Date, nullable=True)
+    source: Mapped[Optional[str]] = mapped_column(String(20), nullable=True)
+
+    inserted_at: Mapped[Optional[date]] = mapped_column(
+        DateTime, server_default=func.now(), nullable=True
+    )
+
+
+# =============================================================================
+# Pydantic schemas (API I/O)
+# =============================================================================
+class ListingBase(BaseModel):
+    """Shared fields antara list & detail."""
+    model_config = ConfigDict(from_attributes=True)
+
+    id: str
+    judul: str
+    harga_per_bulan: Optional[int] = None
+    tipe: Optional[str] = None
+    fasilitas: Optional[list[str]] = None
+    alamat: Optional[str] = None
+    kecamatan: Optional[str] = None
+
+
+class ListingRead(ListingBase):
+    """Schema untuk hasil search (deskripsi present, score dari IR model)."""
+    deskripsi: str = Field(..., description="Original deskripsi; UI truncate ~200 chars")
+    score: float = Field(..., description="Relevance score dari IR model")
+    koordinat: Optional[list[float]] = Field(
+        None, description="[lat, lng] untuk pin Google Maps di frontend"
+    )
+
+
+class ListingDetail(ListingBase):
+    """Schema untuk endpoint /listings/{id} (full detail)."""
+    deskripsi: str
+    deskripsi_processed: Optional[str] = None
+    koordinat: Optional[list[float]] = Field(
+        None, description="[lat, lng] format"
+    )
+    jarak_kampus_km: Optional[float] = None
+    url_source: Optional[str] = None
+    scrape_date: Optional[date] = None
+    source: Optional[str] = None

backend/app/models/search.py

@@ -0,0 +1,28 @@
+"""Pydantic schemas untuk /search endpoint request/response."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from app.models.listing import ListingRead
+
+
+# Allowed IR model names (sinkron dengan IndexBase.name di app.indexing.*)
+Model = Literal["tfidf", "bm25", "indobert", "hybrid", "smart"]
+
+
+class SearchResponse(BaseModel):
+    """Response untuk GET /search."""
+    query: str
+    model: Model
+    top_k: int
+    took_ms: int = Field(..., description="Latency total search dalam millisecond")
+    results: list[ListingRead]
+    understood: dict = Field(
+        default_factory=dict, description="Atribut terdeteksi dari query (mode smart)"
+    )
+    relaxed: list[str] = Field(
+        default_factory=list, description="Filter yang dilonggarkan (mode smart)"
+    )

backend/app/preprocessing/README.md

@@ -0,0 +1,161 @@
+# Preprocessing Module
+
+Indonesian text preprocessing pipeline untuk kos listings, dengan custom jargon
+dictionary (105+ entries).
+
+## Pipeline Order (PENTING)
+
+```
+raw text
+  -> 1. HTML strip (BeautifulSoup)
+  -> 2. Whitespace normalize
+  -> 3. Price extraction (SEBELUM lowercase — preserve `Rp` capitalized)
+  -> 4. Lowercase
+  -> 5. Jargon dictionary substitution (word-boundary, longest-first)
+  -> 6. Spelling correction (typo dictionary)
+  -> 7. Tokenize (whitespace + punctuation)
+  -> 8. Stopword removal (Sastrawi default + custom domain)
+  -> 9. Stem (Sastrawi StemmerFactory, LRU-cached)
+```
+
+**Anti-pattern**: lowercase SEBELUM extract harga → regex `[Rr][Pp]` masih
+match, tapi edge case `Rp850k` (tanpa space) jadi rapuh kalau pattern berubah.
+
+## Usage
+
+```python
+from app.preprocessing import PreprocessingPipeline, PipelineConfig
+
+# Default: semua stage aktif
+pipeline = PreprocessingPipeline()
+result = pipeline.process(
+    "Kos Putra AC WiFi <b>Rp 850.000</b>/bulan dekat unyila"
+)
+
+print(result.processed)         # joined tokens setelah stemming
+print(result.tokens)            # ['putra', 'air', 'condition', ...]
+print(result.extracted_prices)  # [850000]
+print(result.stages_applied)    # ['strip_html', 'normalize_whitespace', ...]
+
+# Shorthand utility
+from app.preprocessing import preprocess
+text = preprocess("kos murah 500k dekat unyila")
+
+# Experiment mode: disable stem
+config = PipelineConfig(stem=False)
+pipeline = PreprocessingPipeline(config)
+```
+
+## What's Done (oleh mentor — scaffold)
+
+- [x] `pipeline.py` — orchestrator dengan stage toggles
+- [x] `normalizer.py` — HTML strip, whitespace, lowercase, inline price extract
+- [x] `jargon.py` — **105+ entries** (abbreviation, location, type, rules,
+  payment, kos form) — `python -m app.preprocessing.jargon` cek count
+- [x] `spelling.py` — typo dictionary baseline (15+ fixes)
+- [x] `tokenizer.py` — whitespace + punctuation tokenizer
+- [x] `stopwords.py` — Sastrawi + custom kos stopwords
+- [x] `stemmer.py` — Sastrawi wrapper dengan LRU cache 10k
+- [x] Tests di `backend/tests/test_preprocessing.py`
+
+## What Tim Anggota B (Preprocessing Engineer) WAJIB Do
+
+### Priority 1 — Tambah jargon dict ke ≥120 entries
+
+File: `jargon.py`. Saat ini 105 entries (sudah lewat threshold 100, tapi
+**lebih bagus 120+ untuk dokumentasi rubric**).
+
+Cara discover entries baru:
+1. Tunggu Anggota A scrape selesai (`data/raw/mamikos.jsonl`)
+2. Random sample 50 listings, baca manual:
+   ```python
+   import json
+   with open("data/raw/mamikos.jsonl") as f:
+       sample = [json.loads(line) for line in list(f)[:50]]
+   for listing in sample:
+       print(listing["deskripsi"][:300])
+       print("---")
+   ```
+3. Catat pattern abbreviation / slang yang belum di `jargon.py`
+4. Tambah ke kategori sesuai (`ABBREVIATIONS`, `LOCATIONS`, etc.)
+5. Run check:
+   ```bash
+   cd backend
+   python -m app.preprocessing.jargon
+   # Output: KOS_JARGON_DICT size: 120+ -- OK: meets rubric requirement
+   ```
+
+### Priority 2 — Custom stopwords dari EDA
+
+File: `stopwords.py` — `DEFAULT_CUSTOM_STOPWORDS`.
+
+Di notebook `01_eda.ipynb`:
+```python
+from collections import Counter
+import json
+
+corpus = [json.loads(line) for line in open("../data/raw/mamikos.jsonl")]
+all_tokens = []
+for listing in corpus:
+    all_tokens.extend(listing["deskripsi"].lower().split())
+
+# Top-50 paling sering
+for token, count in Counter(all_tokens).most_common(50):
+    print(f"{token}: {count}")
+```
+
+Top-50 paling sering biasanya stopword domain — tambah ke
+`DEFAULT_CUSTOM_STOPWORDS` kalau memang noise (bukan informative term).
+
+### Priority 3 — Benchmark preprocessing impact
+
+File: `notebooks/02_preprocessing_experiment.ipynb` (Anggota B buat sendiri).
+
+Untuk **rubric Preprocessing 15%** — wajib show preprocessing impact secara
+empirik:
+
+| Variant | MAP | P@10 | Notes |
+|---------|-----|------|-------|
+| BASELINE (raw text, no preprocess) | 0.XX | 0.XX | reference |
+| + HTML strip + lowercase           | 0.XX | 0.XX | |
+| + jargon dict                      | 0.XX | 0.XX | expect lift |
+| + spelling correction              | 0.XX | 0.XX | |
+| + stopword removal                 | 0.XX | 0.XX | |
+| + stemming                         | 0.XX | 0.XX | |
+| FULL pipeline                      | 0.XX | 0.XX | |
+
+Insight ditulis di laporan akhir + slide.
+
+### Priority 4 — Spelling correction yang lebih robust (optional, Week 3)
+
+Kalau ada budget waktu di Week 2-3, upgrade `spelling.py`:
+- Edit distance (Levenshtein) untuk catch unknown typos
+- Bigram language model untuk context-aware correction
+
+Libraries: `pyspellchecker`, custom n-gram model.
+
+## Testing
+
+```bash
+cd backend
+pip install -r requirements.txt  # include Sastrawi
+pytest tests/test_preprocessing.py -v
+```
+
+Test coverage:
+- Atomic functions (normalizer, tokenizer, jargon, spelling) — no Sastrawi needed
+- Pipeline integration (stopword, stemmer) — needs Sastrawi installed
+- Anti-pattern test: price extraction sebelum lowercase
+- Jargon substitution dengan word boundary + longest-first ordering
+- Config toggles (disable individual stages)
+
+## Performance Notes
+
+Untuk corpus 3000 listings × ~150 tokens avg:
+- HTML strip: ~5ms/doc
+- Jargon substitution: ~2ms/doc (pre-compiled regex)
+- Sastrawi stem (first time): ~500ms-1s setup, ~10-50ms/doc
+- Sastrawi stem (cached): ~1-5ms/doc
+
+Total batch preprocessing 3000 docs: ~30-60 detik (acceptable, run once,
+save corpus.json).

backend/app/preprocessing/__init__.py

@@ -0,0 +1,36 @@
+"""Indonesian text preprocessing pipeline untuk kos listings.
+
+Public API:
+    from app.preprocessing import (
+        PreprocessingPipeline, PipelineConfig, PipelineResult, preprocess,
+        KOS_JARGON_DICT, jargon_count,
+    )
+
+Pipeline ORDER (PENTING):
+1. HTML strip            (BeautifulSoup)
+2. Whitespace normalize
+3. Price extraction      (SEBELUM lowercase — preserve `Rp`)
+4. Lowercase
+5. Jargon dict substitution
+6. Spelling correction
+7. Tokenize
+8. Stopword removal      (Sastrawi + custom)
+9. Stem                  (Sastrawi StemmerFactory, cached)
+"""
+
+from .jargon import KOS_JARGON_DICT, jargon_count
+from .pipeline import (
+    PipelineConfig,
+    PipelineResult,
+    PreprocessingPipeline,
+    preprocess,
+)
+
+__all__ = [
+    "PreprocessingPipeline",
+    "PipelineConfig",
+    "PipelineResult",
+    "preprocess",
+    "KOS_JARGON_DICT",
+    "jargon_count",
+]

backend/app/preprocessing/jargon.py

@@ -0,0 +1,205 @@
+"""Kos-specific jargon dictionary untuk Indonesian preprocessing.
+
+COURSE REQUIREMENT: ≥100 entries domain-specific (drop rubric Preprocessing 15%
+kalau cuma pakai pipeline Sastrawi default tanpa custom dict).
+
+Mentor menyediakan ~105 baseline entries di file ini. Tim Anggota B (Preprocessing
+Engineer) WAJIB tambah minimal 20-30 lagi dari hasil EDA, terutama dari:
+- Slang kos spesifik Bandar Lampung yang ditemukan saat scrape (cek
+  `data/raw/mamikos.jsonl` untuk pattern abbreviation baru)
+- Variant kecamatan / area UNILA yang belum tercakup
+- Brand merchandise / produk yang sering muncul (merek kasur, mereka brand AC)
+- Singkatan unik dari listing pengelola tertentu
+
+Format: variant (apa adanya di listing) -> canonical (bentuk standar).
+Matching: word-boundary regex, case-insensitive, longest-first ordering.
+"""
+
+from __future__ import annotations
+
+
+# =============================================================================
+# Abbreviation umum kos & fasilitas
+# =============================================================================
+ABBREVIATIONS: dict[str, str] = {
+    # Fasilitas kamar mandi
+    "kmd": "kamar mandi dalam",
+    "km dlm": "kamar mandi dalam",
+    "wc dlm": "kamar mandi dalam",
+    "km dalam": "kamar mandi dalam",
+    "wc dalam": "kamar mandi dalam",
+    "km luar": "kamar mandi luar",
+    "wc luar": "kamar mandi luar",
+    # Fasilitas elektronik
+    "ac": "air conditioner",
+    "tv": "televisi",
+    "wf": "wifi",
+    # Harga & periode
+    "jt": "juta",
+    "rb": "ribu",
+    "thn": "tahun",
+    "bln": "bulan",
+    "blnan": "bulanan",
+    "thnan": "tahunan",
+    "min": "minimum",
+    "maks": "maksimum",
+    # General Indonesian abbreviation
+    "dlm": "dalam",
+    "sblm": "sebelum",
+    "ssdh": "sesudah",
+    "yg": "yang",
+    "krn": "karena",
+    "tdk": "tidak",
+    "gak": "tidak",
+    "ngga": "tidak",
+    "engga": "tidak",
+    "dr": "dari",
+    "dgn": "dengan",
+    "utk": "untuk",
+    "skr": "sekarang",
+    "blm": "belum",
+    "udh": "sudah",
+    "udah": "sudah",
+    "byk": "banyak",
+    "sdh": "sudah",
+    "jln": "jalan",
+    "jl": "jalan",
+    "rmh": "rumah",
+    "ortu": "orang tua",
+    "bsk": "besok",
+    "tgl": "tanggal",
+    "tgl2": "tanggal",
+}
+
+# =============================================================================
+# Location variants (Bandar Lampung area sekitar UNILA)
+# =============================================================================
+LOCATIONS: dict[str, str] = {
+    # Kecamatan & area kampus
+    "gdg meneng": "gedong meneng",
+    "gd meneng": "gedong meneng",
+    "rjbs": "rajabasa",
+    "rj basa": "rajabasa",
+    "sumbro": "sumantri brojonegoro",
+    "kdtn": "kedaton",
+    "wh": "way halim",
+    "lab ratu": "labuhan ratu",
+    "tj senang": "tanjung senang",
+    "tlb": "teluk betung",
+    "tjk": "tanjungkarang",
+    "bdl": "bandar lampung",
+    "blampung": "bandar lampung",
+    # Kampus & landmark
+    "unila": "universitas lampung",
+    "unyila": "universitas lampung",
+    "unl": "universitas lampung",
+    "fmipa": "fakultas mipa universitas lampung",
+    "ft unila": "fakultas teknik universitas lampung",
+    "fkip": "fakultas keguruan universitas lampung",
+    "feb": "fakultas ekonomi bisnis universitas lampung",
+    "uin": "uin raden intan lampung",
+    "polnep": "politeknik negeri lampung",
+    "itera": "institut teknologi sumatera",
+    "kampus a": "kampus utama",
+}
+
+# =============================================================================
+# Tipe kos & slang gender
+# =============================================================================
+TYPE_SLANG: dict[str, str] = {
+    "cowo": "putra",
+    "cowok": "putra",
+    "pria": "putra",
+    "laki": "putra",
+    "lk": "putra",
+    "cewe": "putri",
+    "cewek": "putri",
+    "wanita": "putri",
+    "perempuan": "putri",
+    "pr": "putri",
+    "mix": "campur",
+}
+
+# =============================================================================
+# Aturan kos (rules)
+# =============================================================================
+RULES: dict[str, str] = {
+    "jamal": "jam malam",
+    "jam mlm": "jam malam",
+    "no smoking": "dilarang merokok",
+    "no smoke": "dilarang merokok",
+    "free": "bebas",
+}
+
+# =============================================================================
+# Payment & harga terms
+# =============================================================================
+PAYMENT: dict[str, str] = {
+    "dp": "uang muka",
+    "uang dp": "uang muka",
+    "down payment": "uang muka",
+    "deposit": "uang jaminan",
+    "all in": "termasuk semua",
+    "all-in": "termasuk semua",
+    "include": "termasuk",
+    "exclude": "tidak termasuk",
+}
+
+# =============================================================================
+# Bentuk kos (kostan, kontrakan, dll)
+# =============================================================================
+KOS_FORM: dict[str, str] = {
+    "kostan": "kos kosan",
+    "kost-kostan": "kos kosan",
+    "kost2an": "kos kosan",
+    "kost kostan": "kos kosan",
+    "rumah kost": "kos kosan",
+    "kontrakan": "kos kosan",
+    "kos eksekutif": "kos eksklusif",
+    "ekslusive": "eksklusif",
+    "exclusive": "eksklusif",
+    "exclusif": "eksklusif",
+    "exklusif": "eksklusif",
+    "exklusive": "eksklusif",
+    "ready": "tersedia",
+    "available": "tersedia",
+    "ready stock": "tersedia",
+}
+
+# =============================================================================
+# Final merged dict
+# =============================================================================
+KOS_JARGON_DICT: dict[str, str] = {
+    **ABBREVIATIONS,
+    **LOCATIONS,
+    **TYPE_SLANG,
+    **RULES,
+    **PAYMENT,
+    **KOS_FORM,
+}
+
+
+def jargon_count() -> int:
+    """Total unique variants di KOS_JARGON_DICT."""
+    return len(KOS_JARGON_DICT)
+
+
+# Rubric requirement
+MIN_REQUIRED = 100
+
+
+if __name__ == "__main__":
+    count = jargon_count()
+    print(f"KOS_JARGON_DICT size: {count}")
+    print(f"Minimum required for rubric: {MIN_REQUIRED}")
+    if count < MIN_REQUIRED:
+        print(f"WARNING: tambah {MIN_REQUIRED - count} more entries (tim Anggota B)")
+    else:
+        print("OK: meets rubric requirement")
+    print("\nBreakdown:")
+    print(f"  ABBREVIATIONS:  {len(ABBREVIATIONS)}")
+    print(f"  LOCATIONS:      {len(LOCATIONS)}")
+    print(f"  TYPE_SLANG:     {len(TYPE_SLANG)}")
+    print(f"  RULES:          {len(RULES)}")
+    print(f"  PAYMENT:        {len(PAYMENT)}")
+    print(f"  KOS_FORM:       {len(KOS_FORM)}")

backend/app/preprocessing/normalizer.py

@@ -0,0 +1,74 @@
+"""Atomic normalization: HTML strip, whitespace, lowercase, price extraction.
+
+ORDER-SENSITIVE: `extract_prices_inline()` MUST jalan SEBELUM `lowercase()`
+karena regex match `[Rr][Pp]` (case-insensitive), tapi `Rp` capitalized lebih
+robust untuk catch edge case dimana harga tidak dipisah space.
+"""
+
+from __future__ import annotations
+
+import re
+
+from bs4 import BeautifulSoup
+
+
+def strip_html(text: str) -> str:
+    """Strip HTML tags + decode entities, leave plain text dengan space separator."""
+    if not text:
+        return ""
+    soup = BeautifulSoup(text, "lxml")
+    return soup.get_text(separator=" ", strip=True)
+
+
+def normalize_whitespace(text: str) -> str:
+    """Collapse multiple whitespace (space/tab/newline) → single space, strip."""
+    if not text:
+        return ""
+    return re.sub(r"\s+", " ", text).strip()
+
+
+def lowercase(text: str) -> str:
+    """Lowercase. Wajib jalan SETELAH `extract_prices_inline()`."""
+    return (text or "").lower()
+
+
+# =============================================================================
+# Price extraction (inline, return all matches)
+# =============================================================================
+_PRICE_PATTERNS = [
+    # Rp 1.250.000 / Rp1.250.000 / Rp 850,000
+    (re.compile(r"[Rr][Pp]\s*([\d.,]+)", re.IGNORECASE), "rupiah"),
+    # 1.5jt, 1jt, 2,3 jt
+    (re.compile(r"(\d+(?:[.,]\d+)?)\s*jt\b", re.IGNORECASE), "juta"),
+    # 500k, 500rb, 500 ribu
+    (re.compile(r"(\d+(?:[.,]\d+)?)\s*(?:k|rb|ribu)\b", re.IGNORECASE), "ribu"),
+]
+
+
+def extract_prices_inline(text: str) -> list[int]:
+    """Extract semua harga muncul di text. Return list IDR integer.
+
+    Untuk kos listing biasanya 1-2 harga (sewa bulanan + uang muka), tapi
+    return list untuk fleksibilitas.
+    """
+    if not text:
+        return []
+
+    prices: list[int] = []
+    for pattern, unit in _PRICE_PATTERNS:
+        for match in pattern.finditer(text):
+            raw = match.group(1)
+            try:
+                if unit == "rupiah":
+                    value = int(raw.replace(".", "").replace(",", ""))
+                elif unit == "juta":
+                    value = int(float(raw.replace(",", ".")) * 1_000_000)
+                elif unit == "ribu":
+                    value = int(float(raw.replace(",", ".")) * 1_000)
+                else:
+                    continue
+                if value not in prices:
+                    prices.append(value)
+            except ValueError:
+                continue
+    return prices

backend/app/preprocessing/pipeline.py

@@ -0,0 +1,187 @@
+"""Indonesian preprocessing pipeline orchestrator.
+
+Stage-based dengan toggle via PipelineConfig — tim Anggota B pakai untuk
+benchmark `BEFORE vs AFTER tiap stage` di notebook 02_preprocessing_experiment.
+
+Analogi Laravel: ini seperti chain of Middleware. Request (raw text) lewat
+chain, tiap middleware transform.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Optional
+
+from .jargon import KOS_JARGON_DICT
+from .normalizer import (
+    extract_prices_inline,
+    lowercase,
+    normalize_whitespace,
+    strip_html,
+)
+from .spelling import correct_spelling
+from .stemmer import SastrawiStemmer
+from .stopwords import StopwordRemover
+from .tokenizer import simple_tokenize
+
+
+@dataclass
+class PipelineConfig:
+    """Toggle setiap stage untuk experiment.
+
+    Tim Anggota B: pakai config ini di notebooks/02_preprocessing_experiment.ipynb
+    untuk benchmark IR metric BEFORE vs AFTER tiap stage.
+    """
+    strip_html: bool = True
+    normalize_whitespace: bool = True
+    extract_prices: bool = True  # always run — non-destructive
+    lowercase: bool = True
+    apply_jargon_dict: bool = True
+    correct_spelling: bool = True
+    tokenize: bool = True
+    remove_stopwords: bool = True
+    stem: bool = True
+    custom_stopwords: list[str] = field(default_factory=list)
+
+
+@dataclass
+class PipelineResult:
+    """Hasil preprocessing dengan tracking per-stage."""
+    raw: str
+    processed: str
+    tokens: list[str]
+    extracted_prices: list[int] = field(default_factory=list)
+    stages_applied: list[str] = field(default_factory=list)
+    # Snapshot output tiap stage (untuk visualisasi /api/preprocess).
+    # Diisi hanya kalau process(trace=True) — list of {stage, output}.
+    trace: list[dict] = field(default_factory=list)
+
+
+class PreprocessingPipeline:
+    """Indonesian preprocessing pipeline untuk kos listings.
+
+    Sastrawi factory + stopword set di-init sekali (heavy) — instance shared
+    antar dokumen. Untuk batch processing, instantiate sekali, panggil
+    `process()` per dokumen.
+    """
+
+    def __init__(
+        self,
+        config: Optional[PipelineConfig] = None,
+        jargon_dict: Optional[dict[str, str]] = None,
+    ):
+        self.config = config or PipelineConfig()
+        self.jargon_dict = jargon_dict if jargon_dict is not None else KOS_JARGON_DICT
+        # Lazy init heavy components
+        self._stemmer: Optional[SastrawiStemmer] = (
+            SastrawiStemmer() if self.config.stem else None
+        )
+        self._stopword_remover: Optional[StopwordRemover] = (
+            StopwordRemover(custom=self.config.custom_stopwords)
+            if self.config.remove_stopwords
+            else None
+        )
+        # Pre-compile jargon patterns (sorted longest-first untuk hindari
+        # "dlm" match dulu sebelum "km dlm")
+        self._jargon_patterns: list[tuple[re.Pattern[str], str]] = []
+        if self.jargon_dict:
+            sorted_variants = sorted(self.jargon_dict.keys(), key=len, reverse=True)
+            for variant in sorted_variants:
+                pattern = re.compile(
+                    r"\b" + re.escape(variant) + r"\b", re.IGNORECASE
+                )
+                self._jargon_patterns.append((pattern, self.jargon_dict[variant]))
+
+    def process(self, text: str, trace: bool = False) -> PipelineResult:
+        """Run full pipeline. Return PipelineResult dengan tracking stages.
+
+        trace=True merekam snapshot output tiap stage (untuk visualisasi
+        step-by-step di /api/preprocess); default False supaya jalur serving
+        tidak bayar overhead.
+        """
+        result = PipelineResult(raw=text, processed=text or "", tokens=[])
+        if not text:
+            return result
+
+        current = text
+
+        def snap(stage: str, output) -> None:
+            if trace:
+                result.trace.append({"stage": stage, "output": output})
+
+        # Stage 1: HTML strip
+        if self.config.strip_html:
+            current = strip_html(current)
+            result.stages_applied.append("strip_html")
+            snap("strip_html", current)
+
+        # Stage 2: Whitespace normalize
+        if self.config.normalize_whitespace:
+            current = normalize_whitespace(current)
+            result.stages_applied.append("normalize_whitespace")
+            snap("normalize_whitespace", current)
+
+        # Stage 3: Price extraction (PENTING — sebelum lowercase)
+        if self.config.extract_prices:
+            result.extracted_prices = extract_prices_inline(current)
+            result.stages_applied.append("extract_prices")
+            snap("extract_prices", result.extracted_prices)
+
+        # Stage 4: Lowercase
+        if self.config.lowercase:
+            current = lowercase(current)
+            result.stages_applied.append("lowercase")
+            snap("lowercase", current)
+
+        # Stage 5: Jargon dictionary substitution
+        if self.config.apply_jargon_dict and self._jargon_patterns:
+            current = self._apply_jargon(current)
+            result.stages_applied.append("apply_jargon_dict")
+            snap("apply_jargon_dict", current)
+
+        # Stage 6: Spelling correction
+        if self.config.correct_spelling:
+            current = correct_spelling(current)
+            result.stages_applied.append("correct_spelling")
+            snap("correct_spelling", current)
+
+        # Stage 7: Tokenize
+        if self.config.tokenize:
+            tokens = simple_tokenize(current)
+            result.stages_applied.append("tokenize")
+            snap("tokenize", list(tokens))
+        else:
+            tokens = current.split()
+
+        # Stage 8: Stopword removal
+        if self.config.remove_stopwords and self._stopword_remover:
+            tokens = self._stopword_remover.remove(tokens)
+            result.stages_applied.append("remove_stopwords")
+            snap("remove_stopwords", list(tokens))
+
+        # Stage 9: Stem
+        if self.config.stem and self._stemmer:
+            tokens = [self._stemmer.stem(tok) for tok in tokens]
+            result.stages_applied.append("stem")
+            snap("stem", list(tokens))
+
+        result.tokens = tokens
+        result.processed = " ".join(tokens)
+        return result
+
+    def _apply_jargon(self, text: str) -> str:
+        """Replace jargon variants → canonical, longest-first dengan word boundary."""
+        for pattern, canonical in self._jargon_patterns:
+            text = pattern.sub(canonical, text)
+        return text
+
+
+def preprocess(text: str, config: Optional[PipelineConfig] = None) -> str:
+    """Shorthand: run pipeline dengan config default, return processed string.
+
+    Untuk batch lebih efisien, instantiate PreprocessingPipeline sekali,
+    panggil `.process()` per dokumen.
+    """
+    pipeline = PreprocessingPipeline(config)
+    return pipeline.process(text).processed

backend/app/preprocessing/spelling.py

@@ -0,0 +1,68 @@
+"""Domain-specific spelling correction untuk kos listings.
+
+Approach: simple typo dictionary lookup (no statistical model untuk simplicity).
+Tim Anggota B: tambah typo umum yang ditemukan saat scrape data.
+
+Alternative future enhancement (Week 3 kalau ada waktu):
+- Edit distance (Levenshtein) untuk catch typos baru
+- Bigram language model untuk context-aware correction
+- Library: pyspellchecker, atau custom n-gram model
+"""
+
+from __future__ import annotations
+
+import re
+
+
+# Common typos di kos listings (curated baseline)
+TYPO_FIXES: dict[str, str] = {
+    # Fasilitas typos
+    "fasilitsa": "fasilitas",
+    "fasiltas": "fasilitas",
+    "fasilitas2": "fasilitas",
+    "fasilitas-fasilitas": "fasilitas",
+    # Lokasi typos
+    "strategis": "strategis",  # canonical (kept untuk dokumentasi)
+    "stratejis": "strategis",
+    "lokasinyy": "lokasinya",
+    "lokasinya2": "lokasinya",
+    # Eksklusif variants (sering muncul di judul listing + query)
+    "ekslusif": "eksklusif",
+    "ekslusive": "eksklusif",
+    "exclusive": "eksklusif",
+    "exlusive": "eksklusif",
+    # Kondisi
+    "bagys": "bagus",
+    "barangny": "barangnya",
+    "rapih": "rapi",
+    "luas2": "luas",
+    "bersih2": "bersih",
+    # Bahasa gaul yang sering muncul
+    "pny": "punya",
+    "pnya": "punya",
+    "ada2": "ada",
+}
+
+
+# Filter out self-mappings (no-op) saat build pattern
+_REAL_FIXES = {variant: fix for variant, fix in TYPO_FIXES.items() if variant != fix}
+
+
+# Pre-compile patterns sekali (mahal kalau per-call)
+_TYPO_PATTERNS: list[tuple[re.Pattern[str], str]] = [
+    (re.compile(r"\b" + re.escape(typo) + r"\b", re.IGNORECASE), fix)
+    for typo, fix in _REAL_FIXES.items()
+]
+
+
+def correct_spelling(text: str) -> str:
+    """Replace common typos dengan bentuk standar (word-boundary regex)."""
+    if not text:
+        return text
+    for pattern, fix in _TYPO_PATTERNS:
+        text = pattern.sub(fix, text)
+    return text
+
+
+if __name__ == "__main__":
+    print(f"TYPO_FIXES: {len(TYPO_FIXES)} total, {len(_REAL_FIXES)} real fixes")

backend/app/preprocessing/stemmer.py

@@ -0,0 +1,41 @@
+"""Sastrawi stemmer wrapper dengan LRU cache.
+
+Sastrawi stemming bisa lambat untuk corpus besar (3000+ dokumen × N tokens) —
+LRU cache di-method `stem()` mempercepat repeated tokens.
+
+Factory di-init sekali (heavy, ~100ms), instance shared antar dokumen.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
+
+
+class SastrawiStemmer:
+    """Wrapper Sastrawi `StemmerFactory().create_stemmer()` dengan LRU cache.
+
+    Cache size 10k tokens cukup untuk corpus 3000 listings (typical
+    vocabulary unique tokens ~5-8k setelah preprocessing).
+    """
+
+    def __init__(self, cache_size: int = 10_000):
+        factory = StemmerFactory()
+        self._stemmer = factory.create_stemmer()
+        # Wrap stem method dengan LRU cache
+        self.stem = lru_cache(maxsize=cache_size)(self._stem_uncached)
+
+    def _stem_uncached(self, word: str) -> str:
+        """Stem single word tanpa cache (called via cached wrapper)."""
+        if not word:
+            return word
+        return self._stemmer.stem(word)
+
+    def stem_tokens(self, tokens: list[str]) -> list[str]:
+        """Stem list of tokens (gunakan cache per-token)."""
+        return [self.stem(tok) for tok in tokens]
+
+    def cache_info(self):
+        """Debug: LRU cache hit/miss stats."""
+        return self.stem.cache_info()  # type: ignore[attr-defined]

backend/app/preprocessing/stopwords.py

@@ -0,0 +1,68 @@
+"""Indonesian stopword removal: Sastrawi default + custom domain stopwords.
+
+Custom stopwords penting untuk domain kos — kata yang terlalu sering muncul
+turun jadi noise (low IR signal). Tim Anggota B: setelah Anggota A scrape,
+jalanin frequency analysis di `notebooks/01_eda.ipynb`:
+
+    from collections import Counter
+    all_tokens = [tok for listing in corpus for tok in listing['deskripsi'].split()]
+    Counter(all_tokens).most_common(50)
+
+Top-50 paling sering biasanya stopword domain — tambah ke
+`DEFAULT_CUSTOM_STOPWORDS` kalau memang noise (bukan informative term).
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory
+
+
+# Domain stopwords baseline (kos-specific). Tim Anggota B: extend dari EDA.
+DEFAULT_CUSTOM_STOPWORDS: list[str] = [
+    # Form kos
+    "kos", "kost", "kosan", "kostan", "kos-kosan",
+    # Kata generic listing
+    "kamar",
+    "sewa", "harga",
+    "tersedia", "ready",
+    "info", "informasi",
+    "hubungi", "kontak",
+    # Kata penghubung tambahan (Sastrawi sudah handle, tapi extra safety)
+    # "untuk", "yang", "ke", "di"  # already di Sastrawi default
+]
+
+
+class StopwordRemover:
+    """Wrapper Sastrawi StopWordRemover + custom set."""
+
+    def __init__(
+        self,
+        custom: Optional[list[str]] = None,
+        use_sastrawi_default: bool = True,
+    ):
+        if use_sastrawi_default:
+            factory = StopWordRemoverFactory()
+            self.sastrawi_stopwords: set[str] = set(factory.get_stop_words())
+        else:
+            self.sastrawi_stopwords = set()
+
+        self.custom_stopwords: set[str] = set(custom or DEFAULT_CUSTOM_STOPWORDS)
+        self.all_stopwords: set[str] = (
+            self.sastrawi_stopwords | self.custom_stopwords
+        )
+
+    def remove(self, tokens: list[str]) -> list[str]:
+        """Filter tokens — drop yang termasuk stopword."""
+        return [tok for tok in tokens if tok.lower() not in self.all_stopwords]
+
+    def is_stopword(self, word: str) -> bool:
+        return word.lower() in self.all_stopwords
+
+    def count(self) -> dict[str, int]:
+        return {
+            "sastrawi": len(self.sastrawi_stopwords),
+            "custom": len(self.custom_stopwords),
+            "total": len(self.all_stopwords),
+        }