feat: add scholarship metadata export and fit scores to API

- Add scholarship_metadata.npy output alongside embeddings in config
- Export metadata during precompute_text_embeddings.py using _build_scholarship_metadata helper
- Save metadata during export_embeddings.py for serving pipeline
- Add fit_scores field to ScholarshipResult model with academic, leadership, and language alignment scores
- Update README docs with python -m script invocation examples

README.md

@@ -113,21 +113,21 @@ python scripts/precompute_text_embeddings.py # or python -m scripts.precompute_t
 python scripts/train.py --config configs/default.yaml # or python -m scripts.train --config configs/default.yaml
 
 # Step 3 — Evaluasi pada test set (checkpoint paths default to configs/default.yaml)
-python scripts/evaluate.py \
+python scripts/evaluate.py \  # or python -m scripts.evaluate \
   --config configs/default.yaml
 
-# Override checkpoint paths if needed:
-python scripts/evaluate.py \
+# or for Step 3 - Override checkpoint paths if needed:
+python scripts/evaluate.py \  # or python -m scripts.evaluate \ 
   --config configs/default.yaml \
   --student_checkpoint outputs/checkpoints/student_tower_best.keras \
   --scholarship_checkpoint outputs/checkpoints/scholarship_tower_best.keras
 
 # Step 4 — Export scholarship embeddings untuk serving (checkpoint path defaults to config)
-python scripts/export_embeddings.py \
+python scripts/export_embeddings.py \  # or python -m scripts.export_embeddings \
   --config configs/default.yaml
 
-# Override checkpoint path if needed:
-python scripts/export_embeddings.py \
+# or for Step 4 - Override checkpoint path if needed:
+python scripts/export_embeddings.py \  # or python -m scripts.export_embeddings \
   --scholarship_checkpoint outputs/checkpoints/scholarship_tower_best.keras
 ```
 

configs/default.yaml

@@ -68,6 +68,7 @@ hf:
 embeddings:
   scholarship_emb: outputs/embeddings/scholarship_emb.npy
   scholarship_ids: outputs/embeddings/scholarship_ids.npy
+  scholarship_metadata: outputs/embeddings/scholarship_metadata.npy
 
 tensorboard:
   enabled: true

scripts/export_embeddings.py

@@ -14,9 +14,11 @@ import os
 import numpy as np
 import yaml
 import tensorflow as tf
+import pandas as pd
 
 from src.models.student_tower import L2Normalize
 from src.utils.data_loader import load_precomputed_features
+from src.serving.helpers import _build_scholarship_metadata
 
 
 def parse_args():
@@ -65,8 +67,14 @@ def main():
     np.save(cfg["embeddings"]["scholarship_emb"], sch_emb)
     np.save(cfg["embeddings"]["scholarship_ids"], np.array(sch_ids, dtype=object))
 
+    # Build and save metadata alongside embeddings
+    scholarships_df = pd.read_csv(f"{cfg['data']['raw_path']}/scholarships.csv")
+    metadata = _build_scholarship_metadata(scholarships_df)
+    np.save(cfg["embeddings"]["scholarship_metadata"], np.array(metadata, dtype=object))
+
     print(f"\nSaved: {cfg['embeddings']['scholarship_emb']}")
     print(f"Saved: {cfg['embeddings']['scholarship_ids']}")
+    print(f"Saved: {cfg['embeddings']['scholarship_metadata']}")
 
 
 if __name__ == "__main__":

scripts/precompute_text_embeddings.py

@@ -13,8 +13,10 @@ import argparse
 import os
 import numpy as np
 import pandas as pd
+import yaml
 
 from src.utils.feature_engineering import encode_text
+from src.serving.helpers import _build_scholarship_metadata
 
 
 def parse_args():
@@ -88,9 +90,17 @@ def main():
     np.save(SCH_IDS_PATH, np.array(scholarship_ids, dtype=object))
     print(f"Saved: {SCH_IDS_PATH}  ({len(scholarship_ids)} IDs)")
 
+    # ── Scholarship Metadata ────────────────────────────────────────────────
+    SCH_META_PATH = cfg["embeddings"]["scholarship_metadata"]
+    metadata = _build_scholarship_metadata(scholarships_df)
+    np.save(SCH_META_PATH, np.array(metadata, dtype=object))
+    print(f"Saved: {SCH_META_PATH}  ({len(metadata)} entries)")
+
     print("\nDone. Shapes:")
     print(f"  students.npy    : {student_text_emb.shape}")
     print(f"  scholarships.npy: {scholarship_text_emb.shape}")
+    print(f"  scholarship_metadata.npy: {len(metadata)} entries")
+
 
 
 if __name__ == "__main__":

src/serving/api.py

@@ -201,6 +201,10 @@ class ScholarshipResult(BaseModel):
         description="Personalized recommendation explaining why this scholarship is a match",
     )
     metadata: dict
+    fit_scores: Optional[dict] = Field(
+        default=None,
+        description="Fit scores for academic, leadership, and language alignment (0-1 scale)",
+    )
 
 
 class RecommendationResponse(BaseModel):

src/serving/helpers.py

@@ -57,23 +57,76 @@ def _parse_csv_with_json(csv_text: str, json_cols: list[str]) -> pd.DataFrame:
 
 
 def _build_scholarship_metadata(df) -> list[dict]:
-    """Extract lightweight metadata from scholarship DataFrame for API responses."""
+    """Extract enriched metadata from scholarship DataFrame for API responses.
+
+    Includes fields useful for LLM-based recommendation generation:
+    - Core identity (id, name)
+    - Context (mission_statement, selection_criteria)
+    - Location & funding info
+    """
     metadata = []
     for _, row in df.iterrows():
+        # Build human-readable funding summary from individual boolean fields
+        funding_parts = []
+        if row.get("funding_covers_tuition"):
+            funding_parts.append("tuition")
+        if row.get("funding_covers_living"):
+            funding_parts.append("living expenses")
+        if row.get("funding_covers_airfare"):
+            funding_parts.append("airfare")
+        if row.get("funding_covers_insurance"):
+            funding_parts.append("insurance")
+        if funding_parts:
+            funding_summary = f"Covers {', '.join(funding_parts)}"
+        else:
+            funding_summary = "No specified funding coverage"
+
+        # Add monthly stipend info if available
+        stipend = row.get("funding_monthly_stipend")
+        if stipend and float(stipend) > 0:
+            funding_summary += f", plus ${float(stipend):,.0f}/month stipend"
+
         metadata.append({
             "scholarship_id": row.get("scholarship_id"),
+            "name": row.get("name"),
+            "mission_statement": row.get("mission_statement"),
+            "selection_criteria": row.get("selection_criteria"),
             "host_country": row.get("host_country"),
             "host_region": row.get("host_region"),
             "funding_is_full_funding": bool(row.get("funding_is_full_funding", False)),
+            "funding_coverage_summary": funding_summary,
         })
     return metadata
 
 
 def _scholarship_to_metadata(sch: dict) -> dict:
-    """Convert scholarship dict to lightweight metadata for API responses."""
+    """Convert scholarship dict to enriched metadata for API responses."""
+    # Build human-readable funding summary
+    funding_parts = []
+    if sch.get("funding_covers_tuition"):
+        funding_parts.append("tuition")
+    if sch.get("funding_covers_living"):
+        funding_parts.append("living expenses")
+    if sch.get("funding_covers_airfare"):
+        funding_parts.append("airfare")
+    if sch.get("funding_covers_insurance"):
+        funding_parts.append("insurance")
+    if funding_parts:
+        funding_summary = f"Covers {', '.join(funding_parts)}"
+    else:
+        funding_summary = "No specified funding coverage"
+
+    stipend = sch.get("funding_monthly_stipend")
+    if stipend and float(stipend) > 0:
+        funding_summary += f", plus ${float(stipend):,.0f}/month stipend"
+
     return {
         "scholarship_id": sch.get("scholarship_id"),
+        "name": sch.get("name"),
+        "mission_statement": sch.get("mission_statement"),
+        "selection_criteria": sch.get("selection_criteria"),
         "host_country": sch.get("host_country"),
         "host_region": sch.get("host_region"),
         "funding_is_full_funding": bool(sch.get("funding_is_full_funding", False)),
-    }
+        "funding_coverage_summary": funding_summary,
+    }

src/serving/inference_engine.py

@@ -541,6 +541,7 @@ class InferenceEngine:
         """
         sch_emb_path = self.cfg["embeddings"]["scholarship_emb"]
         sch_ids_path = self.cfg["embeddings"]["scholarship_ids"]
+        sch_meta_path = self.cfg["embeddings"]["scholarship_metadata"]
 
         if not (os.path.exists(sch_emb_path) and os.path.exists(sch_ids_path)):
             _print("  No cached embeddings found — will recompute from CSVs.")
@@ -549,18 +550,48 @@ class InferenceEngine:
         try:
             self._sch_emb = np.load(sch_emb_path, allow_pickle=False)
             self._sch_ids = list(np.load(sch_ids_path, allow_pickle=True).tolist())
-            self._sch_metadata = []  # Will be built lazily from CSV if needed
 
-            # Build metadata from the saved IDs (minimal — just enough for API)
-            self._sch_metadata = [{"scholarship_id": sid} for sid in self._sch_ids]
+            # Load enriched metadata (name, mission_statement, selection_criteria, etc.)
+            if os.path.exists(sch_meta_path):
+                self._sch_metadata = list(
+                    np.load(sch_meta_path, allow_pickle=True).tolist()
+                )
+                _print(f"Loaded cached embeddings: {len(self._sch_emb)} scholarships "
+                       f"(shape {self._sch_emb.shape}, metadata: {len(self._sch_metadata)} entries)")
+
+                # Normalize JSON columns in metadata to ensure consistency with /refresh path
+                self._sch_metadata = self._normalize_cached_metadata()
+            else:
+                # Fallback: build minimal metadata from saved IDs if metadata file missing
+                self._sch_metadata = [{"scholarship_id": sid} for sid in self._sch_ids]
+                _print(f"Loaded cached embeddings: {len(self._sch_emb)} scholarships "
+                       f"(shape {self._sch_emb.shape}, minimal metadata — metadata.npy not found)")
 
-            _print(f"Loaded cached embeddings: {len(self._sch_emb)} scholarships "
-                   f"(shape {self._sch_emb.shape})")
             return True
         except Exception as e:
             print(f"  Failed to load cached embeddings: {e}", file=sys.stderr, flush=True)
             return False
 
+    def _normalize_cached_metadata(self) -> list[dict]:
+        """Normalize JSON columns in cached metadata to match /refresh output.
+
+        Reconstructs a DataFrame from the cached IDs, loads fresh CSV data,
+        normalizes JSON columns (selection_criteria, language_requirements),
+        and rebuilds metadata using _build_scholarship_metadata().
+        This ensures consistency between cold-boot and /refresh paths.
+        """
+        # Load fresh scholarships CSV to get properly parsed JSON columns
+        raw_path = self.cfg["data"]["raw_path"]
+        try:
+            scholarships_df = pd.read_csv(f"{raw_path}/scholarships.csv")
+            scholarships_df = self._normalize_json_columns(scholarships_df, SCHOLARSHIP_JSON_COLS)
+
+            # Rebuild metadata from normalized data
+            return _build_scholarship_metadata(scholarships_df)
+        except Exception:
+            # If we can't normalize, return original metadata as-is
+            return self._sch_metadata
+
     def initialize(self):
         """Load both towers and build initial scholarship embedding cache.
 
@@ -649,10 +680,20 @@ class InferenceEngine:
             sch_feat, training=False
         ).numpy()  # (N, 128) L2-normalized
 
-        # Build metadata for API responses
+        # Build metadata for API responses and persist to disk
         self._sch_ids = scholarships_df["scholarship_id"].tolist()
         self._sch_metadata = _build_scholarship_metadata(scholarships_df)
 
+        # Persist metadata to disk so subsequent cold boots have enriched data
+        sch_meta_path = self.cfg["embeddings"]["scholarship_metadata"]
+        np.save(sch_meta_path, np.array(self._sch_metadata, dtype=object))
+
+        # Persist embeddings + IDs to disk (so they survive server restart)
+        sch_emb_path = self.cfg["embeddings"]["scholarship_emb"]
+        sch_ids_path = self.cfg["embeddings"]["scholarship_ids"]
+        np.save(sch_emb_path, self._sch_emb)
+        np.save(sch_ids_path, np.array(self._sch_ids, dtype=object))
+
         print(f"Scholarship cache refreshed: {len(self._sch_ids)} scholarships "
               f"(embedding shape {self._sch_emb.shape})")