feat: implement hybrid similarity ranking engine with dynamic weighting and originality scoring

src/similarity_model/hybrid_ranker.py

@@ -1,4 +1,5 @@
 import logging
+import math
 from typing import List, Dict, Any
 
 import pandas as pd
@@ -140,24 +141,37 @@ def get_baseline_similarity():
 
 def compute_originality(
     hybrid_score: float,
-    unique_query_features: int,
-    total_query_features: int
+    unique_query_features: int = 0,
+    total_query_features: int = 0
 ) -> float:
     """
     Originality Score (0-100).
-    Base: (1 - calibrated_similarity) * 100.
+
+    Uses a shifted sigmoid calibration so that moderate similarity
+    (0.40-0.60) triggers strong originality penalties, while truly
+    novel projects (sim < 0.30) retain high originality.
+
+    Sigmoid parameters:
+        k        = 14    (steepness of the drop-off)
+        midpoint = 0.27  (calibrated similarity where originality ≈ 50%)
     """
+    SIGMOID_K = 14
+    SIGMOID_MIDPOINT = 0.27
+
     hybrid_score = clamp(hybrid_score)
     baseline_sim = get_baseline_similarity()
-    
-    # Subtraction and Min-Max scaling
-    calibrated_similarity = max(0.0, (hybrid_score - baseline_sim) / (1.0 - baseline_sim))
-    originality = 100.0 * (1.0 - calibrated_similarity)
-
-    # Only apply uniqueness bonus when feature comparison was meaningful
-    if total_query_features > 0 and unique_query_features < total_query_features:
-        uniqueness_ratio = unique_query_features / total_query_features
-        originality = min(100.0, originality + (uniqueness_ratio * 10.0))
+
+    # Subtraction and Min-Max scaling (unchanged)
+    calibrated_similarity = max(
+        0.0, (hybrid_score - baseline_sim) / (1.0 - baseline_sim)
+    )
+
+    # Sigmoid mapping: converts calibrated_similarity → [0, 1]
+    sigmoid_output = 1.0 / (
+        1.0 + math.exp(-SIGMOID_K * (calibrated_similarity - SIGMOID_MIDPOINT))
+    )
+
+    originality = 100.0 * (1.0 - sigmoid_output)
 
     return round(max(0.0, min(100.0, originality)), 2)
 

src/similarity_model/similarity_engine.py

@@ -1,5 +1,6 @@
 import logging
 from typing import Dict, Any, List, Optional
+from functools import lru_cache
 
 import pandas as pd
 
@@ -19,6 +20,21 @@ from src.similarity_model import (
     risk_label
 )
 
+# ---------------------------------------------------------------------------
+# Cross-encoder for paraphrase detection (lazy-loaded, cached)
+# ---------------------------------------------------------------------------
+@lru_cache(maxsize=1)
+def _load_cross_encoder():
+    from sentence_transformers import CrossEncoder
+    logger.info("Loading cross-encoder: cross-encoder/stsb-distilroberta-base")
+    return CrossEncoder("cross-encoder/stsb-distilroberta-base", max_length=512)
+
+CROSS_ENCODER_THRESHOLD = 0.60   # minimum cross-score to trigger boost
+CROSS_ENCODER_MAX_BOOST = 0.30   # maximum hybrid_score boost from cross-encoder
+WORKFLOW_COVERAGE_THRESH = 0.50  # minimum coverage to trigger workflow penalty
+WORKFLOW_FEATURE_THRESH  = 0.45  # minimum feature_score to trigger workflow penalty
+WORKFLOW_MAX_BOOST       = 0.10  # maximum hybrid_score boost from workflow overlap
+
 logging.basicConfig(
     level=logging.INFO,
     format="%(asctime)s | %(levelname)s | %(message)s"
@@ -404,7 +420,74 @@ def find_similar_projects(
         ascending=False
     ).reset_index(drop=True)
 
-    # Decaying aggregation over Top-5
+    # -----------------------------------------------------------------
+    # CROSS-ENCODER RE-SCORING (top-1 candidate only)
+    # -----------------------------------------------------------------
+    if len(final_df) > 0:
+        top_row = final_df.iloc[0]
+        candidate_id = int(top_row["project_id"])
+        candidate_row = df.loc[candidate_id]
+
+        # Build full texts for cross-encoder comparison
+        query_full = build_raw_text(
+            title=title, abstract=abstract, description=description
+        )
+        candidate_full = build_raw_text(
+            title=str(candidate_row.get(TITLE_COL, "")),
+            abstract=str(candidate_row.get("abstract", "")),
+            description=str(candidate_row.get("description", ""))
+        )
+
+        try:
+            cross_encoder = _load_cross_encoder()
+            cross_score = float(
+                cross_encoder.predict([(query_full, candidate_full)])[0]
+            )
+            # stsb model already outputs [0, 1] — clamp for safety
+            cross_score = max(0.0, min(1.0, cross_score))
+            logger.info(
+                f"Cross-encoder score (top-1): {cross_score:.4f}"
+            )
+        except Exception as exc:
+            logger.warning(f"Cross-encoder failed, skipping: {exc}")
+            cross_score = 0.0
+
+        # Apply cross-encoder boost if threshold met
+        if cross_score >= CROSS_ENCODER_THRESHOLD:
+            boost = CROSS_ENCODER_MAX_BOOST * (
+                (cross_score - CROSS_ENCODER_THRESHOLD)
+                / (1.0 - CROSS_ENCODER_THRESHOLD)
+            )
+            original_hybrid = float(final_df.loc[0, "hybrid_score"])
+            boosted_hybrid = min(1.0, original_hybrid + boost)
+            final_df.loc[0, "hybrid_score"] = round(boosted_hybrid, 4)
+            logger.info(
+                f"Cross-encoder boost: {original_hybrid:.4f} -> {boosted_hybrid:.4f} "
+                f"(+{boost:.4f})"
+            )
+
+        # -----------------------------------------------------------------
+        # WORKFLOW OVERLAP PENALTY
+        # -----------------------------------------------------------------
+        top_coverage = float(top_row.get("coverage", 0.0))
+        top_feat_score = float(top_row.get("feature_score", 0.0))
+
+        if (top_coverage >= WORKFLOW_COVERAGE_THRESH
+                and top_feat_score >= WORKFLOW_FEATURE_THRESH):
+            workflow_boost = (
+                WORKFLOW_MAX_BOOST * top_coverage * top_feat_score
+            )
+            current_hybrid = float(final_df.loc[0, "hybrid_score"])
+            boosted_hybrid = min(1.0, current_hybrid + workflow_boost)
+            final_df.loc[0, "hybrid_score"] = round(boosted_hybrid, 4)
+            logger.info(
+                f"Workflow overlap boost: {current_hybrid:.4f} -> "
+                f"{boosted_hybrid:.4f} (+{workflow_boost:.4f})"
+            )
+
+    # -----------------------------------------------------------------
+    # DECAYING AGGREGATION over Top-5
+    # -----------------------------------------------------------------
     K_val = min(5, len(final_df))
     if K_val > 0:
         s1 = float(final_df.loc[0, "hybrid_score"])
@@ -414,22 +497,16 @@ def find_similar_projects(
         for i in range(1, K_val):
             si = float(final_df.loc[i, "hybrid_score"])
             density_penalty += np.exp(-lam * i) * si
-        
+
         aggregated_score = min(1.0, s1 + beta * density_penalty)
-        
+
         # Recalculate originality based on aggregated similarity score
-        top_row = final_df.iloc[0]
-        unique_q_feats = top_row.get("unique_query_features", [])
-        total_q_feats = len(query_project[FEATURE_COL])
-        
         aggregated_originality = compute_originality(
-            hybrid_score=aggregated_score,
-            unique_query_features=len(unique_q_feats),
-            total_query_features=total_q_feats
+            hybrid_score=aggregated_score
         )
         if aggregated_score >= 0.90:
             aggregated_originality = 0.0
-            
+
         final_df.loc[0, "originality_score"] = aggregated_originality
     else:
         aggregated_score = 0.0