Sync from GitHub (tests passed)

app/ai_engine.py

@@ -9,6 +9,7 @@ Sentiment Analysis:
 Usage:
     python -m app.ai_engine --run-all --target-symbol HG=F
     python -m app.ai_engine --score-only
+    python -m app.ai_engine --refresh-sentiment
     python -m app.ai_engine --train-only --target-symbol HG=F
 """
 
@@ -45,7 +46,7 @@ logger = logging.getLogger(__name__)
 _FINBERT_OUTPUT_LOGGED = False
 _FINBERT_MISSING_LABELS_WARNED = False
 
-HYBRID_SCORING_VERSION = "hybrid_v1"
+HYBRID_SCORING_VERSION = "hybrid_v2"
 HYBRID_FALLBACK_429_MODEL_NAME = "hybrid_fallback_429"
 HYBRID_FALLBACK_PARSE_MODEL_NAME = "hybrid_fallback_parse"
 LLM_LABELS = {"BULLISH", "BEARISH", "NEUTRAL"}
@@ -328,31 +329,72 @@ def _build_hybrid_reasoning_payload(
     label: str,
     llm_confidence: float,
     finbert_strength: float,
+    finbert_polarity: float,
     llm_reasoning: str,
     llm_model: str,
+    soft_neutral_applied: bool = False,
 ) -> str:
     payload = {
         "label": label,
         "llm_confidence": round(max(0.0, min(1.0, llm_confidence)), 4),
         "finbert_strength": round(max(0.0, min(1.0, finbert_strength)), 4),
+        "finbert_polarity": round(max(-1.0, min(1.0, finbert_polarity)), 4),
         "llm_reasoning": _sanitize_reasoning_text(llm_reasoning),
         "llm_model": llm_model,
+        "soft_neutral_applied": bool(soft_neutral_applied),
         "scoring_version": HYBRID_SCORING_VERSION,
     }
     return json.dumps(payload, ensure_ascii=True)
 
 
-def _compute_hybrid_score(*, label: str, llm_confidence: float, finbert_strength: float) -> float:
-    """Compute final hybrid impact score in [-1, 1] with hard neutral rule."""
+def _compute_hybrid_score(
+    *,
+    label: str,
+    llm_confidence: float,
+    finbert_strength: float,
+    finbert_polarity: Optional[float] = None,
+    non_neutral_boost: float = 1.35,
+    soft_neutral_polarity_threshold: float = 0.12,
+    soft_neutral_max_mag: float = 0.25,
+    soft_neutral_scale: float = 0.8,
+    return_metadata: bool = False,
+) -> float | tuple[float, bool]:
+    """Compute final hybrid impact score in [-1, 1] with boosted non-neutral and soft-neutral rules."""
     normalized_label = str(label).upper().strip()
-    if normalized_label == "NEUTRAL":
-        return 0.0
+    if normalized_label not in LLM_LABELS:
+        normalized_label = "NEUTRAL"
 
-    sign = 1.0 if normalized_label == "BULLISH" else -1.0
     confidence = max(0.0, min(1.0, float(llm_confidence)))
     strength = max(0.0, min(1.0, float(finbert_strength)))
-    magnitude = max(0.0, min(1.0, (0.7 * confidence) + (0.3 * strength)))
-    return sign * magnitude
+    polarity_value = float(finbert_polarity) if finbert_polarity is not None else 0.0
+    polarity = max(-1.0, min(1.0, polarity_value))
+    soft_neutral_applied = False
+
+    if normalized_label == "NEUTRAL":
+        abs_polarity = abs(polarity)
+        if abs_polarity < max(0.0, float(soft_neutral_polarity_threshold)):
+            final_score = 0.0
+        else:
+            neutral_core = (0.6 * abs_polarity) + (0.4 * strength)
+            neutral_mag = min(
+                max(0.0, float(soft_neutral_max_mag)),
+                neutral_core * max(0.0, float(soft_neutral_scale)),
+            )
+            sign = 1.0 if polarity > 0 else -1.0
+            final_score = sign * neutral_mag
+            soft_neutral_applied = True
+
+        if return_metadata:
+            return final_score, soft_neutral_applied
+        return final_score
+
+    sign = 1.0 if normalized_label == "BULLISH" else -1.0
+    base_mag = max(0.0, min(1.0, (0.7 * confidence) + (0.3 * strength)))
+    boosted_mag = min(1.0, base_mag * max(0.0, float(non_neutral_boost)))
+    final_score = sign * boosted_mag
+    if return_metadata:
+        return final_score, soft_neutral_applied
+    return final_score
 
 
 def _extract_chat_message_content(data: dict[str, Any]) -> str:
@@ -660,8 +702,8 @@ def score_unscored_articles(
     Strategy:
     - Primary direction: OpenRouter LLM label + confidence
     - Intensity: FinBERT probabilities for every article
-    - Final score: sign(LLM label) * (0.7*llm_conf + 0.3*finbert_strength)
-    - Hard neutral rule: LLM NEUTRAL always maps to final score 0
+    - Non-neutral boost: (0.7*llm_conf + 0.3*finbert_strength) * boost
+    - Soft neutral: NEUTRAL labels can emit small directional score from FinBERT polarity
     - Chunk size: 12 articles for lower free-tier rate-limit pressure
     - Run budget: cap LLM-scored articles per run, overflow uses FinBERT
     
@@ -686,6 +728,12 @@ def score_unscored_articles(
     total_chunks = (len(unscored) + chunk_size - 1) // chunk_size
     llm_model = settings.resolved_scoring_model
     llm_budget_remaining = max(0, settings.max_llm_articles_per_run)
+    non_neutral_boost = float(getattr(settings, "sentiment_non_neutral_boost", 1.35))
+    soft_neutral_polarity_threshold = float(
+        getattr(settings, "sentiment_soft_neutral_polarity_threshold", 0.12)
+    )
+    soft_neutral_max_mag = float(getattr(settings, "sentiment_soft_neutral_max_mag", 0.25))
+    soft_neutral_scale = float(getattr(settings, "sentiment_soft_neutral_scale", 0.8))
     budget_exhausted_logged = False
     logger.info("LLM scoring budget for this run: %s articles", llm_budget_remaining)
     llm_success = 0
@@ -712,7 +760,7 @@ def score_unscored_articles(
             non_llm_candidates = chunk
             if settings.openrouter_api_key and llm_budget_remaining <= 0 and not budget_exhausted_logged:
                 logger.info(
-                    "LLM budget exhausted (%s articles). Remaining chunks use neutral-hard fallback with FinBERT probs.",
+                    "LLM budget exhausted (%s articles). Remaining chunks use soft-neutral FinBERT fallback.",
                     settings.max_llm_articles_per_run,
                 )
                 budget_exhausted_logged = True
@@ -801,24 +849,33 @@ def score_unscored_articles(
                 label = "NEUTRAL"
 
             llm_confidence = float(llm_result.get("llm_confidence", 0.0))
+            finbert_polarity = float(finbert["prob_positive"]) - float(finbert["prob_negative"])
             finbert_strength = float(
                 finbert.get(
                     "finbert_strength",
-                    abs(float(finbert["prob_positive"]) - float(finbert["prob_negative"])),
+                    abs(finbert_polarity),
                 )
             )
-            final_score = _compute_hybrid_score(
+            final_score, soft_neutral_applied = _compute_hybrid_score(
                 label=label,
                 llm_confidence=llm_confidence,
                 finbert_strength=finbert_strength,
+                finbert_polarity=finbert_polarity,
+                non_neutral_boost=non_neutral_boost,
+                soft_neutral_polarity_threshold=soft_neutral_polarity_threshold,
+                soft_neutral_max_mag=soft_neutral_max_mag,
+                soft_neutral_scale=soft_neutral_scale,
+                return_metadata=True,
             )
 
             reasoning_payload = _build_hybrid_reasoning_payload(
                 label=label,
                 llm_confidence=llm_confidence,
                 finbert_strength=finbert_strength,
+                finbert_polarity=finbert_polarity,
                 llm_reasoning=llm_result.get("llm_reasoning", ""),
                 llm_model=llm_result.get("llm_model", llm_model),
+                soft_neutral_applied=soft_neutral_applied,
             )
             
             sentiment = NewsSentiment(
@@ -1364,6 +1421,11 @@ def main():
         action="store_true",
         help="Only run XGBoost training"
     )
+    parser.add_argument(
+        "--refresh-sentiment",
+        action="store_true",
+        help="Run sentiment scoring + daily aggregation (no training)"
+    )
     parser.add_argument(
         "--target-symbol",
         type=str,
@@ -1387,8 +1449,8 @@ def main():
         logging.getLogger().setLevel(logging.DEBUG)
     
     # Determine what to run
-    score = args.run_all or args.score_only
-    aggregate = args.run_all or args.aggregate_only
+    score = args.run_all or args.score_only or args.refresh_sentiment
+    aggregate = args.run_all or args.aggregate_only or args.refresh_sentiment
     train = args.run_all or args.train_only
     
     if not (score or aggregate or train):

app/inference.py

@@ -263,6 +263,74 @@ def get_sentiment_label(sentiment_index: float) -> str:
         return "Neutral"
 
 
+def _sign(value: float) -> int:
+    """Return numeric sign (-1, 0, 1)."""
+    if value > 0:
+        return 1
+    if value < 0:
+        return -1
+    return 0
+
+
+def _clamp(value: float, lower: float, upper: float) -> float:
+    """Clamp value to [lower, upper]."""
+    return max(lower, min(upper, value))
+
+
+def _apply_sentiment_adjustment(
+    raw_predicted_return: float,
+    sentiment_index: float,
+    news_count_7d: int,
+) -> tuple[float, float, bool, bool]:
+    """
+    Apply aggressive-but-capped sentiment multiplier to raw predicted return.
+
+    Returns:
+        (adjusted_return, multiplier, adjustment_applied, capped)
+    """
+    settings = get_settings()
+    news_ref = max(1.0, float(settings.inference_sentiment_news_ref))
+    power_ref = max(1e-6, float(settings.inference_sentiment_power_ref))
+    news_floor = max(1, int(round(news_ref * 0.4)))  # default: 12 when ref is 30
+
+    news_intensity = min(1.0, max(0.0, float(news_count_7d) / news_ref))
+    sentiment_power = float(np.tanh(abs(float(sentiment_index)) / power_ref))
+
+    raw_sign = _sign(float(raw_predicted_return))
+    sentiment_sign = _sign(float(sentiment_index))
+    direction = 1.0 if raw_sign == 0 or raw_sign == sentiment_sign else -1.0
+
+    multiplier = 1.0 + (direction * sentiment_power * news_intensity)
+    multiplier = _clamp(
+        multiplier,
+        float(settings.inference_sentiment_multiplier_min),
+        float(settings.inference_sentiment_multiplier_max),
+    )
+
+    use_tiny_floor = (
+        abs(float(raw_predicted_return)) < float(settings.inference_tiny_signal_threshold)
+        and abs(float(sentiment_index)) >= power_ref
+        and int(news_count_7d) >= news_floor
+    )
+
+    if use_tiny_floor:
+        adjusted_return = float(sentiment_sign) * float(settings.inference_tiny_signal_floor)
+    else:
+        adjusted_return = float(raw_predicted_return) * multiplier
+
+    cap = abs(float(settings.inference_return_cap))
+    capped = False
+    if adjusted_return > cap:
+        adjusted_return = cap
+        capped = True
+    elif adjusted_return < -cap:
+        adjusted_return = -cap
+        capped = True
+
+    adjustment_applied = use_tiny_floor or capped or abs(multiplier - 1.0) > 1e-9
+    return adjusted_return, multiplier, adjustment_applied, capped
+
+
 def build_features_for_prediction(
     session: Session,
     target_symbol: str,
@@ -431,19 +499,41 @@ def generate_analysis_report(
     
     logger.info(f"Model prediction: raw_output={model_output:.6f}, target_type={target_type}")
     
-    # Compute predicted_return and predicted_price based on target_type
+    # Compute raw predicted return based on target_type
     if target_type == "simple_return":
-        predicted_return = model_output
-        predicted_price = baseline_price * (1 + predicted_return)
+        raw_predicted_return = model_output
     elif target_type == "log_return":
         import math
-        predicted_return = math.exp(model_output) - 1
-        predicted_price = baseline_price * math.exp(model_output)
+        raw_predicted_return = math.exp(model_output) - 1
     elif target_type == "price":
-        predicted_price = model_output
-        predicted_return = (predicted_price / baseline_price) - 1 if baseline_price > 0 else 0
+        raw_predicted_return = (model_output / baseline_price) - 1 if baseline_price > 0 else 0
+    else:
+        raw_predicted_return = 0.0
+
+    # Data quality feeds sentiment multiplier intensity.
+    data_quality = get_data_quality_stats(session, target_symbol)
+    news_count_7d = int(data_quality.get("news_count_7d") or 0)
+
+    predicted_return, sentiment_multiplier, adjustment_applied, predicted_return_capped = (
+        _apply_sentiment_adjustment(
+            raw_predicted_return=float(raw_predicted_return),
+            sentiment_index=float(current_sentiment),
+            news_count_7d=news_count_7d,
+        )
+    )
+    logger.info(
+        "Sentiment adjustment: raw=%.6f adjusted=%.6f multiplier=%.4f applied=%s capped=%s news_count_7d=%s sentiment=%.4f",
+        raw_predicted_return,
+        predicted_return,
+        sentiment_multiplier,
+        adjustment_applied,
+        predicted_return_capped,
+        news_count_7d,
+        current_sentiment,
+    )
+    predicted_price = baseline_price * (1 + predicted_return)
     
-    # Validate prediction (do not clamp by default - expose issues)
+    # Validate prediction after sentiment adjustment/cap.
     prediction_invalid = False
     if predicted_return < -1.0:
         logger.error(f"Invalid prediction: return {predicted_return:.4f} < -100%")
@@ -460,9 +550,6 @@ def generate_analysis_report(
         session, target_symbol, predicted_price
     )
     
-    # Get data quality
-    data_quality = get_data_quality_stats(session, target_symbol)
-    
     # Build influencer descriptions
     descriptions = get_feature_descriptions()
     top_influencers = []
@@ -493,6 +580,10 @@ def generate_analysis_report(
         "baseline_price": round(baseline_price, 4),
         "baseline_price_date": baseline_price_date,
         "predicted_return": round(predicted_return, 6),
+        "raw_predicted_return": round(raw_predicted_return, 6),
+        "sentiment_multiplier": round(sentiment_multiplier, 4),
+        "sentiment_adjustment_applied": bool(adjustment_applied),
+        "predicted_return_capped": bool(predicted_return_capped),
         "predicted_return_pct": round(predicted_return * 100, 2),
         "predicted_price": round(predicted_price, 4),
         "target_type": target_type,

app/schemas.py

@@ -35,6 +35,18 @@ class AnalysisReport(BaseModel):
     # Core prediction data (nullable for degraded modes)
     current_price: Optional[float] = Field(0.0, description="Most recent closing price")
     predicted_return: Optional[float] = Field(0.0, description="Predicted next-day return")
+    raw_predicted_return: Optional[float] = Field(
+        None, description="Raw model output converted to return before sentiment adjustment"
+    )
+    sentiment_multiplier: Optional[float] = Field(
+        None, description="Sentiment-driven multiplier applied to raw predicted return"
+    )
+    sentiment_adjustment_applied: Optional[bool] = Field(
+        None, description="Whether sentiment adjustment layer altered predicted return"
+    )
+    predicted_return_capped: Optional[bool] = Field(
+        None, description="Whether final predicted return was clipped by safety cap"
+    )
     predicted_price: Optional[float] = Field(0.0, description="Predicted next-day price")
     confidence_lower: Optional[float] = Field(0.0, description="Lower bound of confidence interval")
     confidence_upper: Optional[float] = Field(0.0, description="Upper bound of confidence interval")

app/settings.py

@@ -49,6 +49,10 @@ class Settings(BaseSettings):
     # Sentiment aggregation
     sentiment_tau_hours: float = 12.0
     sentiment_missing_fill: float = 0.0
+    sentiment_non_neutral_boost: float = 1.35
+    sentiment_soft_neutral_polarity_threshold: float = 0.12
+    sentiment_soft_neutral_max_mag: float = 0.25
+    sentiment_soft_neutral_scale: float = 0.8
     
     # API settings
     analysis_ttl_minutes: int = 30
@@ -81,6 +85,15 @@ class Settings(BaseSettings):
     
     # Twelve Data (Live Price)
     twelvedata_api_key: Optional[str] = None
+
+    # Inference sentiment adjustment (aggressive but capped)
+    inference_sentiment_multiplier_max: float = 2.0
+    inference_sentiment_multiplier_min: float = 0.5
+    inference_sentiment_news_ref: int = 30
+    inference_sentiment_power_ref: float = 0.20
+    inference_tiny_signal_threshold: float = 0.0015
+    inference_tiny_signal_floor: float = 0.0025
+    inference_return_cap: float = 0.02
     
     # LLM Sentiment Analysis
     # Deprecated - kept for backward compatibility