Add multi-objective candidate utility for optimizer decisions.

Introduce a dynamic utility function that balances BERT phrase gains with cross-metric regression penalties, and use it in candidate selection/ranking to preserve future optimization capacity. Update full documentation to describe the utility-driven acceptance logic.

Made-with: Cursor

docs/FULL_FUNCTIONAL_DOCUMENTATION.md

@@ -439,6 +439,11 @@ HTML extraction pipeline:
 - `_goal_improved`:
   - для BERT: улучшение score целевой фразы минимум на `BERT_GOAL_DELTA_MIN=0.005` **или** снижение `bert_low_count`;
   - для других целей: профильные метрики улучшения.
+- `_candidate_utility`:
+  - многоцелевая функция полезности кандидата с динамическими весами;
+  - учитывает одновременно `bert_phrase_delta`, `chunk_goal_delta`, `score_delta`;
+  - добавляет мягкие штрафы за регрессии по BM25/BERT-low/N-gram/SemanticGap/Title;
+  - в BERT-push режиме (когда фраза ниже порога) усиливает вес phrase-level прогресса.
 - `_is_candidate_valid`:
   - hard constraints (не ухудшать критичные метрики сверх допустимого);
   - режимы `conservative/balanced/aggressive` задают пороги регрессии;
@@ -466,6 +471,7 @@ HTML extraction pipeline:
    - если локально улучшает чанк, но глобально не проходит — кандидат кладется в queue.
    - для BERT учитывается прямой документный `bert_phrase_delta` по целевой фразе: даже небольшой положительный рост считается полезным шагом при отсутствии регрессий по guardrails.
    - если нет `promotable` кандидата, но есть guardrail-valid кандидат с `local_chunk_improved`, применяется режим `applied_local_progress`: правка принимается локально и оптимизация переходит к следующему чанку (накопительная стратегия).
+   - ранжирование и выбор best-кандидата дополнительно учитывают `candidate_utility`, чтобы BERT-оптимизация не вредила следующим этапам по другим метрикам.
 9. batch-логика queue:
    - optimizer пробует совместно применить комбинации из 2..4 локально сильных не конфликтующих правок;
    - batch принимается только при прохождении глобальных ограничений и положительном совокупном локальном приросте.

optimizer.py

@@ -815,6 +815,58 @@ def _bert_phrase_delta(goal_label: str, prev_metrics: Dict[str, Any], next_metri
     return round(next_phrase - prev_phrase, 4)
 
 
+def _safe_delta(prev_metrics: Dict[str, Any], next_metrics: Dict[str, Any], key: str) -> float:
+    try:
+        return float(next_metrics.get(key, 0.0)) - float(prev_metrics.get(key, 0.0))
+    except Exception:
+        return 0.0
+
+
+def _candidate_utility(
+    *,
+    prev_metrics: Dict[str, Any],
+    next_metrics: Dict[str, Any],
+    goal_type: str,
+    goal_label: str,
+    bert_phrase_delta: float,
+    chunk_goal_delta: float,
+    local_chunk_improved: bool,
+) -> float:
+    score_delta = _safe_delta(prev_metrics, next_metrics, "score")
+    bm25_delta = _safe_delta(prev_metrics, next_metrics, "bm25_remove_count")
+    bert_low_delta = _safe_delta(prev_metrics, next_metrics, "bert_low_count")
+    ngram_delta = _safe_delta(prev_metrics, next_metrics, "ngram_signal_count")
+    sem_gap_delta = _safe_delta(prev_metrics, next_metrics, "semantic_gap_count")
+    title_delta = _safe_delta(prev_metrics, next_metrics, "title_bert_score")
+
+    # Dynamic emphasis:
+    # - if target phrase is still far from threshold, prioritize phrase-level BERT gains
+    # - but keep non-BERT regressions as penalties to preserve future optimization capacity
+    key = (goal_label or "").strip().lower()
+    prev_phrase = float((prev_metrics.get("bert_phrase_scores") or {}).get(key, 0.0))
+    bert_push_mode = (goal_type == "bert" and prev_phrase < BERT_TARGET_THRESHOLD)
+
+    w_phrase = 7.5 if bert_push_mode else 3.0
+    w_chunk = 1.6 if bert_push_mode else 1.0
+    w_score = 1.0
+
+    utility = (
+        (w_phrase * float(bert_phrase_delta))
+        + (w_chunk * float(chunk_goal_delta))
+        + (w_score * float(score_delta))
+    )
+    if local_chunk_improved:
+        utility += 0.3
+
+    # Cross-metric guardrails as soft penalties (in addition to hard validity checks).
+    utility -= max(0.0, bm25_delta) * 1.8
+    utility -= max(0.0, bert_low_delta) * 2.4
+    utility -= max(0.0, ngram_delta) * 0.6
+    utility -= max(0.0, sem_gap_delta) * 1.5
+    utility += min(0.0, title_delta) * 1.2
+    return round(float(utility), 4)
+
+
 def _is_candidate_valid(
     prev_metrics: Dict[str, Any],
     next_metrics: Dict[str, Any],
@@ -1069,6 +1121,15 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
                     )
                     delta_score = round(cand_metrics["score"] - current_metrics["score"], 3)
                     bert_phrase_delta = _bert_phrase_delta(goal["label"], current_metrics, cand_metrics) if goal.get("type") == "bert" else 0.0
+                    candidate_utility = _candidate_utility(
+                        prev_metrics=current_metrics,
+                        next_metrics=cand_metrics,
+                        goal_type=str(goal.get("type", "")),
+                        goal_label=str(goal.get("label", "")),
+                        bert_phrase_delta=bert_phrase_delta,
+                        chunk_goal_delta=chunk_delta,
+                        local_chunk_improved=local_chunk_improved,
+                    )
                     md = _metrics_delta(current_metrics, cand_metrics)
                     candidates.append(
                         {
@@ -1097,6 +1158,7 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
                             "invalid_reasons": invalid_reasons,
                             "delta_score": delta_score,
                             "candidate_score": cand_metrics.get("score"),
+                            "candidate_utility": candidate_utility,
                             "metrics_delta": md,
                             "edit_payload": {
                                 "operation": operation,
@@ -1146,6 +1208,7 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
                 c.get("goal_improved")
                 or (goal.get("type") == "bert" and float(c.get("bert_phrase_delta") or 0.0) > 0.0)
                 or float(c.get("candidate_score") or -1) > float(current_metrics["score"])
+                or float(c.get("candidate_utility") or -999.0) > 0.0
             )
             and (
                 goal.get("type") != "bert"
@@ -1166,6 +1229,7 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
                 best_local = sorted(
                     local_progress_candidates,
                     key=lambda c: (
+                        float(c.get("candidate_utility") or -999.0),
                         float(c.get("chunk_goal_delta") or 0.0),
                         float(c.get("bert_phrase_delta") or 0.0),
                         float(c.get("candidate_score") or -999.0),
@@ -1202,6 +1266,7 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
                         "chosen_candidate_index": best_local.get("candidate_index"),
                         "chosen_chunk_goal_delta": best_local.get("chunk_goal_delta"),
                         "chosen_bert_phrase_delta": best_local.get("bert_phrase_delta"),
+                        "chosen_candidate_utility": best_local.get("candidate_utility"),
                         "chosen_metrics_delta": best_local.get("metrics_delta"),
                         "candidates": [
                             {
@@ -1217,6 +1282,7 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
                                 "llm_prompt_debug": c.get("llm_prompt_debug"),
                                 "llm_rationale": c.get("llm_rationale"),
                                 "metrics_delta": c.get("metrics_delta"),
+                                "candidate_utility": c.get("candidate_utility"),
                                 "invalid_reasons": c.get("invalid_reasons", []),
                                 "delta_score": c.get("delta_score"),
                                 "candidate_score": c.get("candidate_score"),
@@ -1241,6 +1307,7 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
             ]
             local_pool.sort(
                 key=lambda c: (
+                    float(c.get("candidate_utility") or -999.0),
                     float(c.get("chunk_goal_delta") or -999.0),
                     float(c.get("candidate_score") or -999.0),
                 ),
@@ -1364,6 +1431,7 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
                                 "llm_prompt_debug": c.get("llm_prompt_debug"),
                                 "llm_rationale": c.get("llm_rationale"),
                                 "metrics_delta": c.get("metrics_delta"),
+                                "candidate_utility": c.get("candidate_utility"),
                                 "invalid_reasons": c.get("invalid_reasons", []),
                                 "delta_score": c.get("delta_score"),
                                 "candidate_score": c.get("candidate_score"),
@@ -1413,6 +1481,7 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
                             "llm_prompt_debug": c.get("llm_prompt_debug"),
                             "llm_rationale": c.get("llm_rationale"),
                             "metrics_delta": c.get("metrics_delta"),
+                            "candidate_utility": c.get("candidate_utility"),
                             "invalid_reasons": c.get("invalid_reasons", []),
                             "delta_score": c.get("delta_score"),
                             "candidate_score": c.get("candidate_score"),
@@ -1434,6 +1503,7 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
             valid_candidates,
             key=lambda c: (
                 1 if c.get("goal_improved") else 0,
+                float(c.get("candidate_utility") or -999.0),
                 float(c.get("bert_phrase_delta") or 0.0),
                 float(c.get("chunk_goal_delta") or 0.0),
                 c["metrics"]["score"],
@@ -1467,6 +1537,7 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
                 "metrics_after": current_metrics,
                 "delta_score": round(current_metrics["score"] - prev_metrics["score"], 3),
                 "chosen_candidate_index": best.get("candidate_index"),
+                "chosen_candidate_utility": best.get("candidate_utility"),
                 "candidates": [
                     {
                         "candidate_index": c.get("candidate_index"),
@@ -1481,6 +1552,7 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
                         "llm_prompt_debug": c.get("llm_prompt_debug"),
                         "llm_rationale": c.get("llm_rationale"),
                         "metrics_delta": c.get("metrics_delta"),
+                        "candidate_utility": c.get("candidate_utility"),
                         "invalid_reasons": c.get("invalid_reasons", []),
                         "delta_score": c.get("delta_score"),
                         "candidate_score": c.get("candidate_score"),