Add configurable BERT stage target and stage visibility in optimizer.

Allow users to set custom BERT threshold for Stage A completion (e.g. accept 0.61), persist it in UI project state, and expose stage transitions in logs for transparent progression to BM25/Semantic/N-gram/Title.

Made-with: Cursor

docs/FULL_FUNCTIONAL_DOCUMENTATION.md

@@ -164,9 +164,10 @@
 ### Вход (`OptimizerRequest`)
 - аналитические данные: `target_text`, `competitors`, `keywords`, `language`, `target_title`, `competitor_titles`
 - LLM: `api_key`, `api_base_url`, `model`, `temperature`
-- стратегия: `max_iterations`, `candidates_per_iteration`, `optimization_mode`, `phrase_strategy_mode`
+- стратегия: `max_iterations`, `candidates_per_iteration`, `optimization_mode`, `phrase_strategy_mode`, `bert_stage_target`
   - `phrase_strategy_mode`: `auto | distributed_preferred | exact_preferred | ensemble`
   - `ensemble`: в пределах итерации циклически пробует несколько phrase-стратегий и ранжирует кандидаты общей utility-функцией.
+  - `bert_stage_target`: пользовательский порог завершения этапа A (BERT), например `0.61` вместо `0.70`.
 
 ### Выход (`OptimizerResponse`)
 - `optimized_text`
@@ -175,6 +176,7 @@
 - `applied_changes`
 - `optimization_mode`
 - `phrase_strategy_mode`
+- `bert_stage_target`
 - `error` (если есть)
 
 ---

models.py

@@ -92,6 +92,7 @@ class OptimizerRequest(BaseModel):
     temperature: float = 0.25
     optimization_mode: str = "balanced"
     phrase_strategy_mode: str = "auto"  # auto | exact_preferred | distributed_preferred | ensemble
+    bert_stage_target: float = 0.70
 
 
 class OptimizerResponse(BaseModel):
@@ -103,4 +104,5 @@ class OptimizerResponse(BaseModel):
     applied_changes: int = 0
     optimization_mode: str = "balanced"
     phrase_strategy_mode: str = "auto"
+    bert_stage_target: float = 0.70
     error: str = ""

optimizer.py

@@ -232,12 +232,18 @@ def _is_semantic_gap(target_weight: float, competitor_avg_weight: float) -> bool
     return (competitor_avg_weight > rel_threshold) and (abs_gap >= SEMANTIC_GAP_MIN_ABS)
 
 
-def _compute_metrics(analysis: Dict[str, Any], semantic: Dict[str, Any], keywords: List[str], language: str) -> Dict[str, Any]:
+def _compute_metrics(
+    analysis: Dict[str, Any],
+    semantic: Dict[str, Any],
+    keywords: List[str],
+    language: str,
+    bert_stage_target: float = BERT_TARGET_THRESHOLD,
+) -> Dict[str, Any]:
     competitor_count = len(analysis.get("word_counts", {}).get("competitors", []))
     min_signal = 1 if competitor_count <= 1 else 2
 
     bert_details = analysis.get("bert_analysis", {}).get("detailed", []) or []
-    bert_low = [d for d in bert_details if float(d.get("my_max_score", 0)) < BERT_TARGET_THRESHOLD]
+    bert_low = [d for d in bert_details if float(d.get("my_max_score", 0)) < float(bert_stage_target)]
     bert_phrase_scores = {
         str(d.get("phrase", "")).strip().lower(): float(d.get("my_max_score", 0) or 0.0)
         for d in bert_details
@@ -346,10 +352,11 @@ def _choose_optimization_goal(
     keywords: List[str],
     language: str,
     stage: str = "bert",
+    bert_stage_target: float = BERT_TARGET_THRESHOLD,
 ) -> Dict[str, Any]:
     candidates: Dict[str, Dict[str, Any]] = {}
     bert_details = analysis.get("bert_analysis", {}).get("detailed", []) or []
-    low_bert = [x for x in bert_details if float(x.get("my_max_score", 0)) < BERT_TARGET_THRESHOLD]
+    low_bert = [x for x in bert_details if float(x.get("my_max_score", 0)) < float(bert_stage_target)]
     if low_bert:
         worst = sorted(low_bert, key=lambda x: float(x.get("my_max_score", 0)))[0]
         focus_terms = _filter_stopwords(_tokenize(worst.get("phrase", "")), language)[:4]
@@ -963,9 +970,11 @@ def _stage_primary_progress(stage: str, prev_metrics: Dict[str, Any], next_metri
     return False
 
 
-def _is_stage_complete(stage: str, metrics: Dict[str, Any]) -> bool:
+def _is_stage_complete(stage: str, metrics: Dict[str, Any], bert_stage_target: float = BERT_TARGET_THRESHOLD) -> bool:
     if stage == "bert":
-        return int(metrics.get("bert_low_count", 0)) == 0
+        scores = [float(v) for v in (metrics.get("bert_phrase_scores") or {}).values()]
+        max_phrase = max([0.0] + scores)
+        return max_phrase >= float(bert_stage_target)
     if stage == "bm25":
         return int(metrics.get("bm25_remove_count", 0)) <= 3
     if stage == "semantic":
@@ -1089,12 +1098,16 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
     phrase_strategy_mode = str(request_data.get("phrase_strategy_mode", "auto") or "auto").strip().lower()
     if phrase_strategy_mode not in {"auto", "exact_preferred", "distributed_preferred", "ensemble"}:
         phrase_strategy_mode = "auto"
+    bert_stage_target = float(request_data.get("bert_stage_target", BERT_TARGET_THRESHOLD) or BERT_TARGET_THRESHOLD)
+    bert_stage_target = max(0.0, min(1.0, bert_stage_target))
 
     baseline_analysis = _build_analysis_snapshot(
         target_text, competitors, keywords, language, target_title, competitor_titles
     )
     baseline_semantic = _build_semantic_snapshot(target_text, competitors, language)
-    baseline_metrics = _compute_metrics(baseline_analysis, baseline_semantic, keywords, language)
+    baseline_metrics = _compute_metrics(
+        baseline_analysis, baseline_semantic, keywords, language, bert_stage_target=bert_stage_target
+    )
 
     current_text = target_text
     current_analysis = baseline_analysis
@@ -1112,7 +1125,9 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
     stage_no_progress_steps = 0
 
     for step in range(max_iterations):
-        while stage_idx < len(STAGE_ORDER) and _is_stage_complete(STAGE_ORDER[stage_idx], current_metrics):
+        while stage_idx < len(STAGE_ORDER) and _is_stage_complete(
+            STAGE_ORDER[stage_idx], current_metrics, bert_stage_target=bert_stage_target
+        ):
             stage_idx += 1
             stage_no_progress_steps = 0
         if stage_idx >= len(STAGE_ORDER):
@@ -1126,6 +1141,7 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
             keywords,
             language,
             stage=active_stage,
+            bert_stage_target=bert_stage_target,
         )
         if goal["type"] == "none":
             stage_idx += 1
@@ -1314,7 +1330,9 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
                         candidate_text, competitors, keywords, language, target_title, competitor_titles
                     )
                     cand_semantic = _build_semantic_snapshot(candidate_text, competitors, language)
-                    cand_metrics = _compute_metrics(cand_analysis, cand_semantic, keywords, language)
+                    cand_metrics = _compute_metrics(
+                        cand_analysis, cand_semantic, keywords, language, bert_stage_target=bert_stage_target
+                    )
                     valid, invalid_reasons, goal_improved = _is_candidate_valid(
                         current_metrics, cand_metrics, goal["type"], goal["label"], optimization_mode
                     )
@@ -1548,7 +1566,9 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
                         batch_text, competitors, keywords, language, target_title, competitor_titles
                     )
                     batch_semantic = _build_semantic_snapshot(batch_text, competitors, language)
-                    batch_metrics = _compute_metrics(batch_analysis, batch_semantic, keywords, language)
+                    batch_metrics = _compute_metrics(
+                        batch_analysis, batch_semantic, keywords, language, bert_stage_target=bert_stage_target
+                    )
                     b_valid, b_reasons, b_goal = _is_candidate_valid(
                         current_metrics, batch_metrics, goal["type"], goal["label"], optimization_mode
                     )
@@ -1801,4 +1821,5 @@ def optimize_text(request_data: Dict[str, Any]) -> Dict[str, Any]:
         "applied_changes": applied_changes,
         "optimization_mode": optimization_mode,
         "phrase_strategy_mode": phrase_strategy_mode,
+        "bert_stage_target": round(bert_stage_target, 4),
     }

templates/index.html

@@ -302,6 +302,10 @@
                                 <label class="form-label small text-muted mb-1">Temp</label>
                                 <input type="number" id="optimizerTemp" class="form-control" min="0" max="1.2" step="0.05" value="0.25">
                             </div>
+                            <div class="col-md-2">
+                                <label class="form-label small text-muted mb-1">BERT target A-stage</label>
+                                <input type="number" id="optimizerBertStageTarget" class="form-control" min="0" max="1" step="0.01" value="0.70">
+                            </div>
                             <div class="col-md-3">
                                 <label class="form-label small text-muted mb-1">Режим оптимизации</label>
                                 <select id="optimizerMode" class="form-select">
@@ -558,6 +562,7 @@
                 optimizer_iterations: Number(document.getElementById('optimizerIterations').value || 2),
                 optimizer_candidates: Number(document.getElementById('optimizerCandidates').value || 2),
                 optimizer_temperature: Number(document.getElementById('optimizerTemp').value || 0.25),
+                optimizer_bert_stage_target: Number(document.getElementById('optimizerBertStageTarget').value || 0.70),
                 optimizer_mode: document.getElementById('optimizerMode').value,
                 optimizer_phrase_strategy: document.getElementById('optimizerPhraseStrategy').value
             },
@@ -609,6 +614,7 @@
         document.getElementById('optimizerIterations').value = 2;
         document.getElementById('optimizerCandidates').value = 2;
         document.getElementById('optimizerTemp').value = 0.25;
+        document.getElementById('optimizerBertStageTarget').value = 0.70;
         document.getElementById('optimizerMode').value = 'balanced';
         document.getElementById('optimizerPhraseStrategy').value = 'auto';
 
@@ -663,6 +669,7 @@
         document.getElementById('optimizerIterations').value = inp.optimizer_iterations ?? 2;
         document.getElementById('optimizerCandidates').value = inp.optimizer_candidates ?? 2;
         document.getElementById('optimizerTemp').value = inp.optimizer_temperature ?? 0.25;
+        document.getElementById('optimizerBertStageTarget').value = inp.optimizer_bert_stage_target ?? 0.70;
         document.getElementById('optimizerMode').value = inp.optimizer_mode || 'balanced';
         document.getElementById('optimizerPhraseStrategy').value = inp.optimizer_phrase_strategy || 'auto';
 
@@ -853,9 +860,12 @@
             const after = it.metrics_after ? it.metrics_after.score : '-';
             const baseline = (it.current_score ?? before);
             const reason = it.reason || (it.candidates ? 'all candidates rejected by constraints' : '-');
+            const stage = (it.stage || (it.goal && it.goal.type) || '-');
+            const advanced = it.advanced_to_stage ? ` → ${it.advanced_to_stage}` : '';
             return `<tr>
                 <td>${it.step}</td>
                 <td>${it.status}</td>
+                <td>${stage}${advanced}</td>
                 <td>${it.goal ? (it.goal.type + ': ' + (it.goal.label || '')) : '-'}</td>
                 <td>L${it.cascade_level || '-'} / ${it.operation || '-'}</td>
                 <td>${baseline}</td>
@@ -919,9 +929,11 @@
                         <div><strong>Шаг ${it.step}</strong> — ${safeHtml(it.status || '-')}</div>
                         <span class="badge bg-secondary">${safeHtml(it.goal ? (it.goal.type + ': ' + (it.goal.label || '')) : '-')}</span>
                     </div>
+                    <div class="small mb-2"><strong>Этап:</strong> ${safeHtml(it.stage || (it.goal ? it.goal.type : '-') || '-')}</div>
                     <div class="small mb-2"><strong>Каскад:</strong> L${safeHtml(it.cascade_level || '-')} / ${safeHtml(it.operation || '-')}</div>
                     <div class="small mb-2"><strong>Причина:</strong> ${safeHtml(it.reason || '-')}</div>
                     ${it.escalated_to_level ? `<div class="small mb-2 text-warning"><strong>Эскалация:</strong> переход на L${safeHtml(it.escalated_to_level)}</div>` : ''}
+                    ${it.advanced_to_stage ? `<div class="small mb-2 text-info"><strong>Смена этапа:</strong> переход на ${safeHtml(it.advanced_to_stage)}</div>` : ''}
                     <div class="small mb-2"><strong>Исходное предложение:</strong><br><span class="text-muted">${sentBefore}</span></div>
                     <div class="small mb-2"><strong>Выбранный вариант:</strong><br><span class="text-muted">${sentAfterChosen}</span></div>
                     <div class="table-responsive">
@@ -945,6 +957,7 @@
                 <div class="small mb-2">
                     Режим: <strong>${data.optimization_mode || 'balanced'}</strong>
                     · Phrase Strategy: <strong>${data.phrase_strategy_mode || 'auto'}</strong>
+                    · BERT target A-stage: <strong>${data.bert_stage_target ?? 0.7}</strong>
                 </div>
                 <div class="table-responsive">
                     <table class="table table-sm table-bordered mb-0">
@@ -957,8 +970,8 @@
                 <h6 class="card-title">Лог итераций</h6>
                 <div class="table-responsive">
                     <table class="table table-sm table-hover mb-0">
-                        <thead><tr><th>#</th><th>Статус</th><th>Цель</th><th>Каскад</th><th>Baseline шага</th><th>Score до</th><th>Score после</th><th>Δ</th><th>Причина/комментарий</th></tr></thead>
-                        <tbody>${iterRows || '<tr><td colspan="9" class="text-muted text-center">Нет данных</td></tr>'}</tbody>
+                        <thead><tr><th>#</th><th>Статус</th><th>Этап</th><th>Цель</th><th>Каскад</th><th>Baseline шага</th><th>Score до</th><th>Score после</th><th>Δ</th><th>Причина/комментарий</th></tr></thead>
+                        <tbody>${iterRows || '<tr><td colspan="10" class="text-muted text-center">Нет данных</td></tr>'}</tbody>
                     </table>
                 </div>
             </div>
@@ -1002,6 +1015,7 @@
             max_iterations: Number(document.getElementById('optimizerIterations').value || 2),
             candidates_per_iteration: Number(document.getElementById('optimizerCandidates').value || 2),
             temperature: Number(document.getElementById('optimizerTemp').value || 0.25),
+            bert_stage_target: Number(document.getElementById('optimizerBertStageTarget').value || 0.70),
             optimization_mode: document.getElementById('optimizerMode').value || 'balanced',
             phrase_strategy_mode: document.getElementById('optimizerPhraseStrategy').value || 'auto'
         };