Add semantic cleanup, canonicalization, and generic-term downweighting

Filter obvious footer/legal CTA noise before graph build, canonicalize key term variants, and apply IDF-style specificity plus generic-domain penalties to improve important term ranking toward TextAnalyst-like behavior.

Made-with: Cursor

semantic_graph.py

@@ -27,6 +27,55 @@ def _extract_significant_lemmas(sent: Dict[str, Any]) -> List[str]:
     return lemmas
 
 
+NOISE_PATTERNS = (
+    "contact us",
+    "responsible gaming",
+    "play responsibly",
+    "copyright",
+    "terms of use",
+    "new customers only",
+    "t&c apply",
+    "18+",
+    "info@",
+)
+
+
+GENERIC_TERMS = {
+    "игра", "играть", "казино", "ставка", "деньга", "деньги",
+    "демо", "режим", "уровень", "шаг", "выигрыш", "риск",
+    "game", "play", "casino", "bet", "demo", "mode", "level", "risk", "win",
+}
+
+
+def _is_noise_sentence(text: str) -> bool:
+    t = " ".join((text or "").lower().split())
+    if not t:
+        return True
+    if any(p in t for p in NOISE_PATTERNS):
+        return True
+    # Частые CTA/служебные короткие строки.
+    if len(t.split()) <= 3 and t in {"играть", "play", "chicken road", "chicken road game"}:
+        return True
+    return False
+
+
+def _canonicalize_term(term: str) -> str:
+    t = " ".join((term or "").lower().replace("ё", "е").replace("-", " ").split())
+    if not t:
+        return t
+
+    # Бренд/наименование игры.
+    if t in {"чикен роад", "chicken road", "chickenroad", "chiken road"}:
+        return "chicken road"
+    if t in {"игры inout", "inout games", "inout game", "inout"}:
+        return "inout games"
+
+    # Полезная нормализация русской фразы под один термин.
+    if t in {"реальные деньги", "реальный деньги"}:
+        return "реальные деньги"
+    return t
+
+
 def _extract_phrase_candidates(sentence_text: str, lang: str) -> List[str]:
     """
     Извлекает фразовые кандидаты через существующую n-gram логику проекта,
@@ -74,15 +123,23 @@ def build_semantic_graph(
     pair_cooc = defaultdict(int)
     phrase_occ = Counter()
     phrase_sent_ids = defaultdict(set)
+    term_sent_ids = defaultdict(set)
     sentence_words: List[List[str]] = []
     sentence_phrases: List[List[str]] = []
 
     for sent_id, sent in enumerate(sentences_data):
+        raw_text = sent.get("raw_text", "")
+        if _is_noise_sentence(raw_text):
+            sentence_words.append([])
+            sentence_phrases.append([])
+            continue
+
         lemmas_raw = _extract_significant_lemmas(sent)
-        lemmas = _normalize_lemma_sequence(lemmas_raw)
+        lemmas = [_canonicalize_term(x) for x in _normalize_lemma_sequence(lemmas_raw)]
+        lemmas = [x for x in lemmas if x]
         sentence_words.append(lemmas)
 
-        phrase_candidates = _extract_phrase_candidates(sent.get("raw_text", ""), lang)
+        phrase_candidates = _extract_phrase_candidates(raw_text, lang)
         # Фильтр мусора фраз: минимум 2 слова, без дублирующегося подряд слова.
         clean_phrases = []
         for p in phrase_candidates:
@@ -92,7 +149,7 @@ def build_semantic_graph(
             bad_repeat = any(parts[i] == parts[i + 1] for i in range(len(parts) - 1))
             if bad_repeat:
                 continue
-            clean_phrases.append(" ".join(parts))
+            clean_phrases.append(_canonicalize_term(" ".join(parts)))
         sentence_phrases.append(clean_phrases)
 
         if not lemmas:
@@ -115,6 +172,8 @@ def build_semantic_graph(
                 neighbors.add(b)
             for b in neighbors:
                 pair_cooc[(a, b)] += 1
+        for w in set(lemmas):
+            term_sent_ids[w].add(sent_id)
 
     # Частоты/охват фраз по предложениям.
     for sent_id, phrases in enumerate(sentence_phrases):
@@ -140,6 +199,7 @@ def build_semantic_graph(
 
         for phrase in uniq_phrases:
             term_occ[phrase] += 1
+            term_sent_ids[phrase].add(sent_id)
             parts = set(phrase.split())
             # Связываем фразу с ее компонентами всегда (ядро термина).
             for w in parts:
@@ -179,18 +239,24 @@ def build_semantic_graph(
     # Для слов: умеренный буст по контекстной связанности.
     # Для фраз: более сильный буст по частоте/охвату, чтобы устойчивые термины поднимались в топ.
     combined_scores = {}
+    total_sent = max(1, len(sentence_words))
     for node, score in pr.items():
         out_deg = graph.out_degree(node)
         in_deg = graph.in_degree(node)
         connectivity_factor = 1.0 + 0.025 * (out_deg + in_deg)
+        sent_df = len(term_sent_ids.get(node, set()))
+        idf_factor = 1.0 + 0.35 * math.log((1.0 + total_sent) / (1.0 + max(1, sent_df)))
 
         if graph.nodes[node].get("term_type") == "phrase":
             freq = max(1, int(graph.nodes[node].get("frequency", 1)))
             sent_cover = len(phrase_sent_ids.get(node, set()))
             termness = (1.0 + 0.22 * math.log1p(freq)) * (1.0 + 0.12 * math.log1p(sent_cover))
-            combined_scores[node] = score * connectivity_factor * termness
+            contains_generic = any(part in GENERIC_TERMS for part in node.split())
+            generic_penalty = 0.85 if contains_generic else 1.0
+            combined_scores[node] = score * connectivity_factor * termness * idf_factor * generic_penalty
         else:
-            combined_scores[node] = score * connectivity_factor
+            generic_penalty = 0.58 if node in GENERIC_TERMS else 1.0
+            combined_scores[node] = score * connectivity_factor * idf_factor * generic_penalty
 
     node_weights = _normalize_to_1_100(combined_scores)