"""Terminology extraction: segmentation, frequency, POS chunking, keyword scoring.""" import re from collections import Counter from dataclasses import dataclass, field @dataclass class TermEntry: """A single extracted term.""" term: str frequency: int = 1 score: float = 0.0 word_type: str = "" # keyword, phrase, proper_noun, textrank, ngram positions: list[int] = field(default_factory=list) # line numbers search_results: list[dict] = field(default_factory=list) # pipeline search results def extract(text: str, top_n: int = 30) -> list[TermEntry]: """Extract key terminology from text. Chinese text: C-Value + pke_zh + jieba TF-IDF + TextRank + POS chunking English text: C-Value + YAKE + regex proper nouns C-Value is a well-established algorithm that handles nested terms (e.g., scores "免疫检查点抑制剂" higher than "检查点抑制剂"). Args: text: Source text. top_n: Maximum number of terms to return. Returns: List of TermEntry sorted by score descending. """ terms: list[TermEntry] = [] cn_chars = len(re.findall(r"[\u4e00-\u9fff]", text)) en_chars = len(re.findall(r"[a-zA-Z]", text)) is_cn = cn_chars > en_chars * 0.5 if is_cn: terms.extend(_extract_jieba(text, top_n)) terms.extend(_extract_textrank(text, top_n)) terms.extend(_extract_pke_zh(text, top_n)) terms.extend(_extract_phrases(text, top_n)) else: terms.extend(_extract_yake(text, top_n)) terms.extend(_extract_proper_nouns(text)) # C-Value: always applied for both languages (re-ranks with nesting awareness) terms.extend(_extract_cvalue(text, top_n, is_cn)) return _merge_and_rank(terms, top_n) def extract_file(filepath: str, top_n: int = 30) -> list[TermEntry]: """Extract terms from a text file.""" with open(filepath, "r", encoding="utf-8") as f: text = f.read() return extract(text, top_n) # ════════════════════════════════════════════════════════════════════ # Method 1: jieba TF-IDF keyword extraction # ════════════════════════════════════════════════════════════════════ def _extract_jieba(text: str, top_n: int) -> list[TermEntry]: """Extract keywords using jieba TF-IDF.""" try: import jieba import jieba.analyse keywords = jieba.analyse.extract_tags(text, topK=top_n, withWeight=True) results: list[TermEntry] = [] for word, weight in keywords: if len(word) < 2: continue results.append(TermEntry(term=word, score=weight * 0.85, word_type="keyword")) return results except ImportError: return _fallback_chinese_freq(text, top_n) def _fallback_chinese_freq(text: str, top_n: int) -> list[TermEntry]: """Fallback: count Chinese bigrams by frequency.""" cn_chars = re.findall(r"[\u4e00-\u9fff]", text) bigrams = ["".join(cn_chars[i : i + 2]) for i in range(len(cn_chars) - 1)] counter = Counter(bigrams) total = max(1, sum(counter.values())) return [ TermEntry(term=bg, frequency=count, score=count / total, word_type="bigram") for bg, count in counter.most_common(top_n) if len(bg) == 2 ] # ════════════════════════════════════════════════════════════════════ # Method 2: jieba TextRank keyword extraction # ════════════════════════════════════════════════════════════════════ def _extract_textrank(text: str, top_n: int) -> list[TermEntry]: """Extract keywords using jieba TextRank algorithm. TextRank uses a graph-based approach similar to PageRank, building a co-occurrence graph of words and ranking them. It often produces different (and sometimes better) results than TF-IDF. """ try: import jieba.analyse keywords = jieba.analyse.textrank( text, topK=top_n, withWeight=True, allowPOS=('ns', 'n', 'vn', 'v', 'nr', 'nt', 'nz', 'a', 'an'), ) results: list[TermEntry] = [] for word, weight in keywords: if len(word) >= 2: # Normalize score: TextRank weights are typically 0-1 results.append(TermEntry(term=word, score=weight * 0.9, word_type="textrank")) return results except Exception: return [] # ════════════════════════════════════════════════════════════════════ # Method 3: POS-based phrase chunking (multi-word term extraction) # ════════════════════════════════════════════════════════════════════ # POS tags that can participate in noun phrases _NOUN_POS = frozenset({ 'n', 'nr', 'ns', 'nt', 'nz', 'ng', # nouns 'vn', # verbal noun (e.g. \u7814\u7a76, \u68c0\u6d4b) 'an', # noun-adjective 'j', 'l', 'i', # abbreviation, idiom }) # POS tags allowed as modifiers before nouns _MOD_POS = frozenset({ 'n', 'nr', 'ns', 'nt', 'nz', 'ng', 'vn', 'an', 'a', 'ad', 'ag', # adjectives 'b', # distinguisher (\u533b\u7597, \u6c11\u4e3b) 'v', 'vd', # verb (allowed only when followed by noun) 'eng', # English }) # Words that should never be part of a terminology phrase _PHRASE_STOP_WORDS = frozenset({ '\u7684', '\u4e86', '\u5728', '\u548c', '\u4e0e', '\u662f', '\u4e0d', '\u8fd9', '\u90a3', '\u5176', '\u4e4b', '\u800c', '\u4e14', '\u53ca', '\u7b49', '\u88ab', '\u628a', '\u5c06', '\u4ece', '\u5bf9', '\u5411', '\u7740', '\u8fc7', '\u5f97', '\u5730', '\u8981', '\u4f1a', '\u80fd', '\u53ef', '\u4ee5', '\u4e0a', '\u4e0b', '\u4e2d', '\u524d', '\u540e', '\u6709', '\u65e0', '\u6765', '\u53bb', '\u5230', '\u51fa', '\u5165', '\u5927', '\u5c0f', '\u591a', '\u5c11', '\u65b0', '\u65e7', '\u5f88', '\u66f4', '\u6700', '\u5c31', '\u90fd', '\u4e5f', '\u8fd8', '\u53c8', '\u624d', # Common grammatical verbs '\u5305\u62ec', '\u6839\u636e', '\u901a\u8fc7', '\u9700\u8981', '\u663e\u793a', '\u6210\u4e3a', '\u4f5c\u4e3a', '\u8fdb\u884c', '\u4f7f\u7528', '\u91c7\u7528', '\u5177\u6709', '\u5b58\u5728', '\u53d1\u751f', '\u51fa\u73b0', '\u9009\u62e9', '\u7ed3\u5408', '\u5bf9\u6bd4', '\u8fd1\u5e74', '\u91cd\u8981', '\u5408\u9002', '\u4f18\u52bf', '\u5176\u9ad8', '\u7ed3\u679c', '\u7a81\u7834', '\u663e\u8457', '\u8f83\u4f4e', '\u5305\u62ec', '\u65b9\u6cd5', '\u9886\u57df', '\u5b9e\u8df5', '\u4f18\u52bf', # Common adjectives that rarely form terminology '\u5e38\u89c1', '\u4e3b\u8981', '\u57fa\u672c', '\u4e00\u822c', '\u67d0\u4e9b', '\u5404\u79cd', '\u4e0d\u540c', '\u76f8\u5173', '\u5176\u4ed6', '\u8fd9\u79cd', }) # POS that are definitely NOT content (punctuation, numbers, etc.) _SKIP_POS = frozenset({'w', 'x', 'm', 'q', 'r', 'e', 'o', 'y', 'z', 'k', 'h', 'f'}) def _extract_phrases(text: str, top_n: int) -> list[TermEntry]: """Extract multi-word phrases by POS chunking with stop-word filtering. Approach: 1. Segment with jieba POS tagging 2. Mark each token as content, stop, or connector 3. Build noun-centered phrases of 2-5 consecutive content tokens 4. Filter: must contain at least one noun, no edge stop words 5. Score by frequency * length bonus This is based on the approach used by JioNLP and textrank4zh. """ try: import jieba.posseg as pseg words = list(pseg.cut(text)) # Classify each token tokens: list[dict] = [] for w in words: word = w.word.strip() flag = w.flag if not word: continue is_skip = ( flag in _SKIP_POS or word in _PHRASE_STOP_WORDS or len(word) == 1 and flag not in ('eng', 'n', 'a', 'v') # single char, non-content ) is_content = not is_skip and ( flag in _MOD_POS or (len(word) >= 2 and flag not in ('t', 'c', 'p', 'd', 'u', 'uj', 'ul', 'uv', 'ug')) ) tokens.append({ "word": word, "flag": flag, "is_content": is_content, "is_skip": is_skip, "len": len(word), }) # Build candidate phrases: 2-5 consecutive content tokens candidates: list[tuple[str, int]] = [] # (phrase, start position) i = 0 while i < len(tokens): if not tokens[i]["is_content"]: i += 1 continue parts = [] j = i while j < len(tokens) and (j - i) < 5: t = tokens[j] if t["is_content"]: parts.append(t["word"]) j += 1 elif t["is_skip"]: break # hard boundary elif t["flag"] in ('uj', 'ul', 'uv'): # \u7684, \u5730, \u5f97 # Allow \u7684 as connector if surrounded by content if len(parts) > 0 and (j + 1) < len(tokens) and tokens[j + 1]["is_content"]: parts.append(t["word"]) j += 1 else: break else: break # soft boundary — stop growing this phrase if len(parts) >= 2: phrase = "".join(parts) # Length check: 3-20 chars (shorter min to catch "靶向治疗") if 3 <= len(phrase) <= 24: # Must contain at least one noun-tagged word has_noun = any( tokens[i + k]["flag"] in _NOUN_POS for k in range(len(parts)) ) if has_noun: candidates.append((phrase, i)) # Advance: skip processed content tokens i += 1 # step by 1 to allow overlapping phrases # Count occurrences counter = Counter(p for p, _ in candidates) total = max(1, sum(counter.values())) results: list[TermEntry] = [] for phrase, count in counter.most_common(top_n * 3): if count < 1: break # Frequency score * length bonus (longer = more specific) length_bonus = min(2.0, 1.0 + max(0, len(phrase) - 4) * 0.1) score = (count / total) * length_bonus results.append(TermEntry( term=phrase, frequency=count, score=score, word_type="phrase", )) return results except Exception: return [] # ════════════════════════════════════════════════════════════════════ # Method 4: Regex-based proper noun extraction # ════════════════════════════════════════════════════════════════════ def _extract_proper_nouns(text: str) -> list[TermEntry]: """Extract proper nouns: capitalized words, acronyms, numbers+units.""" results: list[TermEntry] = [] # Capitalized multi-word phrases (e.g. "Machine Learning", "United Nations") capitalized = re.findall(r"\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b", text) for phrase in set(capitalized): results.append(TermEntry(term=phrase, score=0.5, word_type="proper_noun")) # PascalCase / camelCase identifiers (e.g. WebApplicationFactory) pascal = re.findall(r"\b(?:[A-Z][a-z]+){2,}\b|\b[A-Z][a-zA-Z]*[a-z][A-Z][a-zA-Z]*\b", text) for p in set(pascal): if len(p) > 4: results.append(TermEntry(term=p, score=0.5, word_type="proper_noun")) # Acronyms (2-5 uppercase letters) acronyms = re.findall(r"\b([A-Z]{2,5})\b", text) for acr in set(acronyms): results.append(TermEntry(term=acr, score=0.6, word_type="proper_noun")) # English multi-word terms in Chinese text (e.g. "deep learning", "machine translation") # Only use when text contains Chinese — for pure English, this regex produces garbage cn_count = len(re.findall(r"[\u4e00-\u9fff]", text)) if cn_count > 0: en_phrases = re.findall(r"\b([A-Za-z][a-z]+(?:\s+[a-z]+){1,3})\b", text) for phrase in set(en_phrases): # Only keep reasonably short, meaningful phrases (not sentence fragments) if 6 < len(phrase) < 40 and phrase.count(' ') <= 3: # Filter common stop phrases low = phrase.lower() if not any(w in low for w in (' the ', ' a ', ' an ', ' is ', ' are ', ' was ', ' were ', ' has ', ' have ', ' had ', ' and ', ' for ', ' with ', ' that ', ' this ', ' their ', ' they ', ' shows ', ' plays ', ' role ')): results.append(TermEntry(term=phrase, word_type="proper_noun")) else: # Pure English text: only extract true multi-word proper nouns # (already handled by capitalized pattern above) pass return results # ════════════════════════════════════════════════════════════════════ # Method 5: pke_zh PositionRank (Chinese keyphrase extraction) # ════════════════════════════════════════════════════════════════════ def _extract_pke_zh(text: str, top_n: int) -> list[TermEntry]: """Extract Chinese keyphrases using pke_zh PositionRank. PositionRank is a graph-based algorithm that considers both word frequency and position in the document. It produces better multi-word phrases than pure TF-IDF or TextRank. """ try: from pke_zh import PositionRank m = PositionRank() results_raw = m.extract(text) results: list[TermEntry] = [] for word, score in results_raw[:top_n]: if len(word) >= 2: # Normalize score (PositionRank scores are typically 0-1) results.append(TermEntry( term=word, score=min(1.0, score) * 0.95, word_type="positionrank", )) return results except Exception: return [] # ════════════════════════════════════════════════════════════════════ # Method 6: YAKE (English keyphrase extraction) # ════════════════════════════════════════════════════════════════════ _EN_STOP_WORDS = frozenset({ 'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'shall', 'should', 'may', 'might', 'must', 'can', 'could', 'i', 'me', 'my', 'we', 'our', 'you', 'your', 'he', 'she', 'it', 'they', 'them', 'this', 'that', 'these', 'those', 'in', 'on', 'at', 'to', 'for', 'of', 'from', 'by', 'with', 'about', 'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'between', 'under', 'and', 'but', 'or', 'nor', 'not', 'so', 'yet', 'if', 'than', 'too', 'very', 'just', 'also', 'now', 'then', 'here', 'there', 'all', 'each', 'every', 'both', 'few', 'more', 'most', 'other', 'some', 'such', 'only', 'own', 'same', 'its', 'his', 'her', 'their', 'our', 'no', 'up', 'out', 'over', 'down', 'off', }) def _extract_yake(text: str, top_n: int) -> list[TermEntry]: """Extract English keyphrases using YAKE algorithm. YAKE is an unsupervised, language-independent keyphrase extraction method that uses statistical features (casing, position, frequency, relatedness, dispersion) to score candidate phrases. Falls back to a simple n-gram approach if YAKE is unavailable. """ try: import yake # YAKE parameters kw_extractor = yake.KeywordExtractor( lan="en", n=3, # max 3-gram for better phrase coverage dedupLim=0.65, # more permissive dedup top=top_n * 2, # get more candidates for filtering features=None, ) raw = kw_extractor.extract_keywords(text) results: list[TermEntry] = [] # YAKE returns (phrase, score) where lower = better # Normalize to 0-1 range (higher = better) if not raw: return results scores = [s for _, s in raw] min_s, max_s = min(scores), max(scores) score_range = max_s - min_s if max_s > min_s else 1.0 for phrase, score in raw[:top_n]: phrase = phrase.strip() if len(phrase) < 3: continue # Require at least 2 content words for English phrases if len(phrase.split()) < 2: continue # Filter grammatical fragments if _is_grammar_fragment(phrase): continue normalized = 1.0 - (score - min_s) / score_range # Boost terms found in local domain dictionary boost = _get_term_boost(phrase) results.append(TermEntry( term=phrase, score=round(normalized * 0.85 * boost, 4), word_type="yake", )) return results except Exception: return _fallback_en_phrases(text, top_n) def _fallback_en_phrases(text: str, top_n: int) -> list[TermEntry]: """Fallback English phrase extraction without YAKE.""" import re from collections import Counter # Tokenize into words words = re.findall(r"[a-zA-Z]{2,}", text.lower()) if len(words) < 3: return [] # Build 2-4 word n-grams skipping stop words at edges candidates: list[tuple[str, float]] = [] for n in (2, 3, 4): for i in range(len(words) - n + 1): ngram = words[i:i + n] # Must not start or end with stop word if ngram[0] in _EN_STOP_WORDS or ngram[-1] in _EN_STOP_WORDS: continue # Must not have too many stop words stop_count = sum(1 for w in ngram if w in _EN_STOP_WORDS) if stop_count > n // 3: continue phrase = " ".join(ngram) candidates.append((phrase, n * 0.1)) # length bonus if not candidates: return [] counter = Counter(p for p, _ in candidates) total = max(1, sum(counter.values())) results: list[TermEntry] = [] for phrase, count in counter.most_common(top_n): if count < 2: continue score = (count / total) * (1 + min(0.3, len(phrase.split()) * 0.1)) results.append(TermEntry( term=phrase, frequency=count, score=score, word_type="en-ngram", )) return results # ════════════════════════════════════════════════════════════════════ # Method 7: C-Value (terminology extraction with nesting awareness) # ════════════════════════════════════════════════════════════════════ def _extract_cvalue(text: str, top_n: int, is_cn: bool) -> list[TermEntry]: """Extract terms using the C-Value algorithm. C-Value is an established algorithm that: 1. Extracts candidate noun phrases 2. Scores by frequency × log₂(length) 3. Penalizes nested sub-terms (e.g., "检查点" nested in "免疫检查点抑制剂") 4. Rewards context-independent terms This produces significantly cleaner results than pure frequency-based methods. """ import math from collections import defaultdict # Get candidate phrases (noun phrases or POS-chunked phrases) candidates_raw: list[str] = [] if is_cn: from termprep.extractor import _extract_phrases as _phrases for t in _phrases(text, top_n * 3): candidates_raw.append(t.term) else: # English: use YAKE's phrase extraction try: import yake kw = yake.KeywordExtractor(lan="en", n=2, dedupLim=0.8, top=top_n * 2) for phrase, _ in kw.extract_keywords(text): phrase = phrase.strip() if len(phrase) > 3 and len(phrase.split()) >= 2: candidates_raw.append(phrase) except Exception: pass if not candidates_raw: return [] # Count frequencies of all candidate phrases freq: dict[str, int] = defaultdict(int) for phrase in candidates_raw: freq[phrase] += 1 # Build nesting relationships # For CN: character-level containment. For EN: word-level containment. def is_nested(shorter: str, longer: str) -> bool: if len(shorter) >= len(longer): return False if is_cn: return shorter in longer else: sw = set(shorter.lower().split()) lw = set(longer.lower().split()) return sw.issubset(lw) and shorter.lower() != longer.lower() # C-Value scoring all_phrases = list(freq.keys()) scores: dict[str, float] = {} for phrase in all_phrases: word_count = len(phrase) if is_cn else len(phrase.split()) f_a = freq[phrase] # Find longer phrases that contain this one containing = [p for p in all_phrases if is_nested(phrase, p)] containing_freq_sum = sum(freq[p] for p in containing) if not containing: # Not nested: score = log2(word_count) * frequency scores[phrase] = math.log2(max(2, word_count)) * f_a / max(1, len(phrase) / 4) else: # Nested: penalize c = len(containing) scores[phrase] = math.log2(max(2, word_count)) * (f_a - containing_freq_sum / c) # Build results, filter very low scores min_score = max(0.01, max(scores.values()) * 0.05) if scores else 0 results: list[TermEntry] = [] for phrase, score in sorted(scores.items(), key=lambda x: -x[1]): if score < min_score: continue if len(phrase) < 3: continue results.append(TermEntry( term=phrase, frequency=freq[phrase], score=min(1.0, score / max(1, max(scores.values()))), word_type="cvalue", )) if len(results) >= top_n: break return results def _is_grammar_fragment(phrase: str) -> bool: """Filter out grammatical fragments that aren't real terms.""" words = phrase.lower().split() if not words: return True # Phrases that start/end with prepositions or auxiliary verbs start_stop = {'the', 'a', 'an', 'in', 'on', 'at', 'to', 'for', 'of', 'by', 'with', 'and', 'or', 'is', 'are', 'was', 'were', 'be', 'been', 'versus', 'vs', 'per'} end_stop = {'the', 'a', 'an', 'in', 'on', 'at', 'to', 'for', 'of', 'by', 'with', 'and', 'or', 'is', 'are', 'was', 'were', 'be', 'been'} if words[0] in start_stop: return True if words[-1] in end_stop: return True # Internal prepositions/conjunctions usually indicate a fragment internal_stop = {' of ', ' and ', ' or ', ' but ', ' with ', ' for ', ' in ', ' on ', ' at ', ' by ', ' to ', ' from ', ' as ', ' that ', ' which ', ' who ', ' when ', ' where ', ' like '} lowered = ' ' + phrase.lower() + ' ' if any(token in lowered for token in internal_stop): return True # Verb + preposition fragments like "trial examined", "patients receiving" verb_patterns = {'examined', 'receiving', 'using', 'showed', 'compared', 'treated', 'demonstrated', 'shows', 'plays', 'includes', 'including', 'making', 'taken', 'given', 'found', 'generate', 'revolutionized', 'enabling', 'achieved', 'require', 'requires', 'required', 'requiring', 'learn', 'learns', 'learned', 'learning', 'achieve', 'achieves'} if len(words) >= 2 and words[-1] in verb_patterns: return True # Phrases starting with common verbs are usually sentence fragments start_verbs = {'is', 'are', 'was', 'were', 'has', 'have', 'had', 'can', 'could', 'will', 'would', 'should', 'may', 'might', 'must', 'shall', 'does', 'did', 'do', 'generates', 'revolutionized', 'enables', 'allows', 'makes', 'takes', 'gives', 'shows', 'achieved', 'understand', 'generate', 'enabled', 'learning', 'requires', 'required', 'achieve', 'requiring', 'learned', 'learns', 'require'} if words[0] in start_verbs: return True return False def _get_term_boost(phrase: str) -> float: """Boost score for terms validated by the local domain dictionary. Avoids network calls during the extraction hot path; Wikidata validation is available separately via termbase.lookup_term(). """ try: from termprep.termbase import DOMAIN_TERMS low = phrase.lower() for domain, terms in DOMAIN_TERMS.items(): if any(t.lower() == low for t in terms): return 1.3 # 30% boost for locally verified terms except Exception: pass return 1.0 # ════════════════════════════════════════════════════════════════════ # Merge & Rank # ════════════════════════════════════════════════════════════════════ def _merge_and_rank(terms: list[TermEntry], top_n: int) -> list[TermEntry]: """Merge duplicate terms across extraction methods, remove overlaps, and rank. Strategy: - Normalize keys by lowercasing and strip whitespace - Filter out very short single English words (uninformative) - Remove subsumed terms: if a longer phrase fully contains a shorter one, keep only the longer, more complete phrase - Sum frequencies and scores for merged terms - Prefer more descriptive word_type labels """ TYPE_PRIORITY = {"phrase": 0, "proper_noun": 1, "positionrank": 2, "cvalue": 3, "textrank": 4, "keyword": 5, "yake": 6, "bigram": 7, "en-ngram": 8} # Step 1: Merge duplicates by normalized key merged: dict[str, TermEntry] = {} for t in terms: key = t.term.lower().strip() if not key or len(key) < 2: continue # Skip single English words that are not acronyms (uninformative as terms) if len(key.split()) == 1 and not re.search(r'[\u4e00-\u9fff]', key): # Keep acronyms like BERT, GPT, API (2-5 uppercase chars in original) original = t.term.strip() if len(original) >= 2 and len(original) <= 5 and original.isupper(): pass # keep acronym else: continue # filter common single words like "machine", "learning" if key in merged: existing = merged[key] existing.score += t.score existing.frequency += t.frequency if TYPE_PRIORITY.get(t.word_type, 99) < TYPE_PRIORITY.get(existing.word_type, 99): existing.word_type = t.word_type else: merged[key] = t # Step 2: Remove subsumed terms (keep the longer, more complete phrase) sorted_keys = sorted(merged.keys(), key=lambda k: len(k), reverse=True) to_remove: set[str] = set() for long_key in sorted_keys: if long_key in to_remove: continue long_words = long_key.split() for short_key in sorted_keys: if short_key == long_key or short_key in to_remove: continue if len(short_key) >= len(long_key): continue # Check containment if ' ' in long_key or ' ' in short_key: # English / multi-word: word-level containment (in order) short_words = short_key.split() if len(short_words) < 2 and len(long_words) <= 2: # Don't remove single-word terms from 2-word phrases # e.g., keep "machine" even if "machine learning" exists continue # Check if all short words appear in long words in order try: idx = 0 for sw in short_words: idx = long_words.index(sw, idx) + 1 # All short words found in order in long words to_remove.add(short_key) except ValueError: pass else: # Chinese / no spaces: character-level containment if short_key in long_key and len(long_key) - len(short_key) >= 2: to_remove.add(short_key) # Remove subsumed terms for k in to_remove: if k in merged: del merged[k] # Step 2b: Filter grammar fragments from all extractors (not just YAKE) merged = {k: v for k, v in merged.items() if not _is_grammar_fragment(v.term)} # Step 3: Normalize scores to 0-1 range if merged: all_scores = [t.score for t in merged.values()] max_sc = max(all_scores) if max_sc > 0: for t in merged.values(): t.score = round(t.score / max_sc, 4) sorted_terms = sorted(merged.values(), key=lambda x: (x.score, -len(x.term)), reverse=True) return sorted_terms[:top_n] # ════════════════════════════════════════════════════════════════════ # Diagnostics # ════════════════════════════════════════════════════════════════════ def get_frequency_table(terms: list[TermEntry]) -> str: """Format terms as a frequency table string.""" lines = [f"{'Term':<24} {'Freq':>6} {'Score':>8} {'Type':>14}"] lines.append("-" * 56) for t in terms: lines.append( f"{t.term:<24} {t.frequency:>6} {t.score:>8.4f} {t.word_type:>14}" ) return "\n".join(lines) def debug_extraction(text: str, top_n: int = 10) -> str: """Return a diagnostic string showing what each method extracted.""" parts: list[str] = [] parts.append("═══ TF-IDF ═══") for t in _extract_jieba(text, top_n): parts.append(f" {t.term:20s} score={t.score:.4f}") parts.append("═══ TextRank ═══") for t in _extract_textrank(text, top_n): parts.append(f" {t.term:20s} score={t.score:.4f}") parts.append("═══ POS Phrases ═══") for t in _extract_phrases(text, top_n): parts.append(f" {t.term:20s} freq={t.frequency} score={t.score:.4f}") parts.append("═══ Proper Nouns ═══") for t in _extract_proper_nouns(text): parts.append(f" {t.term:20s} type={t.word_type}") parts.append("═══ Merged Top-{top_n} ═══") for t in extract(text, top_n): parts.append(f" {t.term:20s} score={t.score:.4f} type={t.word_type}") return "\n".join(parts)