Spaces:
Running
Running
| from difflib import SequenceMatcher | |
| from src.synonym_data import _load_groups, _find_synonym_group | |
| def _dedup_exact(tags: list[str]) -> list[str]: | |
| seen = set() | |
| out = [] | |
| for t in tags: | |
| k = t.lower().strip() | |
| if k and k not in seen: | |
| seen.add(k) | |
| out.append(t) | |
| return out | |
| def _dedup_synonym_groups(tags: list[str]) -> list[str]: | |
| result = [] | |
| for t in tags: | |
| tl = t.lower().strip() | |
| group = _find_synonym_group(t) | |
| if group is None: | |
| result.append(t) | |
| continue | |
| conflict = False | |
| for existing in result: | |
| el = existing.lower().strip() | |
| if el in group and el != tl: | |
| conflict = True | |
| break | |
| if not conflict: | |
| result.append(t) | |
| return result | |
| def _ngram_similarity(a: str, b: str) -> float: | |
| return SequenceMatcher(None, a.lower(), b.lower()).ratio() | |
| def _is_attribute_variant(a: str, b: str) -> bool: | |
| """Two tags that share a head or a tail but differ in the other part denote | |
| different attributes (e.g. 'blue eyes' vs 'blue hair', 'red dress' vs 'blue | |
| dress') and must both be kept — fuzzy-merging them would drop a distinct | |
| booru attribute.""" | |
| aw = a.lower().split() | |
| bw = b.lower().split() | |
| if len(aw) < 2 or len(bw) < 2 or len(aw) != len(bw): | |
| return False | |
| head_same = aw[:-1] == bw[:-1] | |
| tail_same = aw[-1] == bw[-1] | |
| # exactly one side differs -> different attribute, same concept | |
| return head_same != tail_same | |
| def _dedup_fuzzy(tags: list[str], threshold: float = 0.85) -> list[str]: | |
| # Sort by length so a more specific (longer) tag survives its shorter fuzzy | |
| # twin (e.g. "very long flowing red hair" beats "long red hair"). | |
| sorted_tags = sorted(tags, key=len, reverse=True) | |
| result = [] | |
| for t in sorted_tags: | |
| if len(t) < 4: | |
| result.append(t) | |
| continue | |
| is_dup = False | |
| for existing in result: | |
| if len(existing) < 4: | |
| continue | |
| if _is_attribute_variant(t, existing): | |
| continue | |
| # Cheap upper bound first — a length mismatch alone can decide. | |
| ratio = SequenceMatcher(None, t.lower(), existing.lower()).ratio() | |
| if ratio >= threshold: | |
| is_dup = True | |
| break | |
| if not is_dup: | |
| result.append(t) | |
| return result | |
| def smart_dedup(tags: list[str], model: str = "anima") -> list[str]: | |
| if not tags: | |
| return [] | |
| no_exact = _dedup_exact(tags) | |
| no_synonym = _dedup_synonym_groups(no_exact) | |
| threshold = 0.75 if model == "anima" else 0.85 | |
| no_fuzzy = _dedup_fuzzy(no_synonym, threshold=threshold) | |
| return no_fuzzy | |
| def reload_groups(): | |
| from src.synonym_data import reload_synonym_groups | |
| reload_synonym_groups() | |
| _load_groups() | |