app/analyzers/similarity.py

"""
Semantic similarity via TF-IDF + cosine (scikit-learn).
Cari pasangan artikel mirip secara makna.
"""
from typing import List, Dict
import re

INDO_STOPWORDS = {
    "yang", "di", "ke", "dari", "untuk", "pada", "dengan", "ini", "itu", "dan",
    "atau", "adalah", "akan", "juga", "tidak", "para", "oleh", "sebagai",
}


def _clean(text: str) -> str:
    text = re.sub(r"[^a-zA-Z\s]", " ", text.lower())
    return re.sub(r"\s+", " ", text).strip()


def find_similar_pairs(items: List, threshold: float = 0.3) -> List[Dict]:
    if len(items) < 2:
        return []

    from sklearn.feature_extraction.text import TfidfVectorizer
    from sklearn.metrics.pairwise import cosine_similarity

    docs = [_clean(it.text) for it in items]
    ids = [it.id for it in items]

    vectorizer = TfidfVectorizer(max_features=3000, stop_words=list(INDO_STOPWORDS))
    try:
        X = vectorizer.fit_transform(docs)
    except ValueError:
        return []

    sim = cosine_similarity(X)
    pairs = []
    n = len(ids)
    for i in range(n):
        for j in range(i + 1, n):
            score = float(sim[i, j])
            if score >= threshold:
                pairs.append({"id_a": ids[i], "id_b": ids[j], "score": round(score, 3)})

    pairs.sort(key=lambda p: p["score"], reverse=True)
    return pairs[:500]