app/analyzers/topics.py

"""
Topic modeling via TF-IDF + KMeans (scikit-learn).
Auto-discover topik tanpa kategori predefined.
"""
from typing import List, Dict
import re

INDO_STOPWORDS = {
    "yang", "di", "ke", "dari", "untuk", "pada", "dengan", "ini", "itu", "dan",
    "atau", "adalah", "akan", "juga", "tidak", "para", "oleh", "sebagai",
    "dalam", "tersebut", "ada", "dapat", "bisa", "harus", "lebih", "sangat",
    "telah", "sudah", "masih", "hanya", "saja", "karena", "namun", "tetapi",
    "tapi", "saat", "ketika", "setelah", "sebelum", "antara", "hingga",
    "republika", "okezone", "detik", "kompas", "tribunnews", "cnn", "tempo",
    "antaranews", "antara", "merdeka", "kumparan", "news", "com",
}


def _clean(text: str) -> str:
    text = re.sub(r"[^a-zA-Z\s]", " ", text.lower())
    return re.sub(r"\s+", " ", text).strip()


def discover_topics(items: List, num_topics: int = 8) -> List[Dict]:
    if len(items) < num_topics:
        num_topics = max(2, len(items) // 2)
    if len(items) < 4:
        return []

    from sklearn.feature_extraction.text import TfidfVectorizer
    from sklearn.cluster import KMeans
    import numpy as np

    docs = [_clean(it.text) for it in items]
    ids = [it.id for it in items]

    vectorizer = TfidfVectorizer(
        max_features=2000,
        stop_words=list(INDO_STOPWORDS),
        min_df=2,
        ngram_range=(1, 2),
    )
    try:
        X = vectorizer.fit_transform(docs)
    except ValueError:
        return []

    if X.shape[1] == 0:
        return []

    k = min(num_topics, X.shape[0])
    km = KMeans(n_clusters=k, random_state=42, n_init=10)
    labels = km.fit_predict(X)

    terms = vectorizer.get_feature_names_out()
    order_centroids = km.cluster_centers_.argsort()[:, ::-1]

    topics = []
    for topic_id in range(k):
        keywords = [terms[ind] for ind in order_centroids[topic_id, :8]]
        member_ids = [ids[i] for i in range(len(ids)) if labels[i] == topic_id]
        topics.append({
            "topic_id": topic_id,
            "label": ", ".join(keywords[:3]),
            "keywords": keywords,
            "article_ids": member_ids,
            "size": len(member_ids),
        })

    topics.sort(key=lambda t: t["size"], reverse=True)
    return topics