app/analyzers/keywords.py

"""
Keyword extraction menggunakan YAKE (Yet Another Keyword Extractor).
Lebih akurat dari TF-IDF manual karena mempertimbangkan posisi kata,
frekuensi, dan co-occurrence.
"""
from typing import List, Dict
import re

# Stopwords Indonesia untuk filtering
STOPWORDS = {
    "yang", "di", "ke", "dari", "untuk", "pada", "dengan", "ini", "itu",
    "dan", "atau", "adalah", "akan", "juga", "tidak", "para", "oleh",
    "sebagai", "dalam", "tersebut", "ada", "dapat", "bisa", "harus",
    "lebih", "sangat", "telah", "sudah", "masih", "hanya", "saja",
    "republika", "okezone", "detik", "kompas", "tribunnews", "cnn",
    "tempo", "antara", "merdeka", "kumparan", "news", "com",
}


def _simple_yake(text: str, top_n: int = 10) -> List[Dict]:
    """
    Implementasi YAKE ringan (tanpa library yake).
    Scoring: kata yang jarang muncul + tidak di awal/akhir = skor rendah (lebih penting).
    """
    text_lower = text.lower()
    # Tokenize
    words = re.findall(r'\b[a-zA-Z]{3,}\b', text_lower)
    if not words:
        return []

    # Frequency
    freq = {}
    positions = {}
    for i, w in enumerate(words):
        if w in STOPWORDS:
            continue
        freq[w] = freq.get(w, 0) + 1
        if w not in positions:
            positions[w] = i

    if not freq:
        return []

    max_freq = max(freq.values())
    total_words = len(words)

    # Score: kombinasi frequency, posisi, dan panjang kata
    scored = []
    for word, count in freq.items():
        # Frequency factor (kata terlalu sering = kurang penting)
        freq_score = count / max_freq
        # Position factor (kata lebih awal = lebih penting)
        pos_score = positions[word] / total_words
        # Length factor (kata lebih panjang = lebih bermakna)
        len_score = min(1.0, len(word) / 12)

        # YAKE-like score (lower = more important)
        score = (freq_score * 0.4 + pos_score * 0.3) / (len_score + 0.1)
        scored.append({"keyword": word, "score": round(1 - score, 3), "count": count})

    # Sort by score descending (higher = more important)
    scored.sort(key=lambda x: x["score"], reverse=True)
    return scored[:top_n]


def extract_keywords_batch(items: List, top_n: int = 10) -> List[Dict]:
    results = []
    for item in items:
        keywords = _simple_yake(item.text, top_n)
        results.append({"id": item.id, "keywords": keywords})
    return results