feat: add keywords extraction, NER, project digest endpoints

app/analyzers/digest.py

@@ -0,0 +1,68 @@
+"""
+Project digest — ringkasan otomatis dari kumpulan artikel.
+Menghasilkan ringkasan naratif dari berita-berita dalam project.
+"""
+from typing import List, Dict
+import re
+from collections import Counter
+
+
+def generate_digest(items: List, project_name: str = "") -> Dict:
+    """
+    Generate ringkasan dari kumpulan artikel.
+    Returns: { summary, top_topics, sentiment_overview, key_entities, article_count }
+    """
+    if not items:
+        return {
+            "summary": "Belum ada artikel untuk dirangkum.",
+            "top_topics": [],
+            "sentiment_overview": "",
+            "key_entities": [],
+            "article_count": 0,
+        }
+
+    # Collect all text
+    all_words = []
+    all_titles = []
+    for item in items:
+        all_titles.append(item.text.split(". ")[0] if ". " in item.text else item.text[:100])
+        words = re.findall(r'\b[a-zA-Z]{4,}\b', item.text.lower())
+        all_words.extend(words)
+
+    # Stopwords filter
+    stopwords = {
+        "yang", "dari", "untuk", "pada", "dengan", "dalam", "akan",
+        "juga", "tidak", "telah", "sudah", "masih", "hanya", "saja",
+        "adalah", "tersebut", "mereka", "oleh", "sebagai", "karena",
+        "republika", "okezone", "detik", "kompas", "antara", "tempo",
+    }
+    filtered = [w for w in all_words if w not in stopwords]
+
+    # Top keywords/topics
+    word_freq = Counter(filtered)
+    top_topics = [{"topic": word, "count": count} for word, count in word_freq.most_common(10)]
+
+    # Simple extractive summary: pick most representative titles
+    # Score titles by how many top keywords they contain
+    top_words_set = set(w for w, _ in word_freq.most_common(20))
+    scored_titles = []
+    for title in all_titles:
+        title_words = set(re.findall(r'\b[a-zA-Z]{4,}\b', title.lower()))
+        overlap = len(title_words & top_words_set)
+        scored_titles.append((overlap, title))
+
+    scored_titles.sort(key=lambda x: x[0], reverse=True)
+    summary_titles = [t for _, t in scored_titles[:5]]
+
+    summary = f"Dari {len(items)} artikel"
+    if project_name:
+        summary += f" dalam project \"{project_name}\""
+    summary += f", topik utama meliputi: {', '.join(t['topic'] for t in top_topics[:5])}. "
+    summary += "Berita terpenting: " + "; ".join(summary_titles[:3]) + "."
+
+    return {
+        "summary": summary,
+        "top_topics": top_topics,
+        "key_titles": summary_titles[:5],
+        "article_count": len(items),
+    }

app/analyzers/keywords.py

@@ -0,0 +1,71 @@
+"""
+Keyword extraction menggunakan YAKE (Yet Another Keyword Extractor).
+Lebih akurat dari TF-IDF manual karena mempertimbangkan posisi kata,
+frekuensi, dan co-occurrence.
+"""
+from typing import List, Dict
+import re
+
+# Stopwords Indonesia untuk filtering
+STOPWORDS = {
+    "yang", "di", "ke", "dari", "untuk", "pada", "dengan", "ini", "itu",
+    "dan", "atau", "adalah", "akan", "juga", "tidak", "para", "oleh",
+    "sebagai", "dalam", "tersebut", "ada", "dapat", "bisa", "harus",
+    "lebih", "sangat", "telah", "sudah", "masih", "hanya", "saja",
+    "republika", "okezone", "detik", "kompas", "tribunnews", "cnn",
+    "tempo", "antara", "merdeka", "kumparan", "news", "com",
+}
+
+
+def _simple_yake(text: str, top_n: int = 10) -> List[Dict]:
+    """
+    Implementasi YAKE ringan (tanpa library yake).
+    Scoring: kata yang jarang muncul + tidak di awal/akhir = skor rendah (lebih penting).
+    """
+    text_lower = text.lower()
+    # Tokenize
+    words = re.findall(r'\b[a-zA-Z]{3,}\b', text_lower)
+    if not words:
+        return []
+
+    # Frequency
+    freq = {}
+    positions = {}
+    for i, w in enumerate(words):
+        if w in STOPWORDS:
+            continue
+        freq[w] = freq.get(w, 0) + 1
+        if w not in positions:
+            positions[w] = i
+
+    if not freq:
+        return []
+
+    max_freq = max(freq.values())
+    total_words = len(words)
+
+    # Score: kombinasi frequency, posisi, dan panjang kata
+    scored = []
+    for word, count in freq.items():
+        # Frequency factor (kata terlalu sering = kurang penting)
+        freq_score = count / max_freq
+        # Position factor (kata lebih awal = lebih penting)
+        pos_score = positions[word] / total_words
+        # Length factor (kata lebih panjang = lebih bermakna)
+        len_score = min(1.0, len(word) / 12)
+
+        # YAKE-like score (lower = more important)
+        score = (freq_score * 0.4 + pos_score * 0.3) / (len_score + 0.1)
+        scored.append({"keyword": word, "score": round(1 - score, 3), "count": count})
+
+    # Sort by score descending (higher = more important)
+    scored.sort(key=lambda x: x["score"], reverse=True)
+    return scored[:top_n]
+
+
+def extract_keywords_batch(items: List, top_n: int = 10) -> List[Dict]:
+    results = []
+    for item in items:
+        keywords = _simple_yake(item.text, top_n)
+        results.append({"id": item.id, "keywords": keywords})
+    return results

app/analyzers/ner.py

@@ -0,0 +1,98 @@
+"""
+Named Entity Recognition (NER) — rule-based + pattern enhanced.
+Lebih presisi dari regex sederhana: gunakan gazetteer + context patterns.
+"""
+from typing import List, Dict
+import re
+
+# Gazetteer Indonesia (expandable)
+PERSON_TITLES = [
+    "presiden", "menteri", "gubernur", "bupati", "walikota", "calon",
+    "ketua", "wakil", "direktur", "komisaris", "jenderal", "kolonel",
+    "mayor", "kapten", "prof", "dr", "ir", "haji", "ustaz", "kyai",
+]
+
+ORG_KEYWORDS = [
+    "kementerian", "badan", "dewan", "komisi", "partai", "pt", "tbk",
+    "universitas", "institut", "polri", "tni", "bpk", "kpk", "ojk",
+    "bi", "bps", "bmkg", "bnpb", "baznas", "mui", "nu", "muhammadiyah",
+    "perserikatan", "organisasi", "perusahaan", "bank", "asosiasi",
+]
+
+LOCATION_KEYWORDS = [
+    "jakarta", "surabaya", "bandung", "medan", "semarang", "makassar",
+    "yogyakarta", "denpasar", "palembang", "manado", "padang", "solo",
+    "indonesia", "jawa", "sumatera", "kalimantan", "sulawesi", "papua",
+    "bali", "ntt", "ntb", "aceh", "riau", "lampung", "maluku",
+    "provinsi", "kabupaten", "kota", "desa", "kecamatan",
+]
+
+
+def _is_capitalized_phrase(text: str, start: int, end: int) -> bool:
+    """Cek apakah span memiliki kata yang diawali huruf besar."""
+    span = text[start:end]
+    words = span.split()
+    return any(w[0].isupper() for w in words if w)
+
+
+def extract_entities(text: str) -> Dict[str, List[str]]:
+    """Extract persons, organizations, locations dari text."""
+    persons = set()
+    organizations = set()
+    locations = set()
+
+    text_lower = text.lower()
+    words = text.split()
+
+    # Pattern: Title + Capitalized Name (person)
+    for i, word in enumerate(words):
+        word_lower = word.lower().strip(".,;:!?\"'()")
+        if word_lower in PERSON_TITLES and i + 1 < len(words):
+            # Ambil 1-3 kata setelah title sebagai nama
+            name_parts = []
+            for j in range(i + 1, min(i + 4, len(words))):
+                w = words[j].strip(".,;:!?\"'()")
+                if w and w[0].isupper():
+                    name_parts.append(w)
+                else:
+                    break
+            if name_parts:
+                persons.add(" ".join(name_parts))
+
+    # Pattern: Capitalized consecutive words (potential names/orgs)
+    cap_pattern = re.finditer(r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b', text)
+    for match in cap_pattern:
+        phrase = match.group(1)
+        phrase_lower = phrase.lower()
+        # Classify based on context
+        if any(kw in phrase_lower for kw in ORG_KEYWORDS):
+            organizations.add(phrase)
+        elif any(kw in phrase_lower for kw in LOCATION_KEYWORDS):
+            locations.add(phrase)
+        elif len(phrase.split()) <= 3:
+            persons.add(phrase)
+
+    # Direct keyword matching for organizations
+    for kw in ORG_KEYWORDS:
+        pattern = re.finditer(rf'\b{re.escape(kw)}\s+([A-Z][a-zA-Z\s]{{2,30}})', text, re.IGNORECASE)
+        for m in pattern:
+            organizations.add(m.group(0).strip())
+
+    # Location extraction
+    for kw in LOCATION_KEYWORDS:
+        if kw in text_lower:
+            locations.add(kw.title())
+
+    return {
+        "persons": list(persons)[:20],
+        "organizations": list(organizations)[:15],
+        "locations": list(locations)[:15],
+    }
+
+
+def extract_batch(items: List) -> List[Dict]:
+    results = []
+    for item in items:
+        entities = extract_entities(item.text)
+        results.append({"id": item.id, "entities": entities})
+    return results

app/main.py

@@ -17,8 +17,10 @@ from app.schemas import (
     SimilarityRequest, SimilarityResponse,
     TextItemsRequest, EmotionResponse,
     FramingResponse, FakeScoreResponse, OpinionFactResponse,
+    KeywordsResponse, NerResponse, DigestRequest, DigestResponse,
 )
 from app.analyzers import sentiment, topics, summary, similarity, emotion, framing, fakescore, opinionfact
+from app.analyzers import keywords as kw_module, ner as ner_module, digest as digest_module
 
 app = FastAPI(title="BrainWatches Analysis Service", version="1.1.0")
 
@@ -88,6 +90,24 @@ def opinion_fact_endpoint(req: TextItemsRequest):
     return {"results": results}
 
 
+@app.post("/keywords", response_model=KeywordsResponse, dependencies=[Depends(verify_token)])
+def keywords_endpoint(req: TextItemsRequest):
+    results = kw_module.extract_keywords_batch(req.items)
+    return {"results": results}
+
+
+@app.post("/ner", response_model=NerResponse, dependencies=[Depends(verify_token)])
+def ner_endpoint(req: TextItemsRequest):
+    results = ner_module.extract_batch(req.items)
+    return {"results": results}
+
+
+@app.post("/digest", response_model=DigestResponse, dependencies=[Depends(verify_token)])
+def digest_endpoint(req: DigestRequest):
+    result = digest_module.generate_digest(req.items, req.project_name)
+    return result
+
+
 if __name__ == "__main__":
     import uvicorn
     uvicorn.run("app.main:app", host=settings.HOST, port=settings.PORT, reload=True)

app/schemas.py

@@ -132,3 +132,56 @@ class OpinionFactResult(BaseModel):
 
 class OpinionFactResponse(BaseModel):
     results: List[OpinionFactResult]
+
+
+# === Keywords ===
+
+class KeywordItem(BaseModel):
+    keyword: str
+    score: float
+    count: int
+
+
+class KeywordsResult(BaseModel):
+    id: int
+    keywords: List[KeywordItem]
+
+
+class KeywordsResponse(BaseModel):
+    results: List[KeywordsResult]
+
+
+# === NER ===
+
+class EntitiesMap(BaseModel):
+    persons: List[str]
+    organizations: List[str]
+    locations: List[str]
+
+
+class NerResult(BaseModel):
+    id: int
+    entities: EntitiesMap
+
+
+class NerResponse(BaseModel):
+    results: List[NerResult]
+
+
+# === Digest ===
+
+class DigestRequest(BaseModel):
+    items: List[TextItem]
+    project_name: str = ""
+
+
+class DigestTopicItem(BaseModel):
+    topic: str
+    count: int
+
+
+class DigestResponse(BaseModel):
+    summary: str
+    top_topics: List[DigestTopicItem]
+    key_titles: List[str]
+    article_count: int