Update botsignal.py

botsignal.py

@@ -35,6 +35,7 @@ SUPPORT_CHATS = [
     "https://t.me/TheDonALPHAJournal",
     "https://t.me/savascalls",
     "https://t.me/Tanjirocall",
+    "https://t.me/Zen_call",
     "https://t.me/ChapoInsider",
     "https://t.me/millionsgems",
     "https://t.me/Milagrosdegencalls",
@@ -93,6 +94,8 @@ def build_client() -> TelegramClient:
 client = build_client()
 recent_hashes: deque[str] = deque(maxlen=DEDUP_BUFFER_SIZE)
 recent_content_hashes: deque[str] = deque(maxlen=DEDUP_BUFFER_SIZE)  # content-only dedup
+# New: entity-based dedup (CA/$ticker)
+recent_entity_keys: deque[str] = deque(maxlen=DEDUP_BUFFER_SIZE)
 
 # Peta id_chat -> "core" / "support"
 chat_roles: Dict[int, str] = {}  # diisi saat startup setelah resolve entity
@@ -109,7 +112,8 @@ def _db():
 
 def _init_db():
     conn = _db()
-    conn.executescript("""
+    conn.executescript(
+        """
     CREATE TABLE IF NOT EXISTS last_posted (
         keyword TEXT PRIMARY KEY,
         msg_id INTEGER NOT NULL,
@@ -121,7 +125,8 @@ def _init_db():
         last_ts INTEGER NOT NULL,
         PRIMARY KEY (keyword, group_key)
     );
-    """)
+    """
+    )
     conn.commit()
     conn.close()
 
@@ -139,17 +144,21 @@ def db_load_state():
 
 def db_save_last_posted(keyword: str, msg_id: int, tier: str):
     conn = _db()
-    conn.execute("INSERT INTO last_posted(keyword, msg_id, tier) VALUES(?,?,?) "
-                 "ON CONFLICT(keyword) DO UPDATE SET msg_id=excluded.msg_id, tier=excluded.tier",
-                 (keyword, msg_id, tier))
+    conn.execute(
+        "INSERT INTO last_posted(keyword, msg_id, tier) VALUES(?,?,?) "
+        "ON CONFLICT(keyword) DO UPDATE SET msg_id=excluded.msg_id, tier=excluded.tier",
+        (keyword, msg_id, tier),
+    )
     conn.commit()
     conn.close()
 
 def db_upsert_kw_seen(keyword: str, group_key: str, ts: datetime):
     conn = _db()
-    conn.execute("INSERT INTO kw_group_seen(keyword, group_key, last_ts) VALUES(?,?,?) "
-                 "ON CONFLICT(keyword, group_key) DO UPDATE SET last_ts=excluded.last_ts",
-                 (keyword, group_key, int(ts.timestamp())))
+    conn.execute(
+        "INSERT INTO kw_group_seen(keyword, group_key, last_ts) VALUES(?,?,?) "
+        "ON CONFLICT(keyword, group_key) DO UPDATE SET last_ts=excluded.last_ts",
+        (keyword, group_key, int(ts.timestamp())),
+    )
     conn.commit()
     conn.close()
 
@@ -177,19 +186,21 @@ def _tokenize_words(s: str) -> List[str]:
 
 def _windows(tokens: List[str], size: int = 20):
     for i in range(0, len(tokens), size):
-        yield " ".join(tokens[i:i+size])
+        yield " ".join(tokens[i : i + size])
 
 # --- Tambahan: bersihkan URL/CA untuk kepentingan SCORING relevansi ---
 CA_SOL_RE = re.compile(r"\b[1-9A-HJ-NP-Za-km-z]{32,48}\b")  # Solana base58 (perkiraan)
-CA_EVM_RE = re.compile(r"\b0x[a-fA-F0-9]{40}\b")            # EVM address
+CA_EVM_RE = re.compile(r"\b0x[a-fA-F0-9]{40}\b")  # EVM address
 CA_LABEL_RE = re.compile(r"\bCA\s*[:=]\s*\S+", re.IGNORECASE)  # "CA: ..." potong tokennya
 
+
 def _strip_urls_and_mentions(s: str) -> str:
     s = re.sub(r"https?://\S+", "", s)
     s = re.sub(r"t\.me/[A-Za-z0-9_]+", "", s)
     s = re.sub(r"@[A-Za-z0-9_]+", "", s)
     return re.sub(r"\s+", " ", s).strip()
 
+
 def strip_contracts_for_scoring(s: str) -> str:
     """
     Hilangkan URL/mention, alamat kontrak, dan token setelah 'CA:'
@@ -201,6 +212,7 @@ def strip_contracts_for_scoring(s: str) -> str:
     s3 = CA_SOL_RE.sub(" ", s2)
     return re.sub(r"\s+", " ", s3).strip()
 
+
 def score_relevance(text: str, keywords: List[str]) -> float:
     """Skor: exact keyword + fuzzy windowed (top-3 rata-rata) agar adil untuk teks panjang."""
     if not text:
@@ -233,6 +245,7 @@ def score_relevance(text: str, keywords: List[str]) -> float:
 
     return exact_score + fuzzy_score
 
+
 def hash_for_dedup(text: str, msg) -> str:
     """Hash campuran (lama) – menahan duplikat per pesan+media."""
     parts = [text or ""]
@@ -249,37 +262,17 @@ def hash_for_dedup(text: str, msg) -> str:
     raw = "|".join(parts).encode("utf-8", errors="ignore")
     return hashlib.sha1(raw).hexdigest()
 
+
 def content_only_hash(text: str) -> str:
     """Hash berbasis isi saja (untuk lintas-grup crosspost)."""
     norm = _strip_urls_and_mentions(normalize_for_filter(text))
     return hashlib.sha1(norm.encode("utf-8", errors="ignore")).hexdigest()
 
-def is_image_message(msg) -> bool:
-    if getattr(msg, "photo", None) is not None:
-        return True
-    doc = getattr(msg, "document", None)
-    if doc and getattr(doc, "mime_type", None):
-        mt = (doc.mime_type or "").lower()
-        if mt.startswith("image/"):
-            if SKIP_STICKERS and ("webp" in mt or "sticker" in mt):
-                return False
-            return True
-        if not ALLOW_GIFS_VIDEOS:
-            return False
-        if mt in ("video/mp4", "image/gif"):
-            return True
-    return False
-
-def media_too_big(msg) -> bool:
-    doc = getattr(msg, "document", None)
-    if doc and getattr(doc, "size", None):
-        return (doc.size or 0) > MAX_MEDIA_MB * 1024 * 1024
-    return False
-
 
 # ========= Class aggregator (windowed unique groups) =========
 keyword_group_last_seen: defaultdict[str, dict[str, datetime]] = defaultdict(dict)
 
+
 def _prune_expired(now: datetime) -> None:
     window = timedelta(minutes=CLASS_WINDOW_MINUTES)
     cutoff = now - window
@@ -293,6 +286,7 @@ def _prune_expired(now: datetime) -> None:
     # db prune
     db_prune_expired(cutoff)
 
+
 def update_and_classify(keyword: str, group_key: str, now: Optional[datetime] = None) -> Tuple[str, int]:
     if not now:
         now = datetime.now(timezone.utc)
@@ -326,7 +320,7 @@ INVITE_PATTERNS = [
     r"(t\.me\/joinchat|t\.me\/\+|telegram\.me\/|discord\.gg\/|wa\.me\/|whatsapp\.com\/)",
     r"(bit\.ly|tinyurl\.com|linktr\.ee)",
     # perluasan: link t.me biasa
-    r"t\.me\/[A-Za-z0-9_]+"
+    r"t\.me\/[A-Za-z0-9_]+",
 ]
 INVITE_REGEXES = [re.compile(p, re.IGNORECASE) for p in INVITE_PATTERNS]
 
@@ -338,6 +332,7 @@ WHITELIST_STRONG_SIGNAL = [
 ]
 WHITELIST_REGEXES = [re.compile(p, re.IGNORECASE) for p in WHITELIST_STRONG_SIGNAL]
 
+
 def _is_invite_sentence(s: str) -> bool:
     t = s.strip()
     if not t:
@@ -348,10 +343,11 @@ def _is_invite_sentence(s: str) -> bool:
     # Jika ada 1+ pola ajakan, buang
     return any(r.search(t) for r in INVITE_REGEXES)
 
+
 def filter_invite_sentences(text: str) -> str:
     if not text:
         return text
-    parts = re.split(r'(?<=[\.\!\?])\s+|\n+', text, flags=re.UNICODE)
+    parts = re.split(r'(?<=[\.!\?])\s+|\n+', text, flags=re.UNICODE)
     kept = [p.strip() for p in parts if p and not _is_invite_sentence(p)]
     cleaned = "\n".join(kept).strip()
     cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)
@@ -362,6 +358,7 @@ def filter_invite_sentences(text: str) -> str:
 TIER_ORDER = {"rendah": 0, "sedang": 1, "kuat": 2}
 last_posted: Dict[str, Dict[str, object]] = {}  # keyword -> {"msg_id": int, "tier": str}
 
+
 async def _send_initial(msg, text: str) -> int:
     if DRY_RUN:
         print("[DRY_RUN] send_initial:", text[:140])
@@ -370,7 +367,9 @@ async def _send_initial(msg, text: str) -> int:
     if INCLUDE_MEDIA and is_image_message(msg) and not media_too_big(msg):
         try:
             if getattr(msg, "photo", None):
-                m = await client.send_file(TARGET_CHAT, msg.photo, caption=text, caption_entities=None, force_document=False)
+                m = await client.send_file(
+                    TARGET_CHAT, msg.photo, caption=text, caption_entities=None, force_document=False
+                )
                 return m.id
             doc = getattr(msg, "document", None)
             if doc:
@@ -385,7 +384,9 @@ async def _send_initial(msg, text: str) -> int:
                             ext_guess = ".jpg"
                         ext = ext_guess
                     bio.name = f"media{ext}"
-                    m = await client.send_file(TARGET_CHAT, bio, caption=text, caption_entities=None, force_document=False)
+                    m = await client.send_file(
+                        TARGET_CHAT, bio, caption=text, caption_entities=None, force_document=False
+                    )
                     return m.id
         except FloodWaitError as e:
             await asyncio.sleep(e.seconds + 1)
@@ -399,6 +400,7 @@ async def _send_initial(msg, text: str) -> int:
         await asyncio.sleep(e.seconds + 1)
         return await _send_initial(msg, text)
 
+
 async def post_or_update(keyword: str, body: str, new_tier: str, src_msg) -> None:
     prefix = f"[{new_tier.upper()}] "
     text = prefix + body
@@ -445,7 +447,9 @@ async def send_as_is(msg, text_override: Optional[str] = None) -> None:
     if INCLUDE_MEDIA and is_image_message(msg) and not media_too_big(msg):
         try:
             if getattr(msg, "photo", None):
-                await client.send_file(TARGET_CHAT, msg.photo, caption=orig_text, caption_entities=entities, force_document=False)
+                await client.send_file(
+                    TARGET_CHAT, msg.photo, caption=orig_text, caption_entities=entities, force_document=False
+                )
                 return
             doc = getattr(msg, "document", None)
             if doc:
@@ -460,7 +464,9 @@ async def send_as_is(msg, text_override: Optional[str] = None) -> None:
                             ext_guess = ".jpg"
                         ext = ext_guess
                     bio.name = f"media{ext}"
-                    await client.send_file(TARGET_CHAT, bio, caption=orig_text, caption_entities=entities, force_document=False)
+                    await client.send_file(
+                        TARGET_CHAT, bio, caption=orig_text, caption_entities=entities, force_document=False
+                    )
                     return
         except FloodWaitError as e:
             await asyncio.sleep(e.seconds + 1)
@@ -478,6 +484,7 @@ async def send_as_is(msg, text_override: Optional[str] = None) -> None:
 TICKER_CLEAN_RE = re.compile(r"\$[A-Za-z0-9]{2,12}")
 TICKER_NOISY_RE = re.compile(r"\$[A-Za-z0-9](?:[^A-Za-z0-9]+[A-Za-z0-9]){1,11}")
 
+
 def _extract_tickers(text_norm: str) -> List[str]:
     """
     Ambil $TICKER dengan dua cara:
@@ -507,6 +514,7 @@ def _extract_tickers(text_norm: str) -> List[str]:
             seen.add(x)
     return uniq
 
+
 def _extract_all_keywords(text_norm: str) -> List[str]:
     """
     Deteksi SEMUA keyword dari THEME_KEYWORDS + $ticker.
@@ -533,6 +541,7 @@ def _extract_all_keywords(text_norm: str) -> List[str]:
             seen.add(kw)
     return uniq
 
+
 def _choose_dominant_keyword(text_norm: str, kws: List[str]) -> Optional[str]:
     if not kws:
         return None
@@ -547,10 +556,12 @@ def _choose_dominant_keyword(text_norm: str, kws: List[str]) -> Optional[str]:
     chosen = sorted(score.items(), key=lambda x: (x[1][0], x[1][1], x[1][2]), reverse=True)[0][0]
     return chosen
 
+
 def _role_of(chat_id: int) -> str:
     # DEFAULT KE SUPPORT agar tidak salah meloloskan chat yang tidak tertag
     return chat_roles.get(chat_id, "support")
 
+
 def _unique_counts_by_role(keyword: str) -> Tuple[int, int]:
     """
     Hitung jumlah grup unik yang menyebut 'keyword' dalam window aktif,
@@ -564,6 +575,30 @@ def _unique_counts_by_role(keyword: str) -> Tuple[int, int]:
     return len(core_ids), len(sup_ids)
 
 
+# ========= NEW: Entity-key extraction (CA > $ticker) =========
+def extract_entity_key(text: str) -> Optional[str]:
+    """Kembalikan kunci entitas kanonik untuk penentuan 'kesamaan':
+    - Jika ada CA -> 'ca:evm:<0x...>' atau 'ca:sol:<base58>'
+    - Else jika ada $ticker -> 'ticker:<lowercase>'
+    - Else None
+    """
+    t = normalize_for_filter(text)
+
+    # Prefer CA lebih dulu
+    m = CA_EVM_RE.search(t) or CA_SOL_RE.search(t)
+    if m:
+        addr = m.group(0)
+        kind = "evm" if addr.lower().startswith("0x") else "sol"
+        return f"ca:{kind}:{addr.lower()}"
+
+    # Fall back ke $ticker (pakai deteksi yang sudah ada)
+    tickers = _extract_tickers(t.lower())
+    if tickers:
+        return f"ticker:{tickers[0][1:].lower()}"
+
+    return None
+
+
 async def process_message(msg, source_chat_id: int) -> None:
     """
     Filter, content-dedup, relevansi, multi-kw -> pilih dominan,
@@ -578,6 +613,12 @@ async def process_message(msg, source_chat_id: int) -> None:
             debug_log("Dilewati karena EXCLUDE_PHRASES", orig_text)
             return
 
+    # === NEW: entity-based dedup (CA/$ticker) lebih awal ===
+    entity_key = extract_entity_key(orig_text)
+    if entity_key and entity_key in recent_entity_keys:
+        debug_log("Entity-duplicate (CA/TICKER), dilewati", orig_text)
+        return
+
     # Content-only dedup (lintas grup)
     ch = content_only_hash(orig_text)
     if ch in recent_content_hashes:
@@ -600,28 +641,34 @@ async def process_message(msg, source_chat_id: int) -> None:
 
     role = _role_of(source_chat_id)  # 'core' / 'support'
 
-    # Multi-kw -> pilih satu dominan untuk agregasi
+    # Multi-kw -> pilih satu dominan untuk agregasi (fallback jika tak ada entity)
     all_kws = _extract_all_keywords(text_norm)
     main_kw = _choose_dominant_keyword(text_norm, all_kws)
-    if not main_kw:
-        debug_log("Tak ada keyword cocok, dilewati", orig_text)
+
+    # === NEW: topic key = entity_key (CA/$ticker) jika ada, else main_kw ===
+    topic_key = entity_key or main_kw
+    if not topic_key:
+        debug_log("Tak ada keyword/entitas cocok, dilewati", orig_text)
         return
 
-    # Agregasi & kelas
+    # Agregasi & kelas (berdasar topic_key)
     group_key = str(source_chat_id)
     now = datetime.now(timezone.utc)
-    class_label, unique_groups = update_and_classify(main_kw, group_key, now)
+    class_label, unique_groups = update_and_classify(topic_key, group_key, now)
 
     # Gating SUPPORT (CORE-anchored)
     if role != "core":
-        core_u, sup_u = _unique_counts_by_role(main_kw)
+        core_u, sup_u = _unique_counts_by_role(topic_key)
         # Aturan:
-        # - Jika sudah ada minimal 1 sebutan dari CORE untuk keyword ini -> izinkan.
+        # - Jika sudah ada minimal 1 sebutan dari CORE untuk key ini -> izinkan.
         # - Jika belum ada anchor CORE, SUPPORT harus >= SUPPORT_MIN_UNIQUE.
         if core_u >= 1:
             pass
         elif sup_u < SUPPORT_MIN_UNIQUE:
-            debug_log(f"Support ditahan (core_u={core_u}, sup_u={sup_u} < {SUPPORT_MIN_UNIQUE})", orig_text)
+            debug_log(
+                f"Support ditahan (core_u={core_u}, sup_u={sup_u} < {SUPPORT_MIN_UNIQUE})",
+                orig_text,
+            )
             return
 
     # Filter kalimat ajakan (whitelist-aware)
@@ -631,15 +678,24 @@ async def process_message(msg, source_chat_id: int) -> None:
         return
 
     # Backfill safety: saat startup, hindari pesan yang terlalu lama
-    cutoff = startup_time_utc - timedelta(minutes=CLASS_WINDOW_MINUTES + BACKFILL_BUFFER_MINUTES)
+    cutoff = startup_time_utc - timedelta(
+        minutes=CLASS_WINDOW_MINUTES + BACKFILL_BUFFER_MINUTES
+    )
     if getattr(msg, "date", None):
         msg_dt = msg.date
         if isinstance(msg_dt, datetime) and msg_dt.replace(tzinfo=timezone.utc) < cutoff:
             debug_log("Lama (lewat cutoff backfill safety), dilewati", orig_text)
             return
 
-    await post_or_update(main_kw, cleaned_body, class_label, msg)
-    debug_log(f"Posted/Edited (role={role}, unique_groups={unique_groups}, kw={main_kw}, tier={class_label})", orig_text)
+    # === NEW: simpan entity-key setelah sukses (untuk dedup) ===
+    if entity_key:
+        recent_entity_keys.append(entity_key)
+
+    await post_or_update(topic_key, cleaned_body, class_label, msg)
+    debug_log(
+        f"Posted/Edited (role={role}, unique_groups={unique_groups}, key={topic_key}, tier={class_label})",
+        orig_text,
+    )
 
 
 async def backfill_history(entity, limit: int) -> None:
@@ -674,6 +730,7 @@ async def _resolve_and_tag_chats(raw_list, role_label: str) -> list:
             print(f"Gagal resolve sumber {src}: {e}")
     return resolved
 
+
 async def start_bot_background() -> None:
     await client.start()
     _init_db()
@@ -695,6 +752,7 @@ async def start_bot_background() -> None:
     print("Kurator berjalan (background task). Menunggu pesan baru...")
     asyncio.create_task(client.run_until_disconnected())
 
+
 async def app_main() -> None:
     await client.start()
     _init_db()