Upload main.py

main.py

@@ -1,3 +1,5 @@
+# ----------------- Imports (Stdlib + Typing) -----------------
+from fastapi import FastAPI, Query, HTTPException, Body
 from fastapi import FastAPI, Query, HTTPException, Body
 from typing import Optional, List, Dict, Any, Tuple, Set
 import os
@@ -13,7 +15,6 @@ import math
 import nltk
 from nltk.sentiment import SentimentIntensityAnalyzer
 from geopy.geocoders import Nominatim
-from geopy.exc import GeocoderUnavailable, GeocoderTimedOut
 from fastapi.middleware.cors import CORSMiddleware
 from countryinfo import CountryInfo
 from sentence_transformers import SentenceTransformer, util
@@ -31,148 +32,64 @@ from transformers import pipeline as hf_pipeline
 os.environ.setdefault("OMP_NUM_THREADS", "1")
 from fastapi.responses import PlainTextResponse, JSONResponse
 
-
-# ----------------- FastAPI -----------------
-app = FastAPI()
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=False,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-app.add_middleware(GZipMiddleware, minimum_size=500)
-
-@app.api_route("/", methods=["GET", "HEAD"], include_in_schema=False)
-def root():
-    # For HEAD, Starlette ignores the body.
-    return JSONResponse({"ok": True, "service": "newsglobe-backend"})
-
-@app.api_route("/healthz", methods=["GET", "HEAD"], include_in_schema=False)
-def healthz():
-    return PlainTextResponse("OK", status_code=200)
-
-@app.get("/favicon.ico", include_in_schema=False)
-def favicon():
-    return PlainTextResponse("", status_code=204)
-
-
-
+# ----------------- Torch Runtime Settings -----------------
 import torch
 torch.set_num_threads(2)
 
-# Optional runtime check for local OPUS tokenizers
+# ----------------- Optional Local Tokenizers -----------------
 try:
-    import sentencepiece as _spm  # noqa: F401
+    import sentencepiece as _spm
     _HAS_SENTENCEPIECE = True
 except Exception:
     _HAS_SENTENCEPIECE = False
 
-
+# ----------------- Runtime Modes / Speed Enum -----------------
 from enum import Enum
 class Speed(str, Enum):
     fast = "fast"
     balanced = "balanced"
     max = "max"
 
+# ----------------- Global Model Handles / Pipelines -----------------
 _local_pipes = {}
 _news_clf = None
 _sbert = None
 
-
-# set a writable cache for tldextract and avoid network PSL fetches
+# ----------------- tldextract (PSL-cached) -----------------
 _TLD_CACHE = os.getenv("TLDEXTRACT_CACHE", "/data/tld_cache")
 try:
-    # suffix_list_urls=None => use cached public suffix list only (no HTTP on startup)
     _tld = tldextract.TLDExtract(cache_dir=_TLD_CACHE, suffix_list_urls=None)
 except Exception:
-    # safe fallback: still parses domains without PSL refresh
     _tld = tldextract.extract
 
-
-# --- Translation runtime flags / caches ---
-ALLOW_HF_REMOTE = os.getenv("ALLOW_HF_REMOTE", "0") == "1"  # default OFF
+# ----------------- Translation Runtime Flags -----------------
+ALLOW_HF_REMOTE = os.getenv("ALLOW_HF_REMOTE", "0") == "1"
 _hf_bad_models: Set[str] = set()
 
+# ----------------- FastAPI App + Middleware -----------------
+app = FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=False,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+app.add_middleware(GZipMiddleware, minimum_size=500)
 
-def _translate_local(text: str, src: str, tgt: str) -> Optional[str]:
-    if not _HAS_SENTENCEPIECE:
-        # Avoid attempting to download/instantiate Marian tokenizers without sentencepiece
-        return None
-    model_id = opus_model_for(src, tgt)
-    if not model_id:
-        return None
-    key = model_id
-    try:
-        if key not in _local_pipes:
-            _local_pipes[key] = hf_pipeline("translation", model=model_id)
-        out = _local_pipes[key](text, max_length=512)
-        return out[0]["translation_text"]
-    except Exception as e:
-        log.warning("Local translate failed for %s: %s", model_id, e)
-        return None
-
-    
-def fetch_gdelt_multi(limit=120, query=None, language=None, timespan="48h", category=None, speed: Speed = Speed.balanced):
-    # If user forced a language, honor it (but add a small English boost for coverage)
-    if language:
-        primary = fetch_gdelt_articles(limit=limit, query=query, language=language, timespan=timespan, category=category)
-        # tiny English booster to catch global wires
-        booster = fetch_gdelt_articles(limit=max(10, limit // 6), query=query, language="en", timespan=timespan, category=category)
-        return primary + booster
-
-    # Otherwise rotate across multiple languages
-    if speed == Speed.fast:
-        langs = LANG_ROTATION[:3]      # quicker
-        timespan = "24h"
-    elif speed == Speed.balanced:
-        langs = LANG_ROTATION[:8]      # good mix
-        timespan = "48h"
-    else:
-        langs = LANG_ROTATION          # max coverage
-        timespan = "3d"
-
-    per_lang = max(8, math.ceil(limit / len(langs)))
-    out = []
-    for lg in langs:
-        out.extend(fetch_gdelt_articles(limit=per_lang, query=query, language=lg, timespan=timespan, category=category))
-
-    # Optional: add a few English pulls biased to different source countries (broadens outlets)
-    if speed != Speed.fast:
-        per_cc = max(4, limit // 30) if speed == Speed.max else max(2, limit // 40)
-        for cc in COUNTRY_SEEDS[: (8 if speed == Speed.balanced else 16)]:
-            out.extend(
-                fetch_gdelt_articles(
-                    limit=per_cc,
-                    query=query,
-                    language="en",
-                    timespan=timespan,
-                    category=category,
-                    extra_tokens=[f"sourcecountry:{cc}"]
-                )
-            )
-
-    return out
-
+@app.api_route("/", methods=["GET", "HEAD"], include_in_schema=False)
+def root():
+    return JSONResponse({"ok": True, "service": "newsglobe-backend"})
 
-def get_news_clf():
-    global _news_clf
-    if _news_clf is None:
-        _news_clf = hf_pipeline(
-            "text-classification",
-            model="cardiffnlp/tweet-topic-21-multi",
-            top_k=1,
-        )
-    return _news_clf
+@app.api_route("/healthz", methods=["GET", "HEAD"], include_in_schema=False)
+def healthz():
+    return PlainTextResponse("OK", status_code=200)
 
-def get_sbert():
-    global _sbert
-    if _sbert is None:
-        _sbert = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
-    return _sbert
+@app.get("/favicon.ico", include_in_schema=False)
+def favicon():
+    return PlainTextResponse("", status_code=204)
 
-# globals
+# ----------------- HTTP Session (connection pooling) -----------------
 SESSION = requests.Session()
 ADAPTER = requests.adapters.HTTPAdapter(pool_connections=64, pool_maxsize=64, max_retries=2)
 SESSION.mount("http://", ADAPTER)
@@ -183,6 +100,7 @@ def _session_get(url, **kwargs):
     headers.setdefault("User-Agent", "Mozilla/5.0 (compatible; NewsGlobe/1.0)")
     return SESSION.get(url, headers=headers, timeout=kwargs.pop("timeout", 12), **kwargs)
 
+# ----------------- Lightweight Reader Fallback (Jina) -----------------
 def _try_jina_reader(url: str, timeout: int) -> Optional[str]:
     try:
         u = url.strip()
@@ -199,10 +117,7 @@ def _try_jina_reader(url: str, timeout: int) -> Optional[str]:
         pass
     return None
 
-
-
-# --- description cleanup helpers ---
-
+# ----------------- Description Cleanup Helpers -----------------
 BOILER_DESC = re.compile(
     r"(subscribe|sign in|sign up|enable javascript|cookies? (policy|settings)|"
     r"privacy (policy|notice)|continue reading|read more|click here|"
@@ -211,57 +126,13 @@ BOILER_DESC = re.compile(
 )
 
 def _split_sentences(text: str) -> List[str]:
-    # light-weight splitter good enough for news blurbs
     parts = re.split(r"(?<=[\.\?\!])\s+(?=[A-Z0-9])", (text or "").strip())
-    # also break on " • " and long dashes if present
     out = []
     for p in parts:
         out.extend(re.split(r"\s+[•–—]\s+", p))
     return [p.strip() for p in out if p and len(p.strip()) >= 2]
 
-def _tidy_description(title: str, desc: str, source_name: str, max_chars: int = 240) -> str:
-    if not desc:
-        return ""
-
-    # remove repeated title
-    desc = _dedupe_title_from_desc(title, desc)
-
-    # strip obvious boilerplate
-    desc = BOILER_DESC.sub("", desc)
-    desc = re.sub(r"\s+", " ", desc).strip(" -–:•|")
-
-    # choose first 1–2 sentences that look like summary
-    sents = _split_sentences(desc)
-    if not sents:
-        sents = [desc]
-
-    best = " ".join(sents[:2]).strip()
-
-    # soft truncate at sentence boundary
-    if len(best) > max_chars:
-        # try only first sentence
-        if len(sents[0]) <= max_chars * 0.9:
-            best = sents[0]
-        else:
-            best = best[:max_chars].rsplit(" ", 1)[0].rstrip(",;:-–—")
-
-    # avoid parroting the headline
-    if _too_similar(title, best):
-        # try next sentence if we have it
-        for alt in sents[1:3]:
-            if not _too_similar(title, alt):
-                best = alt
-                break
-
-    # ensure it ends neatly
-    if best and best[-1] not in ".!?":
-        best += "."
-    return best
-
-
-
 def _too_similar(a: str, b: str, thresh: float = 0.92) -> bool:
-    """Return True if strings are near-duplicates (or one contains the other)."""
     a = (a or "").strip()
     b = (b or "").strip()
     if not a or not b:
@@ -274,25 +145,48 @@ def _too_similar(a: str, b: str, thresh: float = 0.92) -> bool:
     return ratio >= thresh
 
 def _dedupe_title_from_desc(title: str, desc: str) -> str:
-    """If the description contains the title, strip it and tidy up."""
     t = (title or "").strip()
     d = (desc or "").strip()
     if not t or not d:
         return d
-    # Remove exact leading title
     if d.lower().startswith(t.lower()):
         d = d[len(t):].lstrip(" -–:•|")
-    # Remove inner repeats
     d = d.replace(t, "").strip(" -–:•|")
     d = _clean_text(d)
     return d
 
+def _clean_text(s: str) -> str:
+    s = unescape(s or "")
+    s = re.sub(r"\s+", " ", s).strip()
+    return s
+
+def _tidy_description(title: str, desc: str, source_name: str, max_chars: int = 240) -> str:
+    if not desc:
+        return ""
+    desc = _dedupe_title_from_desc(title, desc)
+    desc = BOILER_DESC.sub("", desc)
+    desc = re.sub(r"\s+", " ", desc).strip(" -–:•|")
+    sents = _split_sentences(desc)
+    if not sents:
+        sents = [desc]
+    best = " ".join(sents[:2]).strip()
+    if len(best) > max_chars:
+        if len(sents[0]) <= max_chars * 0.9:
+            best = sents[0]
+        else:
+            best = best[:max_chars].rsplit(" ", 1)[0].rstrip(",;:-–—")
+    if _too_similar(title, best):
+        for alt in sents[1:3]:
+            if not _too_similar(title, alt):
+                best = alt
+                break
+    if best and best[-1] not in ".!?":
+        best += "."
+    return best
 
-# Prevent duplicate upstream fetches when identical requests arrive together
+# ----------------- Inflight Request Coalescing -----------------
 _inflight_locks: Dict[Tuple, threading.Lock] = {}
 _inflight_global_lock = threading.Lock()
-
-
 def _get_inflight_lock(key: Tuple) -> threading.Lock:
     with _inflight_global_lock:
         lk = _inflight_locks.get(key)
@@ -301,40 +195,29 @@ def _get_inflight_lock(key: Tuple) -> threading.Lock:
             _inflight_locks[key] = lk
         return lk
 
-
+# ----------------- Description Fetching (Cache + Extract) -----------------
 DESC_CACHE_LOCK = threading.Lock()
-
 try:
-    from bs4 import BeautifulSoup  # optional but nice to have
+    from bs4 import BeautifulSoup
 except Exception:
     BeautifulSoup = None
 
-
-# -------- Description fetching config --------
-DESC_FETCH_TIMEOUT = 3  # seconds per URL
-DESC_MIN_LEN = 100  # consider shorter text as "weak" and try to upgrade
-DESC_CACHE_TTL = 24 * 3600  # 24h
-MAX_DESC_FETCHES = 24  # cap number of fetches per request
-DESC_WORKERS = 12  # parallel workers
-
-# url -> {"text": str, "t": monotonic()}
+DESC_FETCH_TIMEOUT = 3
+DESC_MIN_LEN = 100
+DESC_CACHE_TTL = 24 * 3600
+MAX_DESC_FETCHES = 24
+DESC_WORKERS = 12
 DESC_CACHE: Dict[str, Dict[str, Any]] = {}
 
-
 def _now_mono():
+    # Monotonic for TTL calculations
     try:
         return monotonic()
     except Exception:
         return time.time()
 
-
-def _clean_text(s: str) -> str:
-    s = unescape(s or "")
-    s = re.sub(r"\s+", " ", s).strip()
-    return s
-
-
 def _extract_desc_from_ld_json(html: str) -> Optional[str]:
+    # Prefer LD-JSON when present (often cleaner summaries)
     if not html or not BeautifulSoup:
         return None
     try:
@@ -342,11 +225,9 @@ def _extract_desc_from_ld_json(html: str) -> Optional[str]:
         for tag in soup.find_all("script", {"type": "application/ld+json"}):
             try:
                 import json
-
                 data = json.loads(tag.string or "")
             except Exception:
                 continue
-
             def find_desc(obj):
                 if not isinstance(obj, (dict, list)):
                     return None
@@ -356,21 +237,18 @@ def _extract_desc_from_ld_json(html: str) -> Optional[str]:
                         if v:
                             return v
                     return None
-                # dict
                 for key in ("description", "abstract", "articleBody"):
                     val = obj.get(key)
                     if isinstance(val, str):
                         txt = _clean_text(val)
                         if len(txt) >= 40:
                             return txt
-                # nested
                 for k, v in obj.items():
                     if isinstance(v, (dict, list)):
                         got = find_desc(v)
                         if got:
                             return got
                 return None
-
             d = find_desc(data)
             if d and len(d) >= 40:
                 return d
@@ -378,32 +256,25 @@ def _extract_desc_from_ld_json(html: str) -> Optional[str]:
         pass
     return None
 
-
+# Heuristic to detect consent walls and jump to reader fallback
 CONSENT_HINTS = re.compile(r"(consent|gdpr|privacy choices|before you continue|we value your privacy)", re.I)
 
-
 def _looks_like_consent_wall(html: str) -> bool:
     if not html:
         return False
-    if "consent.yahoo.com" in html.lower():  # common interstitial
+    if "consent.yahoo.com" in html.lower():
         return True
-    # generic phrasing
     return bool(CONSENT_HINTS.search(html))
 
-
 def _extract_desc_from_html(html: str) -> Optional[str]:
     html = html or ""
     if BeautifulSoup:
         soup = BeautifulSoup(html, "html.parser")
-
-        # ✅ JSON-LD early
         ld = _extract_desc_from_ld_json(html)
         if ld:
             txt = _clean_text(ld)
             if 40 <= len(txt) <= 480:
                 return txt
-
-
         for sel, attr in [
             ('meta[property="og:description"]', "content"),
             ('meta[name="twitter:description"]', "content"),
@@ -414,13 +285,11 @@ def _extract_desc_from_html(html: str) -> Optional[str]:
                 txt = _clean_text(tag.get(attr, ""))
                 if len(txt) >= 40:
                     return txt
-        # Fallback: first meaningful <p>
         for p in soup.find_all("p"):
             txt = _clean_text(p.get_text(" "))
             if len(txt) >= 80:
                 return txt
     else:
-        # regex fallbacks (as you had)
         for pat in [
             r'<meta[^>]+property=["\']og:description["\'][^>]+content=["\']([^"\']+)["\']',
             r'<meta[^>]+name=["\']twitter:description["\'][^>]+content=["\']([^"\']+)["\']',
@@ -436,10 +305,8 @@ def _extract_desc_from_html(html: str) -> Optional[str]:
             txt = _clean_text(re.sub("<[^>]+>", " ", m.group(1)))
             if len(txt) >= 80:
                 return txt
-        # JSON-LD as last regex-free fallback not available w/o bs4
     return None
 
-
 def _desc_cache_get(url: str) -> Optional[str]:
     if not url:
         return None
@@ -451,14 +318,13 @@ def _desc_cache_get(url: str) -> Optional[str]:
         return None
     return entry["text"]
 
-
 def _desc_cache_put(url: str, text: str):
     if url and text:
         with DESC_CACHE_LOCK:
             DESC_CACHE[url] = {"text": text, "t": _now_mono()}
 
-
 def _attempt_fetch(url: str, timeout: int) -> Optional[str]:
+    # Fetch page and extract description; fallback to reader if needed
     headers = {
         "User-Agent": "Mozilla/5.0 (compatible; NewsGlobe/1.0; +mailto:you@yourdomain.com)",
         "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
@@ -473,7 +339,6 @@ def _attempt_fetch(url: str, timeout: int) -> Optional[str]:
         if "html" not in ct and "<html" not in txt.lower():
             return None
         if _looks_like_consent_wall(txt):
-            # jump straight to Jina if a consent wall is detected
             jd = _try_jina_reader(url, timeout)
             if jd:
                 return jd
@@ -482,39 +347,28 @@ def _attempt_fetch(url: str, timeout: int) -> Optional[str]:
         if desc and 40 <= len(desc) <= 480:
             return desc
     except Exception:
-        # fall through to Jina
         pass
-
-    # Last-resort: Jina reader
     jd = _try_jina_reader(url, timeout)
     if jd and 40 <= len(jd) <= 480:
         return jd
     return None
 
-
-
 def fetch_page_description(url: str) -> Optional[str]:
-    """Fetch and cache a best-effort article description from the page (incl. AMP retries)."""
+    # Public entry: consult cache -> fetch -> AMP variants -> cache
     if not url:
         return None
     cached = _desc_cache_get(url)
     if cached:
         return cached
-
-    # Try the original URL first
     desc = _attempt_fetch(url, DESC_FETCH_TIMEOUT)
     if not desc:
-        # Try common AMP variants
         amp_candidates = []
         try:
             p = urlparse(url)
-            # /amp path
             if not p.path.endswith("/amp"):
                 amp_candidates.append(urlunparse(p._replace(path=(p.path.rstrip("/") + "/amp"))))
-            # ?amp
             q = p.query
             amp_candidates.append(urlunparse(p._replace(query=(q + ("&" if q else "") + "amp=1"))))
-            # ?outputType=amp (CNN, some US sites)
             amp_candidates.append(urlunparse(p._replace(query=(q + ("&" if q else "") + "outputType=amp"))))
         except Exception:
             pass
@@ -522,14 +376,13 @@ def fetch_page_description(url: str) -> Optional[str]:
             desc = _attempt_fetch(amp_url, DESC_FETCH_TIMEOUT)
             if desc:
                 break
-
     if desc:
         _desc_cache_put(url, desc)
         return desc
     return None
 
-
 def _needs_desc_upgrade(a: Dict[str, Any]) -> bool:
+    # Decide if we should try to refetch a better description
     url = a.get("url") or ""
     if not url:
         return False
@@ -539,38 +392,37 @@ def _needs_desc_upgrade(a: Dict[str, Any]) -> bool:
         return True
     if len(desc) < DESC_MIN_LEN:
         return True
-    # NEW: if desc ≈ title, trigger upgrade
     if _too_similar(title, desc):
         return True
     return False
 
-
 def prefetch_descriptions(raw_articles: List[Dict[str, Any]], speed: Speed = Speed.balanced):
+    # Parallel prefetch for weak descriptions (bounded to avoid stampedes)
     candidates, seen = [], set()
     max_fetches = 6 if speed == Speed.fast else 8 if speed == Speed.balanced else 16
     timeout = 1 if speed == Speed.fast else 2
     workers = 3 if speed == Speed.fast else 4 if speed == Speed.balanced else 8
-
-
     for a in raw_articles:
-        url = a.get("url"); 
-        if not url or url in seen: continue
+        url = a.get("url")
+        if not url or url in seen:
+            continue
         seen.add(url)
         if _needs_desc_upgrade(a) and not _desc_cache_get(url):
             candidates.append(url)
-        if len(candidates) >= max_fetches: break
-
-    if not candidates: return
+        if len(candidates) >= max_fetches:
+            break
+    if not candidates:
+        return
     with ThreadPoolExecutor(max_workers=workers) as ex:
         futs = [ex.submit(fetch_page_description, u) for u in candidates]
-        for _ in as_completed(futs): pass
-
+        for _ in as_completed(futs):
+            pass
 
 def prefetch_descriptions_async(raw_articles, speed: Speed = Speed.balanced):
     threading.Thread(target=prefetch_descriptions, args=(raw_articles, speed), daemon=True).start()
 
-# news_clf = pipeline("text-classification", model="cardiffnlp/tweet-topic-21-multi", top_k=1)
-DetectorFactory.seed = 0  # deterministic
+# ----------------- Category / Keyword Heuristics -----------------
+DetectorFactory.seed = 0
 
 SECTION_HINTS = {
     "sports": "sports",
@@ -611,20 +463,29 @@ KEYWORDS = {
     "politics": r"\b(president|parliament|congress|minister|policy|campaign|election)\b",
 }
 
+def get_news_clf():
+    # Lazy-init topic classifier
+    global _news_clf
+    if _news_clf is None:
+        _news_clf = hf_pipeline(
+            "text-classification",
+            model="cardiffnlp/tweet-topic-21-multi",
+            top_k=1,
+        )
+    return _news_clf
 
 def _infer_category_from_url_path(url_path: str) -> Optional[str]:
+    # Order: provided -> URL path -> keyword -> ML fallback
     parts = [p for p in url_path.lower().split("/") if p]
     for p in parts:
         if p in SECTION_HINTS:
             return SECTION_HINTS[p]
-    # also try hyphenated tokens like 'us-business' or 'tech-news'
     for p in parts:
         for tok in re.split(r"[-_]", p):
             if tok in SECTION_HINTS:
                 return SECTION_HINTS[tok]
     return None
 
-
 def _infer_category_from_text(text: str) -> Optional[str]:
     if not text:
         return None
@@ -633,13 +494,11 @@ def _infer_category_from_text(text: str) -> Optional[str]:
             return cat
     return None
 
-
 def infer_category(article_url, title, description, provided):
     if provided:
         p = provided.strip().lower()
         if p:
             return p
-    # url rules
     try:
         p = urlparse(article_url).path or ""
         cat = _infer_category_from_url_path(p)
@@ -647,13 +506,12 @@ def infer_category(article_url, title, description, provided):
             return cat
     except Exception:
         pass
-    # keyword rules
     text = f"{title or ''} {description or ''}".strip()
     cat = _infer_category_from_text(text)
     if cat:
         return cat
     try:
-        preds = get_news_clf()(text[:512])  # lazy-loaded
+        preds = get_news_clf()(text[:512])
         if isinstance(preds[0], list):
             label = preds[0][0]["label"]
         else:
@@ -663,193 +521,35 @@ def infer_category(article_url, title, description, provided):
         log.warning(f"ML category failed: {e}")
         return "general"
 
-
-
-BOILER = re.compile(r"\b(live updates|breaking|what we know|in pictures|opinion)\b", re.I)
-
-
-def _norm_text(s: str) -> str:
-    s = (s or "").strip()
-    s = re.sub(r"\s+", " ", s)
-    return s
-
-
-def _cluster_text(a):
-    base = f"{a.get('orig_title') or a.get('title') or ''} {a.get('orig_description') or a.get('description') or ''}"
-    base = BOILER.sub("", base)
-    base = re.sub(r"\b(\d{1,2}:\d{2}\s?(AM|PM))|\b(\d{1,2}\s\w+\s\d{4})", "", base, flags=re.I)
-    return _norm_text(base)
-
-
-def _canonical_url(u: str) -> str:
-    if not u:
-        return u
-    p = urlparse(u)
-    # drop tracking params
-    qs = [(k, v) for (k, v) in parse_qsl(p.query, keep_blank_values=False) if not k.lower().startswith(("utm_", "fbclid", "gclid"))]
-    clean = p._replace(query="&".join([f"{k}={v}" for k, v in qs]), fragment="")
-    # some sites add trailing slashes inconsistently
-    path = clean.path.rstrip("/") or "/"
-    clean = clean._replace(path=path)
-    return urlunparse(clean)
-
-
+# ----------------- Language Detection / Embeddings -----------------
 def detect_lang(text: str) -> Optional[str]:
     try:
-        return detect(text)  # returns 'en','fr','de',...
+        return detect(text)
     except Exception:
         return None
 
+def get_sbert():
+    global _sbert
+    if _sbert is None:
+        _sbert = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+    return _sbert
 
 def _embed_texts(texts: List[str]):
     embs = get_sbert().encode(texts, convert_to_tensor=True, normalize_embeddings=True, show_progress_bar=False)
     return embs
 
-
-# ---- cache helpers ----
-CACHE_TTL_SECS = 900
-SIM_THRESHOLD = 0.6
-_events_cache: Dict[Tuple, Dict[str, Any]] = {}
-
-
-def cache_key_for(q, category, language, limit_each, translate=False, target_lang=None, speed=Speed.balanced):
-    return (q or "", category or "", language or "", int(limit_each or 50),
-            bool(translate), (target_lang or "").lower(), speed.value)
-
-
-_first_real_build = True  # module-global
-
-def get_or_build_events_cache(q, category, language, translate, target_lang, limit_each, speed=Speed.balanced):
-    global _first_real_build
-    key = cache_key_for(q, category, language, limit_each, translate, target_lang, speed)
-    now = monotonic()
-
-    if speed == Speed.fast:
-        use_timespan, use_limit = "24h", min(limit_each, 20)
-    elif speed == Speed.balanced:
-        use_timespan, use_limit = "48h", min(limit_each, 150)
-    else:  # max
-        use_timespan, use_limit = "3d", limit_each
-
-    entry = _events_cache.get(key)
-    if entry and now - entry["t"] < CACHE_TTL_SECS:
-        log.info(f"CACHE HIT for {key}")
-        return key, entry["enriched"], entry["clusters"]
-
-    lock = _get_inflight_lock(key)
-    with lock:
-        entry = _events_cache.get(key)
-        if entry and now - entry["t"] < CACHE_TTL_SECS:
-            log.info(f"CACHE HIT (post-lock) for {key}")
-            return key, entry["enriched"], entry["clusters"]
-
-        if _first_real_build:
-            use_timespan = "24h" if use_timespan != "24h" else use_timespan
-            use_limit = min(use_limit, 100)
-
-        log.info(f"CACHE MISS for {key} — fetching (timespan={use_timespan}, limit_each={use_limit})")
-
-        raw = combine_raw_articles(
-            category=category,  # providers may use it; inference ignores it
-            query=q,
-            language=language,
-            limit_each=use_limit,
-            timespan=use_timespan,
-            speed=speed,
-        )
-        prefetch_descriptions_async(raw, speed)
-
-        enriched_all = [enrich_article(a, language=language, translate=False, target_lang=None) for a in raw]
-
-        if category:
-            cat_norm = (category or "").strip().lower()
-            enriched = [e for e in enriched_all if (e.get("category") or "").lower() == cat_norm]
-        else:
-            enriched = enriched_all
-
-        clusters = cluster_articles(enriched, sim_threshold=SIM_THRESHOLD, speed=speed)
-
-        _events_cache[key] = {"t": monotonic(), "enriched": enriched, "clusters": clusters}
-        _first_real_build = False
-        return key, enriched, clusters
-
-
-# Which languages to rotate when user didn't restrict language
-LANG_ROTATION = ["en", "es", "fr", "de", "ar", "ru", "pt", "zh", "hi", "ja", "ko"]
-
-# A few sourcecountry seeds for English to diversify outlets (optional)
-COUNTRY_SEEDS = ["US", "GB", "IN", "CA", "AU", "ZA", "SG", "NG", "DE", "FR", "BR", "MX", "ES", "RU", "JP", "KR", "CN"]
-
-
-# ----------------- Config / Keys -----------------
-USE_GNEWS_API = False
-USE_NEWSDATA_API = False
-USE_GDELT_API = True
-USE_NEWSAPI = False
-
-NEWSAPI_KEY = os.getenv("NEWSAPI_KEY", "ea734c66dc4044fa8e4501ad7b90e753")
-GNEWS_API_KEY = os.getenv("GNEWS_API_KEY", "5419897c95e8a4b21074e0d3fe95a3dd")
-NEWSDATA_API_KEY = os.getenv("NEWSDATA_API_KEY", "pub_1feb49a71a844719af68d0844fb43a61")
-HUGGINGFACE_API_TOKEN = os.getenv("HUGGINGFACE_API_TOKEN")
-
-logging.basicConfig(
-    level=logging.WARNING,
-    format="%(levelname)s:%(name)s:%(message)s",
-)
-
-log = logging.getLogger("newsglobe")
-log.setLevel(logging.WARNING)
-
-fetch_log = logging.getLogger("newsglobe.fetch_summary")
-fetch_log.setLevel(logging.INFO)
-_fetch_handler = logging.StreamHandler()
-_fetch_handler.setLevel(logging.INFO)
-_fetch_handler.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s"))
-fetch_log.addHandler(_fetch_handler)
-fetch_log.propagate = False  # don't pass to root (which is WARNING)
-
-logging.getLogger("urllib3").setLevel(logging.WARNING)
-logging.getLogger("urllib3.connectionpool").setLevel(logging.WARNING)
-logging.getLogger("requests.packages.urllib3").setLevel(logging.WARNING)
-logging.getLogger("sentence_transformers").setLevel(logging.WARNING)
-logging.getLogger("transformers").setLevel(logging.WARNING)
-
-for name in ("urllib3", "urllib3.connectionpool", "requests.packages.urllib3"):
-    lg = logging.getLogger(name)
-    lg.setLevel(logging.ERROR)
-    lg.propagate = False
-
-
-def _newsapi_enabled() -> bool:
-    if not NEWSAPI_KEY:
-        log.warning("NewsAPI disabled: missing NEWSAPI_KEY env var")
-        return False
-    return True
-
-
-def cluster_id(cluster, enriched_articles):
-    urls = sorted([(enriched_articles[i].get("url") or "") for i in cluster["indices"] if enriched_articles[i].get("url")])
-    base = "|".join(urls) if urls else "empty"
-    return hashlib.md5(base.encode("utf-8")).hexdigest()[:10]
-
-
-# ----------------- NLTK / VADER -----------------
+# ----------------- NLTK / VADER Sentiment -----------------
 NLTK_DATA_DIR = os.environ.get("NLTK_DATA", "/app/nltk_data")
-
-# Make sure NLTK looks in the baked, writable dir first
 if NLTK_DATA_DIR not in nltk.data.path:
     nltk.data.path.insert(0, NLTK_DATA_DIR)
-
 try:
     nltk.data.find("sentiment/vader_lexicon")
 except LookupError:
-    # As a fallback, try downloading into the writable dir (won't run if already baked)
     try:
         os.makedirs(NLTK_DATA_DIR, exist_ok=True)
         nltk.download("vader_lexicon", download_dir=NLTK_DATA_DIR, quiet=True)
     except Exception:
-        pass  # don't crash if download is blocked
-
+        pass
 try:
     _vader = SentimentIntensityAnalyzer()
 except Exception:
@@ -864,8 +564,7 @@ def classify_sentiment(text: str) -> str:
     c = scores["compound"]
     return "positive" if c >= 0.2 else "negative" if c <= -0.2 else "neutral"
 
-
-# ----------------- Geocode helpers -----------------
+# ----------------- Geocoding / Domain → Country -----------------
 def get_country_centroid(country_name):
     if not country_name or country_name == "Unknown":
         return {"lat": 0, "lon": 0, "country": "Unknown"}
@@ -877,7 +576,6 @@ def get_country_centroid(country_name):
         log.info(f"Could not get centroid for {country_name}: {e}")
         return {"lat": 0, "lon": 0, "country": country_name or "Unknown"}
 
-
 def resolve_domain_to_ip(domain):
     if not domain:
         return None
@@ -886,7 +584,6 @@ def resolve_domain_to_ip(domain):
     except socket.gaierror:
         return None
 
-
 def geolocate_ip(ip):
     try:
         r = _session_get(f"https://ipwho.is/{ip}?fields=success,country,latitude,longitude", timeout=8)
@@ -897,7 +594,7 @@ def geolocate_ip(ip):
         pass
     return None
 
-
+# Nominatim for a light refinement pass (async)
 geolocator = Nominatim(user_agent="newsglobe-app (contact: you@example.com)")
 domain_geo_cache: Dict[str, Dict[str, Any]] = {}
 
@@ -915,32 +612,23 @@ MAJOR_OUTLETS = {
     "lefigaro.fr": "France",
     "kyodonews.net": "Japan",
     "straitstimes.com": "Singapore",
-    "thesun.my": "Malaysia",  # <-- add this
+    "thesun.my": "Malaysia",
 }
 
-
 def geocode_source(source_text: str, domain: str = "", do_network: bool = False):
     cache_key = f"{source_text}|{domain}"
     if cache_key in domain_geo_cache:
         return domain_geo_cache[cache_key]
-
     ext = _tld(domain or "")
     fqdn = ".".join([p for p in (ext.domain, ext.suffix) if p]) if (ext.domain or ext.suffix) else ""
-
-    # 0) Major outlets / domain map
     if fqdn in MAJOR_OUTLETS:
         coords = get_country_centroid(MAJOR_OUTLETS[fqdn]); domain_geo_cache[cache_key] = coords; return coords
     if ext.domain in domain_country_map:
         coords = get_country_centroid(domain_country_map[ext.domain]); domain_geo_cache[cache_key] = coords; return coords
-
-    # 1) Suffix fallback (instant)
     coords = get_country_centroid(_suffix_country(ext.suffix))
     domain_geo_cache[cache_key] = coords
-
-    # 2) Optional async refinement (never block hot path)
     if do_network:
         threading.Thread(target=_refine_geo_async, args=(cache_key, source_text, fqdn), daemon=True).start()
-
     return coords
 
 def _suffix_country(suffix: Optional[str]) -> str:
@@ -955,18 +643,14 @@ def _suffix_country(suffix: Optional[str]) -> str:
     }
     return m.get(s, "United States" if s in ("com","org","net") else "Unknown")
 
-
-
 def _refine_geo_async(cache_key, source_text, fqdn):
     try:
-        # Try IP geo (cheap)
         ip = resolve_domain_to_ip(fqdn) if fqdn else None
         if ip:
             coords = geolocate_ip(ip)
             if coords:
                 domain_geo_cache[cache_key] = coords
                 return
-        # Try Nominatim FAST (lower timeout)
         location = geolocator.geocode(f"{source_text} News Headquarters", timeout=2)
         if location and hasattr(location, "raw"):
             coords = {
@@ -978,10 +662,9 @@ def _refine_geo_async(cache_key, source_text, fqdn):
     except Exception:
         pass
 
-# ----------------- HuggingFace translate (optional) -----------------
-HF_MODEL_PRIMARY = None  # disable NLLB remote (avoids 404 spam); use OPUS + pivot/LibreTranslate
+# ----------------- Translation (HF / Libre / Local) -----------------
+HF_MODEL_PRIMARY = None
 
-# 2-letter ISO -> NLLB codes
 NLLB_CODES = {
     "en": "eng_Latn",
     "es": "spa_Latn",
@@ -997,8 +680,6 @@ NLLB_CODES = {
     "ko": "kor_Hang",
 }
 
-
-# OPUS-MT model map for common pairs (expand as needed)
 def opus_model_for(src2: str, tgt2: str) -> Optional[str]:
     pairs = {
         ("es", "en"): "Helsinki-NLP/opus-mt-es-en",
@@ -1026,10 +707,8 @@ def opus_model_for(src2: str, tgt2: str) -> Optional[str]:
     }
     return pairs.get((src2, tgt2))
 
-
 SUPPORTED = {"en", "fr", "de", "es", "it", "hi", "ar", "ru", "ja", "ko", "pt", "zh"}
-
-LIBRETRANSLATE_URL = os.getenv("LIBRETRANSLATE_URL")  # e.g., http://127.0.0.1:5000
+LIBRETRANSLATE_URL = os.getenv("LIBRETRANSLATE_URL")
 
 def _translate_via_libre(text: str, src: str, tgt: str) -> Optional[str]:
     url = LIBRETRANSLATE_URL
@@ -1051,14 +730,11 @@ def _translate_via_libre(text: str, src: str, tgt: str) -> Optional[str]:
         log.warning("LibreTranslate failed: %s", e)
     return None
 
-
 def _hf_call(model_id: str, payload: dict) -> Optional[str]:
-    # require both a token and explicit opt-in
     if not (HUGGINGFACE_API_TOKEN and ALLOW_HF_REMOTE):
         return None
     if model_id in _hf_bad_models:
         return None
-
     url = f"https://api-inference.huggingface.co/models/{model_id}"
     headers = {
         "Authorization": f"Bearer {HUGGINGFACE_API_TOKEN}",
@@ -1079,7 +755,6 @@ def _hf_call(model_id: str, payload: dict) -> Optional[str]:
     except Exception as e:
         log.warning("HF request failed: %s", e)
         return None
-
     if isinstance(j, list) and j and isinstance(j[0], dict):
         if "generated_text" in j[0]:
             return j[0]["generated_text"]
@@ -1095,20 +770,14 @@ def _hf_call(model_id: str, payload: dict) -> Optional[str]:
 def _translate_cached(text: str, src: str, tgt: str) -> str:
     if not text or src == tgt:
         return text
-
-    # 0) Local LibreTranslate (fast & free, if running)
     out = _translate_via_libre(text, src, tgt)
     if out:
         return out
-
-    # 1) OPUS serverless (direct pair) – try this first
     opus_model = opus_model_for(src, tgt)
     if opus_model:
         out = _hf_call(opus_model, {"inputs": text})
         if out:
             return out
-
-    # 2) NLLB serverless (optional; disabled if HF_MODEL_PRIMARY is None)
     try:
         if HF_MODEL_PRIMARY and (src in NLLB_CODES) and (tgt in NLLB_CODES):
             out = _hf_call(
@@ -1123,26 +792,18 @@ def _translate_cached(text: str, src: str, tgt: str) -> str:
                 return out
     except Exception:
         pass
-
-    # 3) Two-hop pivot via English for non-English↔non-English
     if src != "en" and tgt != "en":
         step_en = _translate_cached(text, src, "en")
         if step_en and step_en != text:
             out = _translate_cached(step_en, "en", tgt)
             if out:
                 return out
-
-    # 4) Local OPUS fallback (direct pair with local pipeline)
     out = _translate_local(text, src, tgt)
     if out:
         return out
-
     log.warning("All translate paths failed (%s->%s); returning original.", src, tgt)
     return text
 
-
-
-
 def translate_text(text: str, target_lang: Optional[str], fallback_src: Optional[str] = None) -> str:
     if not text or not target_lang:
         return text
@@ -1163,12 +824,25 @@ def translate_text(text: str, target_lang: Optional[str], fallback_src: Optional
             src = "en"
     return _translate_cached(text, src, tgt)
 
+def _translate_local(text: str, src: str, tgt: str) -> Optional[str]:
+    if not _HAS_SENTENCEPIECE:
+        return None
+    model_id = opus_model_for(src, tgt)
+    if not model_id:
+        return None
+    key = model_id
+    try:
+        if key not in _local_pipes:
+            _local_pipes[key] = hf_pipeline("translation", model=model_id)
+        out = _local_pipes[key](text, max_length=512)
+        return out[0]["translation_text"]
+    except Exception as e:
+        log.warning("Local translate failed for %s: %s", model_id, e)
+        return None
 
-
-
-# === Warm config ===
-WARM_LIMIT_EACH = 20        # smaller bite to prime caches
-WARM_TIMESPAN = "24h"       # narrower GDELT window for faster first fetch
+# ----------------- Warmup Settings & Routine -----------------
+WARM_LIMIT_EACH = 20
+WARM_TIMESPAN = "24h"
 WARM_PREFETCH_DESCRIPTIONS = False
 
 def _fmt_mmss(ms: float) -> str:
@@ -1180,40 +854,27 @@ def _warm_once():
     try:
         log.info("WARM: starting background warm-up (limit_each=%d, timespan=%s)", WARM_LIMIT_EACH, WARM_TIMESPAN)
         t0 = time.perf_counter()
-
-        # models (you already call these in startup, but keep them here too)
         get_sbert()
         get_news_clf()
-
-        # fetch a small set with shorter timespan
         t1 = time.perf_counter()
         raw = combine_raw_articles(
             category=None, query=None, language="en",
             limit_each=WARM_LIMIT_EACH, timespan=WARM_TIMESPAN,
-            log_summary=False  # ← silence warm-up summary
+            log_summary=False
         )
         t_fetch = (time.perf_counter() - t1) * 1000
-
-        # optional: skip description prefetch during warm to save time
         if WARM_PREFETCH_DESCRIPTIONS:
             prefetch_descriptions_async(raw)
-
-        # enrich + cluster once (no translation on warm)
         t2 = time.perf_counter()
         enriched = [enrich_article(a, language="en", translate=False, target_lang=None) for a in raw]
         t_enrich = (time.perf_counter() - t2) * 1000
-
         t3 = time.perf_counter()
         clusters = cluster_articles(enriched, sim_threshold=SIM_THRESHOLD)
         t_cluster = (time.perf_counter() - t3) * 1000
-
-        # stash in cache under the common default key so /news and /events hit warm data
         key = cache_key_for(q=None, category=None, language="en",
                     limit_each=WARM_LIMIT_EACH, translate=False, target_lang=None,
-                    speed=Speed.balanced)  # 👈 add speed
-
+                    speed=Speed.balanced)
         _events_cache[key] = {"t": monotonic(), "enriched": enriched, "clusters": clusters}
-
         t_total = (time.perf_counter() - t0) * 1000
         log.info(
             "WARM: fetch=%s, enrich=%s, cluster=%s, total=%s (raw=%d, enriched=%d, clusters=%d)",
@@ -1225,14 +886,11 @@ def _warm_once():
 
 @app.on_event("startup")
 def warm():
-    # keep your existing model warms
     get_sbert()
     get_news_clf()
-    # fire-and-forget warm in a background thread so startup stays snappy
     threading.Thread(target=_warm_once, daemon=True).start()
 
-# ----------------- Providers -----------------
-# ISO -> GDELT 'sourcelang:' names (keep yours)
+# ----------------- GDELT Query Helpers -----------------
 _GDELT_LANG = {
     "en": "english",
     "es": "spanish",
@@ -1248,7 +906,6 @@ _GDELT_LANG = {
     "zh": "chinese",
 }
 
-
 def _gdelt_safe_query(user_q, language):
     parts = []
     if user_q:
@@ -1259,11 +916,10 @@ def _gdelt_safe_query(user_q, language):
     if language and (lg := _GDELT_LANG.get(language.lower())):
         parts.append(f"sourcelang:{lg}")
     if not parts:
-        # rotate or randomly choose one to diversify
         parts.append("sourcelang:english")
     return " ".join(parts)
 
-
+# ----------------- GDELT Fetchers -----------------
 def fetch_gdelt_articles(
     limit=50,
     query=None,
@@ -1304,7 +960,6 @@ def fetch_gdelt_articles(
 
     data = _do_request(params)
     if data is None:
-        # Retry narrower and smaller if needed
         p2 = {**params, "timespan": "24h", "maxrecords": min(100, params["maxrecords"])}
         data = _do_request(p2)
     if not data:
@@ -1330,144 +985,112 @@ def fetch_gdelt_articles(
                 "publishedAt": a.get("seendate"),
                 "api_source": "gdelt",
                 "gdelt_sourcecountry": a.get("sourcecountry"),
-                # Keep the user's chosen category only for debugging/reference; do NOT use for inference.
                 "requested_category": category,
             }
         )
     log.info(f"GDELT returned {len(results)}")
     return results
 
+def fetch_gdelt_multi(limit=120, query=None, language=None, timespan="48h", category=None, speed: Speed = Speed.balanced):
+    if language:
+        primary = fetch_gdelt_articles(limit=limit, query=query, language=language, timespan=timespan, category=category)
+        booster = fetch_gdelt_articles(limit=max(10, limit // 6), query=query, language="en", timespan=timespan, category=category)
+        return primary + booster
+    if speed == Speed.fast:
+        langs = LANG_ROTATION[:3]
+        timespan = "24h"
+    elif speed == Speed.balanced:
+        langs = LANG_ROTATION[:8]
+        timespan = "48h"
+    else:
+        langs = LANG_ROTATION
+        timespan = "3d"
+    per_lang = max(8, math.ceil(limit / len(langs)))
+    out = []
+    for lg in langs:
+        out.extend(fetch_gdelt_articles(limit=per_lang, query=query, language=lg, timespan=timespan, category=category))
+    if speed != Speed.fast:
+        per_cc = max(4, limit // 30) if speed == Speed.max else max(2, limit // 40)
+        for cc in COUNTRY_SEEDS[: (8 if speed == Speed.balanced else 16)]:
+            out.extend(
+                fetch_gdelt_articles(
+                    limit=per_cc,
+                    query=query,
+                    language="en",
+                    timespan=timespan,
+                    category=category,
+                    extra_tokens=[f"sourcecountry:{cc}"]
+                )
+            )
+    return out
 
 
-def fetch_newsdata_articles(category=None, limit=20, query=None, language=None):
-    base_url = "https://newsdata.io/api/1/news"
-    allowed = [
-        "business",
-        "entertainment",
-        "environment",
-        "food",
-        "health",
-        "politics",
-        "science",
-        "sports",
-        "technology",
-        "top",
-        "world",
-    ]
-    params = {"apikey": NEWSDATA_API_KEY, "language": (language or "en")}
-    if category and category in allowed:
-        params["category"] = category
-    if query:
-        params["q"] = query
-
-    all_articles, next_page = [], None
-    while len(all_articles) < limit:
-        if next_page:
-            params["page"] = next_page
-        resp = _session_get(base_url, params=params, timeout=12)
-        if resp.status_code != 200:
-            break
-        data = resp.json()
-        articles = data.get("results", [])
-        for a in articles:
-            a["api_source"] = "newsdata"
-        all_articles.extend(articles)
-        next_page = data.get("nextPage")
-        if not next_page:
-            break
-
-    # normalize timestamps if available
-    for a in all_articles:
-        a["publishedAt"] = a.get("pubDate")
-    return all_articles[:limit]
+# ----------------- Provider Flags / Keys / Logging -----------------
+USE_GNEWS_API = False
+USE_NEWSDATA_API = False
+USE_GDELT_API = True
+USE_NEWSAPI = False
 
+NEWSAPI_KEY = os.getenv("NEWSAPI_KEY", "ea734c66dc4044fa8e4501ad7b90e753")
+GNEWS_API_KEY = os.getenv("GNEWS_API_KEY", "5419897c95e8a4b21074e0d3fe95a3dd")
+NEWSDATA_API_KEY = os.getenv("NEWSDATA_API_KEY", "pub_1feb49a71a844719af68d0844fb43a61")
+HUGGINGFACE_API_TOKEN = os.getenv("HUGGINGFACE_API_TOKEN")
 
-def fetch_gnews_articles(limit=20, query=None, language=None):
-    url = f"https://gnews.io/api/v4/top-headlines?lang={(language or 'en')}&max={limit}&token={GNEWS_API_KEY}"
-    if query:
-        url += f"&q={requests.utils.quote(query)}"
-    try:
-        r = _session_get(url, timeout=12)
-        if r.status_code != 200:
-            return []
-        arts = r.json().get("articles", [])
-        for a in arts:
-            a["api_source"] = "gnews"
-        return arts
-    except Exception:
-        return []
-
-
-NEWSAPI_COUNTRIES = ["us", "gb", "ca", "au", "in", "za", "sg", "ie", "nz"]
-
+logging.basicConfig(
+    level=logging.WARNING,
+    format="%(levelname)s:%(name)s:%(message)s",
+)
+log = logging.getLogger("newsglobe")
+log.setLevel(logging.WARNING)
+fetch_log = logging.getLogger("newsglobe.fetch_summary")
+fetch_log.setLevel(logging.INFO)
+_fetch_handler = logging.StreamHandler()
+_fetch_handler.setLevel(logging.INFO)
+_fetch_handler.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s"))
+fetch_log.addHandler(_fetch_handler)
+fetch_log.propagate = False
 
-def fetch_newsapi_headlines_multi(limit=50, language=None):
-    if not _newsapi_enabled():
-        return []
-    all_ = []
-    per = max(1, math.ceil(limit / max(1, len(NEWSAPI_COUNTRIES))))
-    per = min(per, 100)  # NewsAPI pageSize cap
-    for c in NEWSAPI_COUNTRIES:
-        url = f"https://newsapi.org/v2/top-headlines?country={c}&pageSize={per}&apiKey={NEWSAPI_KEY}"
-        r = _session_get(url, timeout=12)
-        if r.status_code != 200:
-            log.warning(f"NewsAPI top-headlines {c} -> HTTP {r.status_code}: {r.text[:200]}")
-            continue
-        arts = r.json().get("articles", [])
-        for a in arts:
-            a["api_source"] = "newsapi"
-        all_.extend(arts)
-        time.sleep(0.2)
-    return all_[:limit]  # ✅ enforce exact limit
+for name in ("urllib3", "urllib3.connectionpool", "requests.packages.urllib3"):
+    lg = logging.getLogger(name)
+    lg.setLevel(logging.ERROR)
+    lg.propagate = False
 
+def _newsapi_enabled() -> bool:
+    if not NEWSAPI_KEY:
+        log.warning("NewsAPI disabled: missing NEWSAPI_KEY env var")
+        return False
+    return True
 
-def fetch_newsapi_articles(category=None, limit=20, query=None, language=None):
-    if not _newsapi_enabled():
-        return []
+# ----------------- Clustering Helpers -----------------
+def cluster_id(cluster, enriched_articles):
+    urls = sorted([(enriched_articles[i].get("url") or "") for i in cluster["indices"] if enriched_articles[i].get("url")])
+    base = "|".join(urls) if urls else "empty"
+    return hashlib.md5(base.encode("utf-8")).hexdigest()[:10]
 
-    # If a query is provided, use /everything (language allowed here)
-    if query:
-        url = f"https://newsapi.org/v2/everything?pageSize={limit}&apiKey={NEWSAPI_KEY}&q={requests.utils.quote(query)}"
-        if language:
-            url += f"&language={language}"
-        try:
-            r = _session_get(url, timeout=12)
-            if r.status_code != 200:
-                log.warning(f"NewsAPI /everything HTTP {r.status_code}: {r.text[:200]}")
-                return []
-            arts = r.json().get("articles", [])
-            for a in arts:
-                a["api_source"] = "newsapi"
-            # DO NOT stamp category here; we infer later
-            return arts[:limit]
-        except Exception as e:
-            log.warning(f"NewsAPI /everything request failed: {e}")
-            return []
+BOILER = re.compile(r"\b(live updates|breaking|what we know|in pictures|opinion)\b", re.I)
 
-    # Otherwise, rotate /top-headlines by country (no language param)
-    results = []
-    per_country = max(5, limit // len(NEWSAPI_COUNTRIES))
-    for c in NEWSAPI_COUNTRIES:
-        url = f"https://newsapi.org/v2/top-headlines?country={c}&pageSize={per_country}&apiKey={NEWSAPI_KEY}"
-        if category:
-            url += f"&category={category}"
-        try:
-            r = _session_get(url, timeout=12)
-            if r.status_code != 200:
-                log.warning(f"NewsAPI top-headlines {c} -> HTTP {r.status_code}: {r.text[:200]}")
-                continue
-            arts = r.json().get("articles", [])
-            for a in arts:
-                a["api_source"] = "newsapi"
-            # DO NOT stamp category here; we infer later
-            results.extend(arts)
-        except Exception as e:
-            log.warning(f"NewsAPI top-headlines {c} failed: {e}")
-        time.sleep(0.2)
-    return results[:limit]
+def _norm_text(s: str) -> str:
+    s = (s or "").strip()
+    s = re.sub(r"\s+", " ", s)
+    return s
 
+def _cluster_text(a):
+    base = f"{a.get('orig_title') or a.get('title') or ''} {a.get('orig_description') or a.get('description') or ''}"
+    base = BOILER.sub("", base)
+    base = re.sub(r"\b(\d{1,2}:\d{2}\s?(AM|PM))|\b(\d{1,2}\s\w+\s\d{4})", "", base, flags=re.I)
+    return _norm_text(base)
 
+def _canonical_url(u: str) -> str:
+    if not u:
+        return u
+    p = urlparse(u)
+    qs = [(k, v) for (k, v) in parse_qsl(p.query, keep_blank_values=False) if not k.lower().startswith(("utm_", "fbclid", "gclid"))]
+    clean = p._replace(query="&".join([f"{k}={v}" for k, v in qs]), fragment="")
+    path = clean.path.rstrip("/") or "/"
+    clean = clean._replace(path=path)
+    return urlunparse(clean)
 
+# ----------------- Normalizers / Enrichment -----------------
 def normalize_newsdata_article(article):
     return {
         "title": article.get("title"),
@@ -1479,10 +1102,7 @@ def normalize_newsdata_article(article):
         "category": ((article.get("category") or [None])[0] if isinstance(article.get("category"), list) else article.get("category")),
     }
 
-
-# ----------------- Enrichment -----------------
 def enrich_article(a, language=None, translate=False, target_lang=None):
-    # Normalize source name
     source_name = (a.get("source", {}) or {}).get("name", "").strip() or "Unknown"
     s_lower = source_name.lower()
     if "newsapi" in s_lower:
@@ -1491,16 +1111,12 @@ def enrich_article(a, language=None, translate=False, target_lang=None):
         source_name = "GNews"
     elif "newsdata" in s_lower:
         source_name = "NewsData.io"
-
-    # Canonicalize URL & derive domain
     article_url = _canonical_url(a.get("url") or "")
     try:
         ext = _tld(article_url)
         domain = ".".join([p for p in (ext.domain, ext.suffix) if p]) if (ext.domain or ext.suffix) else ""
     except Exception:
         domain = ""
-
-    # Country guess (GDELT provides ISO2)
     country_guess = None
     if a.get("api_source") == "gdelt":
         sc = a.get("gdelt_sourcecountry")
@@ -1516,17 +1132,11 @@ def enrich_article(a, language=None, translate=False, target_lang=None):
                 "PH": "Philippines", "ID": "Indonesia", "NZ": "New Zealand",
             }
             country_guess = iso2map.get(str(sc).upper(), sc if len(str(sc)) > 2 else None)
-
     coords = get_country_centroid(country_guess) if country_guess else geocode_source(source_name, domain, do_network=False)
-
-    # Title / description (raw)
     title = (a.get("title") or "").strip() or "(untitled)"
     description = (a.get("description") or "").strip()
-
     if description.lower().startswith("no description"):
         description = ""
-
-    # Prefer cached page summary when weak/title-like
     cached_desc = _desc_cache_get(article_url)
     need_upgrade = (
         (not description)
@@ -1536,26 +1146,18 @@ def enrich_article(a, language=None, translate=False, target_lang=None):
     )
     if need_upgrade and cached_desc:
         description = cached_desc
-
     if description:
         description = _tidy_description(title, description, source_name)
     if (not description) or _too_similar(title, description):
         description = f"Quick take: {title.rstrip('.')}."
-
-    # Save originals for categorization and debug
     orig_title = title
     orig_description = description
-
-    # Language detection / sentiment
     detected_lang = (detect_lang(f"{title} {description}") or "").lower()
     ml_text = f"{orig_title}. {orig_description}".strip()
     sentiment = classify_sentiment(f"{orig_title} {orig_description}")
-
-    # Stable id & category (ALWAYS infer; ignore provider/requested categories)
     seed = f"{source_name}|{article_url}|{title}"
     uid = hashlib.md5(seed.encode("utf-8")).hexdigest()[:12]
     cat = infer_category(article_url, orig_title, orig_description, None)
-
     return {
         "id": uid,
         "title": title,
@@ -1576,25 +1178,18 @@ def enrich_article(a, language=None, translate=False, target_lang=None):
         "category": cat,
     }
 
-
-
-
-# ----------------- Clustering into Events -----------------
-# sbert_model = SentenceTransformer("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
-
-# cluster_articles()
-def cluster_articles(articles: List[Dict[str, Any]], sim_threshold=SIM_THRESHOLD, speed=Speed.balanced):
+# ----------------- Clustering (Semantic, single-pass + merge) -----------------
+def cluster_articles(articles: List[Dict[str, Any]], sim_threshold=0.6, speed: Speed = Speed.balanced):
     if speed == Speed.fast:
-        articles = articles[:150]  # early cap
+        articles = articles[:150]
         sim_threshold = max(sim_threshold, 0.64)
     elif speed == Speed.balanced:
         articles = articles[:]
         sim_threshold = max(sim_threshold, 0.62)
     texts = [_cluster_text(a) for a in articles]
     embs = get_sbert().encode(texts, convert_to_tensor=True, normalize_embeddings=True, show_progress_bar=False)
-    clusters = []  # [{indices:[...], centroid:tensor}]
+    clusters = []
     centroids = []
-
     for i, emb in enumerate(embs):
         best_idx, best_sim = -1, -1.0
         for ci, c_emb in enumerate(centroids):
@@ -1609,21 +1204,14 @@ def cluster_articles(articles: List[Dict[str, Any]], sim_threshold=SIM_THRESHOLD
             centroids[best_idx] = new_c
             clusters[best_idx]["centroid"] = new_c
         else:
-            # use texts[i] now (titles[] no longer exists)
             event_id = hashlib.md5(texts[i].encode("utf-8")).hexdigest()[:10]
             clusters.append({"id": event_id, "indices": [i], "centroid": emb})
             centroids.append(emb)
-
-    # second-pass merge to reduce fragmenting
     merged = _merge_close_clusters(clusters, embs, threshold=0.70)
-
-    # keep ids stable: recompute with URLs of member articles
     for c in merged:
         c["id"] = cluster_id(c, articles)
-
     return merged
 
-
 def event_payload_from_cluster(cluster, enriched_articles):
     idxs = cluster["indices"]
     arts = [enriched_articles[i] for i in idxs]
@@ -1634,7 +1222,7 @@ def event_payload_from_cluster(cluster, enriched_articles):
     countries = {a["country"] for a in arts if a["country"] and a["country"] != "Unknown"}
     ts = [a.get("publishedAt") for a in arts if a.get("publishedAt")]
     return {
-        "event_id": cluster_id(cluster, enriched_articles),  # <-- stable id
+        "event_id": cluster_id(cluster, enriched_articles),
         "title": canonical_title,
         "keywords": keywords,
         "article_count": len(arts),
@@ -1644,30 +1232,24 @@ def event_payload_from_cluster(cluster, enriched_articles):
         "sample_urls": [a["url"] for a in arts[:3] if a.get("url")],
     }
 
-
 def aggregate_event_by_country(cluster, enriched_articles):
     idxs = cluster["indices"]
     arts = [enriched_articles[i] for i in idxs]
     by_country: Dict[str, Dict[str, Any]] = {}
-
     for a in arts:
         c = a.get("country") or "Unknown"
         if c not in by_country:
             coords = get_country_centroid(c)
             by_country[c] = {"country": c, "lat": coords["lat"], "lon": coords["lon"], "articles": []}
         by_country[c]["articles"].append(a)
-
-    # summarize per country
     results = []
     for c, block in by_country.items():
         arr = block["articles"]
-        # avg sentiment mapped to -1/0/+1
         to_num = {"negative": -1, "neutral": 0, "positive": 1}
         vals = [to_num.get(a["sentiment"], 0) for a in arr]
         avg = sum(vals) / max(len(vals), 1)
         avg_sent = "positive" if avg > 0.15 else "negative" if avg < -0.15 else "neutral"
         top_sources = [s for s, _ in Counter([a["source"] for a in arr]).most_common(3)]
-        # tiny extractive summary: top 2 headlines
         summary = " • ".join([a["title"] for a in arr[:2]])
         results.append(
             {
@@ -1682,7 +1264,7 @@ def aggregate_event_by_country(cluster, enriched_articles):
                     {
                         "title": a["title"],
                         "orig_title": a.get("orig_title"),
-                        "orig_description": a.get("orig_description"),  # 👈 add this
+                        "orig_description": a.get("orig_description"),
                         "url": a["url"],
                         "source": a["source"],
                         "sentiment": a["sentiment"],
@@ -1694,9 +1276,7 @@ def aggregate_event_by_country(cluster, enriched_articles):
         )
     return results
 
-
 def _merge_close_clusters(clusters, embs, threshold=0.68):
-    # clusters: [{"indices":[...], "centroid":tensor}, ...] – add centroid in your first pass
     merged = []
     used = set()
     for i in range(len(clusters)):
@@ -1710,23 +1290,242 @@ def _merge_close_clusters(clusters, embs, threshold=0.68):
             sim = util.cos_sim(base["centroid"], clusters[j]["centroid"]).item()
             if sim >= threshold:
                 group.append(j)
-        # merge those groups
         all_idx = []
         cents = []
         for g in group:
             used.add(g)
             all_idx.extend(clusters[g]["indices"])
             cents.append(clusters[g]["centroid"])
-        # new centroid
         newc = torch.stack(cents, dim=0).mean(dim=0)
         newc = newc / newc.norm()
         merged.append({"indices": sorted(set(all_idx)), "centroid": newc})
     return merged
 
+# ----------------- Event Cache / Keys -----------------
+CACHE_TTL_SECS = 900
+SIM_THRESHOLD = 0.6
+_events_cache: Dict[Tuple, Dict[str, Any]] = {}
+
+def cache_key_for(q, category, language, limit_each, translate=False, target_lang=None, speed=Speed.balanced):
+    return (q or "", category or "", language or "", int(limit_each or 50),
+            bool(translate), (target_lang or "").lower(), speed.value)
+
+_first_real_build = True
+
+def get_or_build_events_cache(q, category, language, translate, target_lang, limit_each, speed=Speed.balanced):
+    global _first_real_build
+    key = cache_key_for(q, category, language, limit_each, translate, target_lang, speed)
+    now = monotonic()
+    if speed == Speed.fast:
+        use_timespan, use_limit = "24h", min(limit_each, 20)
+    elif speed == Speed.balanced:
+        use_timespan, use_limit = "48h", min(limit_each, 150)
+    else:
+        use_timespan, use_limit = "3d", limit_each
+    entry = _events_cache.get(key)
+    if entry and now - entry["t"] < CACHE_TTL_SECS:
+        log.info(f"CACHE HIT for {key}")
+        return key, entry["enriched"], entry["clusters"]
+    lock = _get_inflight_lock(key)
+    with lock:
+        entry = _events_cache.get(key)
+        if entry and now - entry["t"] < CACHE_TTL_SECS:
+            log.info(f"CACHE HIT (post-lock) for {key}")
+            return key, entry["enriched"], entry["clusters"]
+        if _first_real_build:
+            use_timespan = "24h" if use_timespan != "24h" else use_timespan
+            use_limit = min(use_limit, 100)
+        log.info(f"CACHE MISS for {key} — fetching (timespan={use_timespan}, limit_each={use_limit})")
+        raw = combine_raw_articles(
+            category=category,
+            query=q,
+            language=language,
+            limit_each=use_limit,
+            timespan=use_timespan,
+            speed=speed,
+        )
+        prefetch_descriptions_async(raw, speed)
+        enriched_all = [enrich_article(a, language=language, translate=False, target_lang=None) for a in raw]
+        if category:
+            cat_norm = (category or "").strip().lower()
+            enriched = [e for e in enriched_all if (e.get("category") or "").lower() == cat_norm]
+        else:
+            enriched = enriched_all
+        clusters = cluster_articles(enriched, sim_threshold=SIM_THRESHOLD, speed=speed)
+        _events_cache[key] = {"t": monotonic(), "enriched": enriched, "clusters": clusters}
+        _first_real_build = False
+        return key, enriched, clusters
 
-# ----------------- Endpoints -----------------
-prefetch = False
+# ----------------- Language Rotation / Seeds -----------------
+LANG_ROTATION = ["en", "es", "fr", "de", "ar", "ru", "pt", "zh", "hi", "ja", "ko"]
+COUNTRY_SEEDS = ["US", "GB", "IN", "CA", "AU", "ZA", "SG", "NG", "DE", "FR", "BR", "MX", "ES", "RU", "JP", "KR", "CN"]
 
+# ----------------- Other Providers (NewsData/GNews/NewsAPI) -----------------
+def fetch_newsdata_articles(category=None, limit=20, query=None, language=None):
+    base_url = "https://newsdata.io/api/1/news"
+    allowed = [
+        "business",
+        "entertainment",
+        "environment",
+        "food",
+        "health",
+        "politics",
+        "science",
+        "sports",
+        "technology",
+        "top",
+        "world",
+    ]
+    params = {"apikey": NEWSDATA_API_KEY, "language": (language or "en")}
+    if category and category in allowed:
+        params["category"] = category
+    if query:
+        params["q"] = query
+    all_articles, next_page = [], None
+    while len(all_articles) < limit:
+        if next_page:
+            params["page"] = next_page
+        resp = _session_get(base_url, params=params, timeout=12)
+        if resp.status_code != 200:
+            break
+        data = resp.json()
+        articles = data.get("results", [])
+        for a in articles:
+            a["api_source"] = "newsdata"
+        all_articles.extend(articles)
+        next_page = data.get("nextPage")
+        if not next_page:
+            break
+    for a in all_articles:
+        a["publishedAt"] = a.get("pubDate")
+    return all_articles[:limit]
+
+def fetch_gnews_articles(limit=20, query=None, language=None):
+    url = f"https://gnews.io/api/v4/top-headlines?lang={(language or 'en')}&max={limit}&token={GNEWS_API_KEY}"
+    if query:
+        url += f"&q={requests.utils.quote(query)}"
+    try:
+        r = _session_get(url, timeout=12)
+        if r.status_code != 200:
+            return []
+        arts = r.json().get("articles", [])
+        for a in arts:
+            a["api_source"] = "gnews"
+        return arts
+    except Exception:
+        return []
+
+NEWSAPI_COUNTRIES = ["us", "gb", "ca", "au", "in", "za", "sg", "ie", "nz"]
+
+def fetch_newsapi_headlines_multi(limit=50, language=None):
+    if not _newsapi_enabled():
+        return []
+    all_ = []
+    per = max(1, math.ceil(limit / max(1, len(NEWSAPI_COUNTRIES))))
+    per = min(per, 100)
+    for c in NEWSAPI_COUNTRIES:
+        url = f"https://newsapi.org/v2/top-headlines?country={c}&pageSize={per}&apiKey={NEWSAPI_KEY}"
+        r = _session_get(url, timeout=12)
+        if r.status_code != 200:
+            log.warning(f"NewsAPI top-headlines {c} -> HTTP {r.status_code}: {r.text[:200]}")
+            continue
+        arts = r.json().get("articles", [])
+        for a in arts:
+            a["api_source"] = "newsapi"
+        all_.extend(arts)
+        time.sleep(0.2)
+    return all_[:limit]
+
+def fetch_newsapi_articles(category=None, limit=20, query=None, language=None):
+    if not _newsapi_enabled():
+        return []
+    if query:
+        url = f"https://newsapi.org/v2/everything?pageSize={limit}&apiKey={NEWSAPI_KEY}&q={requests.utils.quote(query)}"
+        if language:
+            url += f"&language={language}"
+        try:
+            r = _session_get(url, timeout=12)
+            if r.status_code != 200:
+                log.warning(f"NewsAPI /everything HTTP {r.status_code}: {r.text[:200]}")
+                return []
+            arts = r.json().get("articles", [])
+            for a in arts:
+                a["api_source"] = "newsapi"
+            return arts[:limit]
+        except Exception as e:
+            log.warning(f"NewsAPI /everything request failed: {e}")
+            return []
+    results = []
+    per_country = max(5, limit // len(NEWSAPI_COUNTRIES))
+    for c in NEWSAPI_COUNTRIES:
+        url = f"https://newsapi.org/v2/top-headlines?country={c}&pageSize={per_country}&apiKey={NEWSAPI_KEY}"
+        if category:
+            url += f"&category={category}"
+        try:
+            r = _session_get(url, timeout=12)
+            if r.status_code != 200:
+                log.warning(f"NewsAPI top-headlines {c} -> HTTP {r.status_code}: {r.text[:200]}")
+                continue
+            arts = r.json().get("articles", [])
+            for a in arts:
+                a["api_source"] = "newsapi"
+            results.extend(arts)
+        except Exception as e:
+            log.warning(f"NewsAPI top-headlines {c} failed: {e}")
+        time.sleep(0.2)
+    return results[:limit]
+
+# ----------------- Provider Combiner / Dedup -----------------
+def combine_raw_articles(category=None, query=None, language=None, limit_each=30,
+                         timespan="3d", speed=Speed.balanced, log_summary: bool = True):
+    if speed == Speed.fast:
+        timespan = "24h"
+        limit_each = min(limit_each, 20)
+    elif speed == Speed.balanced:
+        timespan = "48h"
+        limit_each = min(limit_each, 150)
+    a1 = []
+    if USE_NEWSAPI:
+        if not query:
+            a1 = fetch_newsapi_headlines_multi(limit=limit_each, language=language)
+        else:
+            a1 = fetch_newsapi_articles(category=category, limit=limit_each, query=query, language=language)
+    a2 = []
+    if USE_NEWSDATA_API:
+        a2 = [
+            normalize_newsdata_article(a)
+            for a in fetch_newsdata_articles(category=category, limit=limit_each, query=query, language=language)
+            if a.get("link")
+        ]
+    a3 = fetch_gnews_articles(limit=limit_each, query=query, language=language) if USE_GNEWS_API else []
+    gdelt_limit = limit_each
+    a4 = fetch_gdelt_multi(
+        limit=gdelt_limit,
+        query=query,
+        language=language,
+        timespan=timespan,
+        category=category,
+        speed=speed,
+    )
+    seen, merged = set(), []
+    for a in a1 + a3 + a2 + a4:
+        if a.get("url"):
+            a["url"] = _canonical_url(a["url"])
+            url = a["url"]
+            if url not in seen:
+                seen.add(url)
+                merged.append(a)
+    if log_summary:
+        fetch_log.info("----- Article Fetch Summary -----")
+        fetch_log.info(f"📊 NewsAPI returned: {len(a1)} articles")
+        fetch_log.info(f"📊 NewsData.io returned: {len(a2)} articles")
+        fetch_log.info(f"📊 GNews returned: {len(a3)} articles")
+        fetch_log.info(f"📊 GDELT returned: {len(a4)} articles")
+        fetch_log.info(f"✅ Total merged articles after deduplication: {len(merged)}")
+        fetch_log.info("---------------------------------")
+    return merged
+
+# ----------------- API: /events -----------------
 @app.get("/events")
 def get_events(
     q: Optional[str] = Query(None),
@@ -1740,13 +1539,9 @@ def get_events(
     min_articles: int = Query(2, ge=1, le=200),
     speed: Speed = Query(Speed.balanced),
 ):
-
-    # always build cache on untranslated data
     cache_key, enriched, clusters = get_or_build_events_cache(
         q, category, language, False, None, limit_each, speed=speed
     )
-
-    # optional post-translate view (does not mutate cache)
     view = enriched
     if translate and target_lang:
         view = [dict(i) for i in enriched]
@@ -1755,13 +1550,12 @@ def get_events(
             i["title"] = translate_text(i.get("title") or "", target_lang, fallback_src=src_hint)
             i["description"] = translate_text(i.get("description") or "", target_lang, fallback_src=src_hint)
             i["translated"] = True
-
     events = [event_payload_from_cluster(c, view) for c in clusters]
     events = [e for e in events if (e["country_count"] >= min_countries and e["article_count"] >= min_articles)]
     events.sort(key=lambda e: e["article_count"], reverse=True)
-
     return {"events": events[:max_events], "cache_key": "|".join(map(str, cache_key))}
 
+# ----------------- API: /event/{event_id} -----------------
 @app.get("/event/{event_id}")
 def get_event_details(
     event_id: str,
@@ -1773,14 +1567,13 @@ def get_event_details(
     target_lang: Optional[str] = Query(None),
     limit_each: int = Query(150, ge=5, le=250),
 ):
-    # /event/{event_id}
     if cache_key:
         parts = cache_key.split("|")
         if len(parts) != 7:
             raise HTTPException(status_code=400, detail="Bad cache_key")
         speed_str = parts[6]
         try:
-            speed_obj = Speed(speed_str)  # "fast" | "balanced" | "max"
+            speed_obj = Speed(speed_str)
         except ValueError:
             speed_obj = Speed.balanced
         key_tuple = (parts[0], parts[1], parts[2], int(parts[3]),
@@ -1788,17 +1581,13 @@ def get_event_details(
     else:
         speed_obj = Speed.balanced
         key_tuple = cache_key_for(q, category, language, limit_each, translate, target_lang, speed=speed_obj)
-
     entry = _events_cache.get(key_tuple)
     if not entry:
-        # always build untranslated
         _, enriched, clusters = get_or_build_events_cache(
             q, category, language, False, None, limit_each, speed=speed_obj
         )
     else:
         enriched, clusters = entry["enriched"], entry["clusters"]
-
-    # optional post-translate view (do not mutate cache)
     eview = enriched
     if translate and target_lang:
         eview = [dict(i) for i in enriched]
@@ -1807,17 +1596,15 @@ def get_event_details(
             i["title"] = translate_text(i.get("title") or "", target_lang, fallback_src=src_hint)
             i["description"] = translate_text(i.get("description") or "", target_lang, fallback_src=src_hint)
             i["translated"] = True
-
     cluster = next((c for c in clusters if cluster_id(c, enriched) == event_id), None)
     if not cluster:
         raise HTTPException(status_code=404, detail="Event not found with current filters")
-
     payload = event_payload_from_cluster(cluster, eview)
     countries = aggregate_event_by_country(cluster, eview)
     payload["articles_in_event"] = sum(c["count"] for c in countries)
     return {"event": payload, "countries": countries}
 
-
+# ----------------- API: /news -----------------
 @app.get("/news")
 def get_news(
     cache_key: Optional[str] = Query(None),
@@ -1834,25 +1621,21 @@ def get_news(
     page_size: int = Query(120, ge=5, le=300),
 ):
     enriched: List[Dict[str, Any]] = []
-
-    # Pull from cache if provided
     if cache_key:
         parts = cache_key.split("|")
         if len(parts) == 7:
             key_tuple = (
-                parts[0],  # q
-                parts[1],  # category
-                parts[2],  # language
-                int(parts[3]),  # limit_each
-                parts[4] == "True",  # translate
-                parts[5].lower(),  # target_lang
-                parts[6],  # speed
+                parts[0],
+                parts[1],
+                parts[2],
+                int(parts[3]),
+                parts[4] == "True",
+                parts[5].lower(),
+                parts[6],
             )
             entry = _events_cache.get(key_tuple)
             if entry:
                 enriched = entry["enriched"]
-
-    # Fetch fresh if no cached items
     if not enriched:
         raw = combine_raw_articles(category=category, query=q, language=language, limit_each=limit_each, speed=speed)
         prefetch_descriptions_async(raw, speed)
@@ -1863,12 +1646,9 @@ def get_news(
         else:
             enriched = enriched_all
     else:
-        # If we got cached items but want to ensure the selected category is enforced:
         if category:
             cat_norm = (category or "").strip().lower()
             enriched = [e for e in enriched if (e.get("category") or "").lower() == cat_norm]
-
-    # Translation (optional)
     if translate and target_lang:
         enriched = [dict(i) for i in enriched]
         for i in enriched:
@@ -1880,25 +1660,18 @@ def get_news(
             i["translated"] = True
             i["translated_from"] = (src_hint or "").lower()
             i["translated_to"] = target_lang.lower()
-
-    # Optional sentiment filter
     if sentiment:
         s = sentiment.strip().lower()
         enriched = [i for i in enriched if i.get("sentiment", "").lower() == s]
-
-    # Pagination
     total = len(enriched)
     start = (page - 1) * page_size
     end = start + page_size
     items = [dict(i) for i in enriched[start:end]]
-
-    # Trim debug fields
     if lite:
         drop = {"_ml_text"}
         for i in items:
             for k in drop:
                 i.pop(k, None)
-
     return {
         "items": items,
         "total": total,
@@ -1906,74 +1679,7 @@ def get_news(
         "page_size": page_size
     }
 
-
-
-
-
-def combine_raw_articles(category=None, query=None, language=None, limit_each=30,
-                         timespan="3d", speed=Speed.balanced, log_summary: bool = True):
-    if speed == Speed.fast:
-        timespan = "24h"
-        limit_each = min(limit_each, 20)
-    elif speed == Speed.balanced:
-        timespan = "48h"
-        limit_each = min(limit_each, 150)
-
-    a1 = []
-    if USE_NEWSAPI:
-        if not query:
-            a1 = fetch_newsapi_headlines_multi(limit=limit_each, language=language)
-        else:
-            a1 = fetch_newsapi_articles(category=category, limit=limit_each, query=query, language=language)
-
-    a2 = []
-    if USE_NEWSDATA_API:
-        a2 = [
-            normalize_newsdata_article(a)
-            for a in fetch_newsdata_articles(category=category, limit=limit_each, query=query, language=language)
-            if a.get("link")
-        ]
-
-    a3 = fetch_gnews_articles(limit=limit_each, query=query, language=language) if USE_GNEWS_API else []
-    # a4 = fetch_gdelt_articles(
-    #     limit=min(100, limit_each * 2),
-    #     query=query,
-    #     language=language,
-    #     timespan=timespan,
-    #     category=category
-    # )
-    gdelt_limit = limit_each
-    a4 = fetch_gdelt_multi(
-        limit=gdelt_limit,
-        query=query,
-        language=language,   # if provided, we honor it (with small EN boost)
-        timespan=timespan,
-        category=category,
-        speed=speed,
-    )
-
-    # Dedup by canonical URL (maintain source precedence)
-    seen, merged = set(), []
-    for a in a1 + a3 + a2 + a4:
-        if a.get("url"):
-            a["url"] = _canonical_url(a["url"])
-            url = a["url"]
-            if url not in seen:
-                seen.add(url)
-                merged.append(a)
-
-    if log_summary:
-        fetch_log.info("----- Article Fetch Summary -----")
-        fetch_log.info(f"📊 NewsAPI returned: {len(a1)} articles")
-        fetch_log.info(f"📊 NewsData.io returned: {len(a2)} articles")
-        fetch_log.info(f"📊 GNews returned: {len(a3)} articles")
-        fetch_log.info(f"📊 GDELT returned: {len(a4)} articles")
-        fetch_log.info(f"✅ Total merged articles after deduplication: {len(merged)}")
-        fetch_log.info("---------------------------------")
-
-    return merged
-
-
+# ----------------- API: /related -----------------
 @app.get("/related")
 def related_articles(
     id: Optional[str] = Query(None, description="article id from /news"),
@@ -1985,13 +1691,10 @@ def related_articles(
     limit_each: int = Query(50, ge=5, le=100),
     k: int = Query(10, ge=1, le=50),
 ):
-    # ensure we have a working article list (enriched) to search over
     raw = combine_raw_articles(category=category, query=q, language=language, limit_each=limit_each)
     enriched = [enrich_article(a, language=language, translate=False, target_lang=None) for a in raw]
     if not enriched:
         return {"items": []}
-
-    # pick the query vector
     if id:
         base = next((a for a in enriched if a.get("id") == id), None)
         if not base:
@@ -2002,15 +1705,10 @@ def related_articles(
         if not text:
             raise HTTPException(400, "provide either id or title/description")
         query_text = text
-
     corpus_texts = [a["_ml_text"] for a in enriched]
     corpus_embs = _embed_texts(corpus_texts)
     query_emb = _embed_texts([query_text])[0]
-
-    # cosine similarities
     sims = util.cos_sim(query_emb, corpus_embs).cpu().numpy().flatten()
-
-    # take top-k excluding the query itself (if id provided)
     idxs = sims.argsort()[::-1]
     items = []
     for idx in idxs:
@@ -2020,9 +1718,9 @@ def related_articles(
         items.append({**a, "similarity": float(sims[idx])})
         if len(items) >= k:
             break
-
     return {"items": items}
 
+# ----------------- Middleware: Request Timing -----------------
 @app.middleware("http")
 async def timing_middleware(request, call_next):
     start = time.perf_counter()
@@ -2032,21 +1730,21 @@ async def timing_middleware(request, call_next):
         return response
     finally:
         dur_ms = (time.perf_counter() - start) * 1000
-        # log.info(f"{request.method} {request.url.path} -> {dur_ms:.1f} ms ({_fmt_mmss(dur_ms)})")
         if response is not None:
             try:
                 response.headers["X-Process-Time-ms"] = f"{dur_ms:.1f}"
             except Exception:
                 pass
 
+# ----------------- Misc: Client Metrics -----------------
 @app.post("/client-metric")
 def client_metric(payload: Dict[str, Any] = Body(...)):
     name = (payload.get("name") or "").strip()
-    # Drop redraw spam if it ever slips through again
     if name in {"Load all article markers on globe", "Load event country markers on globe"}:
         return {"ok": True}
     return {"ok": True}
 
+# ----------------- Diagnostics: Translation Health -----------------
 @app.get("/diag/translate")
 def diag_translate():
     remote = _hf_call("Helsinki-NLP/opus-mt-es-en", {"inputs":"Hola mundo"})