Delete app.py

app.py

@@ -1,491 +0,0 @@
-import os
-import re
-import urllib.request
-import xml.etree.ElementTree as ET
-from email.utils import parsedate_to_datetime
-from urllib.parse import quote
-
-import gradio as gr
-import pandas as pd
-import torch
-from peft import PeftModel
-from transformers import AutoModelForSequenceClassification, AutoTokenizer
-
-# ================================================================
-# CONFIG
-# ================================================================
-
-BASE_MODEL = "distilbert-base-uncased"
-ADAPTER_PATH = "."
-MAX_LENGTH = 128
-BATCH_SIZE = 32
-
-LABELS = {0: "negative", 1: "neutral", 2: "positive"}
-
-DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-# ================================================================
-# MODEL LOADING (DistilBERT + LoRA)
-# ================================================================
-
-print("Device:", DEVICE)
-
-tokenizer = AutoTokenizer.from_pretrained(ADAPTER_PATH, use_fast=True)
-
-base_model = AutoModelForSequenceClassification.from_pretrained(
-    BASE_MODEL,
-    num_labels=3,
-    id2label=LABELS,
-    label2id={v: k for k, v in LABELS.items()},
-    ignore_mismatched_sizes=True,
-)
-
-model = PeftModel.from_pretrained(base_model, ADAPTER_PATH)
-try:
-    model = model.merge_and_unload()
-except Exception as e:
-    print("Merge skipped:", e)
-
-model = model.to(DEVICE)
-model.eval()
-print("Model ready.")
-
-
-# ================================================================
-# SENTIMENT (batch)
-# ================================================================
-
-def predict_batch(texts):
-    """Her metin için (p_neg, p_neu, p_pos) döndürür."""
-    out = []
-    for i in range(0, len(texts), BATCH_SIZE):
-        chunk = texts[i:i + BATCH_SIZE]
-        inputs = tokenizer(
-            chunk,
-            return_tensors="pt",
-            truncation=True,
-            max_length=MAX_LENGTH,
-            padding=True,
-        )
-        inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
-        with torch.inference_mode():
-            probs = torch.softmax(model(**inputs).logits, dim=-1).cpu().tolist()
-        out.extend(probs)
-    return out
-
-
-# ================================================================
-# RISK KURALLARI (anahtar kelime tabanlı sinyaller)
-# ================================================================
-
-RISK_SIGNALS = {
-    "Hasar / Claim": [
-        "claim", "claims", "payout", "payouts", "loss", "losses",
-        "combined ratio", "reserve", "reserves",
-    ],
-    "Doğal afet": [
-        "hurricane", "earthquake", "flood", "flooding", "wildfire", "storm",
-        "catastrophe", "cyclone", "typhoon", "tornado", "hailstorm", "drought",
-    ],
-    "Büyük kaza": [
-        "crash", "explosion", "collapse", "derailment", "blast", "accident",
-        "fire", "sinking", "spill",
-    ],
-    "Dava / Fraud": [
-        "lawsuit", "sued", "litigation", "fraud", "fine", "fined", "penalty",
-        "investigation", "probe", "settlement", "class action", "regulator",
-    ],
-    "Finansal stres": [
-        "downgrade", "downgraded", "profit warning", "solvency", "writedown",
-        "write-down", "impairment", "insolvency", "bankruptcy", "capital shortfall",
-        "negative outlook", "slump", "plunge",
-    ],
-    "Siber / Operasyonel": [
-        "cyberattack", "cyber attack", "data breach", "breach", "ransomware",
-        "outage",
-    ],
-}
-
-TOPICS = {
-    "Motor insurance": ["motor", "auto insurance", "car insurance", "vehicle", "automobile"],
-    "Health insurance": ["health insurance", "health", "medical", "hospital"],
-    "Catastrophe risk": ["catastrophe", "hurricane", "earthquake", "flood", "wildfire",
-                         "storm", "cyclone", "typhoon", "tornado", "hailstorm"],
-    "Life insurance": ["life insurance", "life insurer", "annuity", "annuities", "pension", "mortality"],
-    "Reinsurance": ["reinsur", "retrocession", "cat bond", "catastrophe bond"],
-    "Property": ["property insurance", "homeowner", "home insurance", "real estate", "fire"],
-    "Cyber": ["cyber", "ransomware", "data breach"],
-}
-
-COMPANIES = [
-    "Allianz", "AXA", "Zurich", "Generali", "Aviva", "Munich Re", "Swiss Re",
-    "Hannover Re", "Chubb", "AIG", "MetLife", "Prudential", "Allstate",
-    "Progressive", "State Farm", "Mapfre", "Ageas", "Lloyd's", "Travelers",
-    "Liberty Mutual", "Tokio Marine", "Ping An",
-]
-
-
-def _compile(words):
-    return re.compile(r"\b(?:%s)" % "|".join(re.escape(w) for w in words), re.I)
-
-
-SIGNAL_RE = {cat: _compile(words) for cat, words in RISK_SIGNALS.items()}
-TOPIC_RE = {t: _compile(words) for t, words in TOPICS.items()}
-COMPANY_RE = {c: re.compile(r"\b%s\b" % re.escape(c), re.I) for c in COMPANIES}
-
-
-def find_signals(text):
-    hits = []
-    for cat, rx in SIGNAL_RE.items():
-        m = rx.findall(text)
-        if m:
-            hits.append((cat, sorted({x.lower() for x in m})[:2]))
-    return hits
-
-
-def find_topics(text):
-    return [t for t, rx in TOPIC_RE.items() if rx.search(text)]
-
-
-def find_companies(text):
-    return [c for c, rx in COMPANY_RE.items() if rx.search(text)]
-
-
-def risk_level(score, high=60, medium=35):
-    if score >= high:
-        return "🔴 HIGH"
-    if score >= medium:
-        return "🟠 MEDIUM"
-    return "🟢 LOW"
-
-
-# ================================================================
-# TEK HABER ANALİZİ
-# ================================================================
-
-def score_item(text, probs):
-    p_neg, p_neu, p_pos = probs
-    sentiment = p_pos - p_neg                       # -1 .. +1
-    signals = find_signals(text)
-    n_sig = len(signals)
-    # Risk indeksi: %65 negatif olasılık + %35 risk sinyali yoğunluğu
-    risk = 100 * (0.65 * p_neg + 0.35 * min(n_sig, 3) / 3)
-    label = LABELS[max(range(3), key=lambda i: probs[i])]
-    reason = "; ".join(f"{c} ({', '.join(k)})" for c, k in signals) or "-"
-    return {
-        "label": label,
-        "sentiment": round(sentiment, 3),
-        "signals": n_sig,
-        "risk_score": round(risk, 1),
-        "risk": risk_level(risk),
-        "reason": reason,
-    }
-
-
-def analyze_news(text):
-    text = (text or "").strip()
-    if not text:
-        return "Lütfen bir sigorta haber başlığı girin.", "", ""
-
-    probs = predict_batch([text])[0]
-    s = score_item(text, probs)
-
-    companies = find_companies(text) or ["-"]
-    topics = find_topics(text) or ["-"]
-
-    result = (
-        f"Sentiment: {s['label'].upper()}\n"
-        f"Skor: {s['sentiment']:+.2f}\n"
-        f"Güven: {max(probs) * 100:.1f}%"
-    )
-    prob_text = (
-        f"Negative: {probs[0] * 100:.1f}%\n"
-        f"Neutral: {probs[1] * 100:.1f}%\n"
-        f"Positive: {probs[2] * 100:.1f}%"
-    )
-    risk_text = (
-        f"Şirket: {', '.join(companies)}\n"
-        f"Risk: {s['risk']} ({s['risk_score']}/100)\n"
-        f"Risk sinyali: {s['signals']}\n"
-        f"Neden: {s['reason']}\n"
-        f"Konu: {', '.join(topics)}"
-    )
-    return result, prob_text, risk_text
-
-
-# ================================================================
-# TOPLU ANALİZ PIPELINE
-# ================================================================
-
-def analyze_items(items):
-    """items: [{'headline':str, 'date':datetime|None, 'company':str|None}]"""
-    if not items:
-        return None, None, None
-
-    texts = [it["headline"] for it in items]
-    probs = predict_batch(texts)
-
-    rows = []
-    for it, p in zip(items, probs):
-        s = score_item(it["headline"], p)
-        companies = [it["company"]] if it.get("company") else find_companies(it["headline"])
-        rows.append({
-            "date": it.get("date"),
-            "headline": it["headline"],
-            "companies": companies or ["(Sektör)"],
-            "topics": find_topics(it["headline"]),
-            **s,
-        })
-    df = pd.DataFrame(rows)
-
-    has_dates = df["date"].notna().sum() >= 4
-    if has_dates:
-        df["date"] = pd.to_datetime(df["date"], utc=True, errors="coerce")
-        mid = df["date"].dropna().sort_values().iloc[len(df["date"].dropna()) // 2]
-        df["recent"] = df["date"] >= mid
-    else:
-        df["recent"] = False
-
-    # ---- Şirket tablosu ----
-    ex = df.explode("companies")
-    comp_rows = []
-    for comp, g in ex.groupby("companies"):
-        risk = g["risk_score"].mean()
-        trend = "—"
-        if has_dates:
-            r, e = g[g["recent"]], g[~g["recent"]]
-            if len(r) and len(e):
-                d = r["sentiment"].mean() - e["sentiment"].mean()
-                trend = ("Improving ↑" if d > 0.15
-                         else "Deteriorating ↓" if d < -0.15 else "Stable →")
-        comp_rows.append({
-            "Company": comp,
-            "News": len(g),
-            "Avg sentiment": round(g["sentiment"].mean(), 2),
-            "Risk signals": int(g["signals"].sum()),
-            "High-risk news": int((g["risk_score"] >= 60).sum()),
-            "Risk score": round(risk, 1),
-            "Risk": risk_level(risk, high=55, medium=30),
-            "Trend": trend,
-        })
-    comp_df = pd.DataFrame(comp_rows).sort_values("Risk score", ascending=False)
-
-    # ---- Konu / trend tablosu ----
-    topic_rows = []
-    n_recent = max(int(df["recent"].sum()), 1)
-    n_early = max(int((~df["recent"]).sum()), 1)
-    for topic in TOPICS:
-        mask = df["topics"].apply(lambda t: topic in t)
-        if not mask.any():
-            continue
-        arrow = "—"
-        if has_dates:
-            share_r = (mask & df["recent"]).sum() / n_recent
-            share_e = (mask & ~df["recent"]).sum() / n_early
-            if share_e == 0:
-                arrow = "↑↑"
-            else:
-                ratio = share_r / share_e
-                arrow = ("↑↑" if ratio >= 1.5 else "↑" if ratio >= 1.15
-                         else "↓" if ratio <= 0.8 else "→")
-        topic_rows.append({
-            "Topic": topic,
-            "News": int(mask.sum()),
-            "Avg sentiment": round(df.loc[mask, "sentiment"].mean(), 2),
-            "Trend": arrow,
-        })
-    topic_df = pd.DataFrame(topic_rows).sort_values("News", ascending=False) \
-        if topic_rows else pd.DataFrame(columns=["Topic", "News", "Avg sentiment", "Trend"])
-
-    # ---- Haber tablosu ----
-    news_df = df.assign(
-        companies=df["companies"].apply(", ".join),
-        topics=df["topics"].apply(", ".join),
-        date=df["date"].apply(lambda d: d.strftime("%Y-%m-%d") if pd.notna(d) else ""),
-    )[["date", "companies", "headline", "label", "sentiment", "risk", "risk_score", "reason", "topics"]]
-    news_df = news_df.sort_values("risk_score", ascending=False)
-
-    return news_df, comp_df, topic_df
-
-
-# ================================================================
-# TOPLU GİRİŞ (metin / CSV)
-# ================================================================
-
-def _pick(cols, names):
-    for n in names:
-        if n in cols:
-            return n
-    return None
-
-
-def batch_analyze(text, file):
-    items = []
-
-    path = file if isinstance(file, str) else getattr(file, "name", None)
-    if path:
-        df = pd.read_csv(path)
-        df.columns = [c.strip().lower() for c in df.columns]
-        h = _pick(df.columns, ["headline", "title", "text", "news", "haber", "baslik"])
-        d = _pick(df.columns, ["date", "published", "pubdate", "tarih"])
-        c = _pick(df.columns, ["company", "sirket", "şirket"])
-        if h is None:
-            raise gr.Error("CSV'de 'headline' (veya title/text/news) kolonu bulunamadı.")
-        for _, r in df.iterrows():
-            if pd.isna(r[h]) or not str(r[h]).strip():
-                continue
-            items.append({
-                "headline": str(r[h]).strip(),
-                "date": pd.to_datetime(r[d], utc=True, errors="coerce") if d else None,
-                "company": str(r[c]).strip() if c and pd.notna(r[c]) else None,
-            })
-
-    for line in (text or "").splitlines():
-        line = line.strip()
-        if line:
-            items.append({"headline": line, "date": None, "company": None})
-
-    if not items:
-        raise gr.Error("Haber girin (her satıra bir başlık) veya CSV yükleyin.")
-    return analyze_items(items)
-
-
-# ================================================================
-# CANLI TARAMA (Google News RSS)
-# ================================================================
-
-def fetch_news(company, days, limit):
-    q = quote(f'"{company}" insurance when:{int(days)}d')
-    url = f"https://news.google.com/rss/search?q={q}&hl=en-US&gl=US&ceid=US:en"
-    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
-    with urllib.request.urlopen(req, timeout=15) as resp:
-        root = ET.fromstring(resp.read())
-
-    items = []
-    for it in root.iter("item"):
-        title = (it.findtext("title") or "").strip()
-        title = re.sub(r"\s+-\s+[^-]+$", "", title)    # " - Kaynak" kısmını at
-        try:
-            date = parsedate_to_datetime(it.findtext("pubDate"))
-        except Exception:
-            date = None
-        if title:
-            items.append({"headline": title, "date": date, "company": company})
-        if len(items) >= limit:
-            break
-    return items
-
-
-def live_scan(company_text, days, limit):
-    companies = [c.strip() for c in (company_text or "").split(",") if c.strip()]
-    if not companies:
-        raise gr.Error("En az bir şirket adı girin.")
-
-    items, errors = [], []
-    for c in companies:
-        try:
-            items += fetch_news(c, days, int(limit))
-        except Exception as e:
-            errors.append(f"{c}: {e}")
-
-    if not items:
-        raise gr.Error("Haber alınamadı. İnternet erişimini kontrol edin. " + " | ".join(errors))
-    return analyze_items(items)
-
-
-# ================================================================
-# GRADIO UI
-# ================================================================
-
-EXAMPLES = [
-    ["Insurance company reports record premium growth"],
-    ["Allianz faces major claims increase after severe floods"],
-    ["Zurich maintained stable operations in the third quarter."],
-    ["AXA downgraded after reserve shortfall and fraud investigation"],
-    ["Hurricane losses push reinsurance rates higher"],
-]
-
-BATCH_PLACEHOLDER = (
-    "Allianz reports record premium growth\n"
-    "Generali hit by wildfire claims surge\n"
-    "Zurich faces lawsuit over auto claims"
-)
-
-NEWS_COLS = ["date", "companies", "headline", "label", "sentiment", "risk", "risk_score", "reason", "topics"]
-
-with gr.Blocks(title="Insurance News Risk Scanner") as demo:
-
-    gr.Markdown(
-        "# 🛡️ Insurance News Risk Scanner\n"
-        "Sigorta haberleri için **sentiment + risk sinyali + erken uyarı**.  \n"
-        "Model: DistilBERT + LoRA · Val Macro F1: 78.06% · "
-        "Haber başlıkları **İngilizce** olmalı."
-    )
-
-    # ---------------- TAB 1 ----------------
-    with gr.Tab("📰 Tek Haber"):
-        with gr.Row():
-            with gr.Column():
-                text_input = gr.Textbox(label="Insurance News", lines=4,
-                                        placeholder="Allianz faces major claims increase...")
-                btn = gr.Button("🔍 Analiz Et", variant="primary")
-            with gr.Column():
-                out_pred = gr.Textbox(label="Sentiment", lines=3)
-                out_prob = gr.Textbox(label="Olasılıklar", lines=3)
-                out_risk = gr.Textbox(label="Risk Raporu", lines=5)
-        outs = [out_pred, out_prob, out_risk]
-        btn.click(analyze_news, text_input, outs)
-        text_input.submit(analyze_news, text_input, outs)
-        gr.Examples(EXAMPLES, inputs=text_input)
-
-    # ---------------- TAB 2 ----------------
-    with gr.Tab("📚 Toplu Analiz"):
-        gr.Markdown(
-            "Her satıra bir başlık yazın **veya** CSV yükleyin "
-            "(kolonlar: `headline`, opsiyonel `date`, `company`). "
-            "Tarih varsa **trend** hesaplanır."
-        )
-        b_text = gr.Textbox(label="Başlıklar (satır başına bir haber)", lines=6,
-                            placeholder=BATCH_PLACEHOLDER)
-        b_file = gr.File(label="CSV (opsiyonel)", file_types=[".csv"])
-        b_btn = gr.Button("🚀 Toplu Analiz", variant="primary")
-        gr.Markdown("### 🏢 Şirket Risk Tablosu (Underwriting / Rakip İstihbaratı)")
-        b_comp = gr.Dataframe(wrap=True)
-        gr.Markdown("### 📈 Sektör Konu Trendi")
-        b_topic = gr.Dataframe(wrap=True)
-        gr.Markdown("### 🗞️ Haber Detayı (riskten düşüğe)")
-        b_news = gr.Dataframe(wrap=True)
-        b_btn.click(batch_analyze, [b_text, b_file], [b_news, b_comp, b_topic])
-
-    # ---------------- TAB 3 ----------------
-    with gr.Tab("🌐 Canlı Tarama"):
-        gr.Markdown(
-            "Google News RSS'ten şirket haberlerini çekip otomatik analiz eder "
-            "(sunucunun internet erişimi gerekir)."
-        )
-        l_comp = gr.Textbox(label="Şirketler (virgülle)", value="Allianz, AXA, Zurich, Generali")
-        with gr.Row():
-            l_days = gr.Slider(7, 90, value=30, step=1, label="Son kaç gün")
-            l_limit = gr.Slider(5, 50, value=20, step=1, label="Şirket başına haber")
-        l_btn = gr.Button("🔎 Haberleri Tara", variant="primary")
-        gr.Markdown("### 🏢 Rakip Karşılaştırma")
-        l_compdf = gr.Dataframe(wrap=True)
-        gr.Markdown("### 📈 Konu Trendi")
-        l_topic = gr.Dataframe(wrap=True)
-        gr.Markdown("### 🗞️ Haberler")
-        l_news = gr.Dataframe(wrap=True)
-        l_btn.click(live_scan, [l_comp, l_days, l_limit], [l_news, l_compdf, l_topic])
-
-    gr.Markdown(
-        "---\n"
-        "**Not:** Risk skoru = %65 model negatif olasılığı + %35 anahtar-kelime risk sinyali. "
-        "Karar destek amaçlıdır; tek başına underwriting kararı için kullanılmamalıdır."
-    )
-
-
-if __name__ == "__main__":
-    demo.launch(
-        server_name="0.0.0.0",
-        server_port=int(os.environ.get("PORT", 7860)),
-        show_error=True,
-    )