app.py

import os
import re
import urllib.request
import xml.etree.ElementTree as ET
from email.utils import parsedate_to_datetime
from urllib.parse import quote

import gradio as gr
import pandas as pd
import torch
from peft import PeftModel
from transformers import AutoModelForSequenceClassification, AutoTokenizer

# ================================================================
# CONFIG
# ================================================================

BASE_MODEL = "distilbert-base-uncased"
ADAPTER_PATH = "."
MAX_LENGTH = 128
BATCH_SIZE = 32

LABELS = {0: "negative", 1: "neutral", 2: "positive"}

DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# ================================================================
# MODEL LOADING (DistilBERT + LoRA)
# ================================================================

print("Device:", DEVICE)

tokenizer = AutoTokenizer.from_pretrained(ADAPTER_PATH, use_fast=True)

base_model = AutoModelForSequenceClassification.from_pretrained(
    BASE_MODEL,
    num_labels=3,
    id2label=LABELS,
    label2id={v: k for k, v in LABELS.items()},
    ignore_mismatched_sizes=True,
)

model = PeftModel.from_pretrained(base_model, ADAPTER_PATH)
try:
    model = model.merge_and_unload()
except Exception as e:
    print("Merge skipped:", e)

model = model.to(DEVICE)
model.eval()
print("Model ready.")


# ================================================================
# SENTIMENT (batch)
# ================================================================

def predict_batch(texts):
    """Her metin için (p_neg, p_neu, p_pos) döndürür."""
    out = []
    for i in range(0, len(texts), BATCH_SIZE):
        chunk = texts[i:i + BATCH_SIZE]
        inputs = tokenizer(
            chunk,
            return_tensors="pt",
            truncation=True,
            max_length=MAX_LENGTH,
            padding=True,
        )
        inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
        with torch.inference_mode():
            probs = torch.softmax(model(**inputs).logits, dim=-1).cpu().tolist()
        out.extend(probs)
    return out


# ================================================================
# RISK KURALLARI (anahtar kelime tabanlı sinyaller)
# ================================================================

RISK_SIGNALS = {
    "Hasar / Claim": [
        "claim", "claims", "payout", "payouts", "loss", "losses",
        "combined ratio", "reserve", "reserves",
    ],
    "Doğal afet": [
        "hurricane", "earthquake", "flood", "flooding", "wildfire", "storm",
        "catastrophe", "cyclone", "typhoon", "tornado", "hailstorm", "drought",
    ],
    "Büyük kaza": [
        "crash", "explosion", "collapse", "derailment", "blast", "accident",
        "fire", "sinking", "spill",
    ],
    "Dava / Fraud": [
        "lawsuit", "sued", "litigation", "fraud", "fine", "fined", "penalty",
        "investigation", "probe", "settlement", "class action", "regulator",
    ],
    "Finansal stres": [
        "downgrade", "downgraded", "profit warning", "solvency", "writedown",
        "write-down", "impairment", "insolvency", "bankruptcy", "capital shortfall",
        "negative outlook", "slump", "plunge",
    ],
    "Siber / Operasyonel": [
        "cyberattack", "cyber attack", "data breach", "breach", "ransomware",
        "outage",
    ],
}

TOPICS = {
    "Motor insurance": ["motor", "auto insurance", "car insurance", "vehicle", "automobile"],
    "Health insurance": ["health insurance", "health", "medical", "hospital"],
    "Catastrophe risk": ["catastrophe", "hurricane", "earthquake", "flood", "wildfire",
                         "storm", "cyclone", "typhoon", "tornado", "hailstorm"],
    "Life insurance": ["life insurance", "life insurer", "annuity", "annuities", "pension", "mortality"],
    "Reinsurance": ["reinsur", "retrocession", "cat bond", "catastrophe bond"],
    "Property": ["property insurance", "homeowner", "home insurance", "real estate", "fire"],
    "Cyber": ["cyber", "ransomware", "data breach"],
}

COMPANIES = [
    "Allianz", "AXA", "Zurich", "Generali", "Aviva", "Munich Re", "Swiss Re",
    "Hannover Re", "Chubb", "AIG", "MetLife", "Prudential", "Allstate",
    "Progressive", "State Farm", "Mapfre", "Ageas", "Lloyd's", "Travelers",
    "Liberty Mutual", "Tokio Marine", "Ping An",
]


def _compile(words):
    return re.compile(r"\b(?:%s)" % "|".join(re.escape(w) for w in words), re.I)


SIGNAL_RE = {cat: _compile(words) for cat, words in RISK_SIGNALS.items()}
TOPIC_RE = {t: _compile(words) for t, words in TOPICS.items()}
COMPANY_RE = {c: re.compile(r"\b%s\b" % re.escape(c), re.I) for c in COMPANIES}


def find_signals(text):
    hits = []
    for cat, rx in SIGNAL_RE.items():
        m = rx.findall(text)
        if m:
            hits.append((cat, sorted({x.lower() for x in m})[:2]))
    return hits


def find_topics(text):
    return [t for t, rx in TOPIC_RE.items() if rx.search(text)]


def find_companies(text):
    return [c for c, rx in COMPANY_RE.items() if rx.search(text)]


def risk_level(score, high=60, medium=35):
    if score >= high:
        return "🔴 HIGH"
    if score >= medium:
        return "🟠 MEDIUM"
    return "🟢 LOW"


# ================================================================
# TEK HABER ANALİZİ
# ================================================================

def score_item(text, probs):
    p_neg, p_neu, p_pos = probs
    sentiment = p_pos - p_neg                       # -1 .. +1
    signals = find_signals(text)
    n_sig = len(signals)
    # Risk indeksi: %65 negatif olasılık + %35 risk sinyali yoğunluğu
    risk = 100 * (0.65 * p_neg + 0.35 * min(n_sig, 3) / 3)
    label = LABELS[max(range(3), key=lambda i: probs[i])]
    reason = "; ".join(f"{c} ({', '.join(k)})" for c, k in signals) or "-"
    return {
        "label": label,
        "sentiment": round(sentiment, 3),
        "signals": n_sig,
        "risk_score": round(risk, 1),
        "risk": risk_level(risk),
        "reason": reason,
    }


def analyze_news(text):
    text = (text or "").strip()
    if not text:
        return "Lütfen bir sigorta haber başlığı girin.", "", ""

    probs = predict_batch([text])[0]
    s = score_item(text, probs)

    companies = find_companies(text) or ["-"]
    topics = find_topics(text) or ["-"]

    result = (
        f"Sentiment: {s['label'].upper()}\n"
        f"Skor: {s['sentiment']:+.2f}\n"
        f"Güven: {max(probs) * 100:.1f}%"
    )
    prob_text = (
        f"Negative: {probs[0] * 100:.1f}%\n"
        f"Neutral: {probs[1] * 100:.1f}%\n"
        f"Positive: {probs[2] * 100:.1f}%"
    )
    risk_text = (
        f"Şirket: {', '.join(companies)}\n"
        f"Risk: {s['risk']} ({s['risk_score']}/100)\n"
        f"Risk sinyali: {s['signals']}\n"
        f"Neden: {s['reason']}\n"
        f"Konu: {', '.join(topics)}"
    )
    return result, prob_text, risk_text


# ================================================================
# TOPLU ANALİZ PIPELINE
# ================================================================

def analyze_items(items):
    """items: [{'headline':str, 'date':datetime|None, 'company':str|None}]"""
    if not items:
        return None, None, None

    texts = [it["headline"] for it in items]
    probs = predict_batch(texts)

    rows = []
    for it, p in zip(items, probs):
        s = score_item(it["headline"], p)
        companies = [it["company"]] if it.get("company") else find_companies(it["headline"])
        rows.append({
            "date": it.get("date"),
            "headline": it["headline"],
            "companies": companies or ["(Sektör)"],
            "topics": find_topics(it["headline"]),
            **s,
        })
    df = pd.DataFrame(rows)

    has_dates = df["date"].notna().sum() >= 4
    if has_dates:
        df["date"] = pd.to_datetime(df["date"], utc=True, errors="coerce")
        mid = df["date"].dropna().sort_values().iloc[len(df["date"].dropna()) // 2]
        df["recent"] = df["date"] >= mid
    else:
        df["recent"] = False

    # ---- Şirket tablosu ----
    ex = df.explode("companies")
    comp_rows = []
    for comp, g in ex.groupby("companies"):
        risk = g["risk_score"].mean()
        trend = "—"
        if has_dates:
            r, e = g[g["recent"]], g[~g["recent"]]
            if len(r) and len(e):
                d = r["sentiment"].mean() - e["sentiment"].mean()
                trend = ("Improving ↑" if d > 0.15
                         else "Deteriorating ↓" if d < -0.15 else "Stable →")
        comp_rows.append({
            "Company": comp,
            "News": len(g),
            "Avg sentiment": round(g["sentiment"].mean(), 2),
            "Risk signals": int(g["signals"].sum()),
            "High-risk news": int((g["risk_score"] >= 60).sum()),
            "Risk score": round(risk, 1),
            "Risk": risk_level(risk, high=55, medium=30),
            "Trend": trend,
        })
    comp_df = pd.DataFrame(comp_rows).sort_values("Risk score", ascending=False)

    # ---- Konu / trend tablosu ----
    topic_rows = []
    n_recent = max(int(df["recent"].sum()), 1)
    n_early = max(int((~df["recent"]).sum()), 1)
    for topic in TOPICS:
        mask = df["topics"].apply(lambda t: topic in t)
        if not mask.any():
            continue
        arrow = "—"
        if has_dates:
            share_r = (mask & df["recent"]).sum() / n_recent
            share_e = (mask & ~df["recent"]).sum() / n_early
            if share_e == 0:
                arrow = "↑↑"
            else:
                ratio = share_r / share_e
                arrow = ("↑↑" if ratio >= 1.5 else "↑" if ratio >= 1.15
                         else "↓" if ratio <= 0.8 else "→")
        topic_rows.append({
            "Topic": topic,
            "News": int(mask.sum()),
            "Avg sentiment": round(df.loc[mask, "sentiment"].mean(), 2),
            "Trend": arrow,
        })
    topic_df = pd.DataFrame(topic_rows).sort_values("News", ascending=False) \
        if topic_rows else pd.DataFrame(columns=["Topic", "News", "Avg sentiment", "Trend"])

    # ---- Haber tablosu ----
    news_df = df.assign(
        companies=df["companies"].apply(", ".join),
        topics=df["topics"].apply(", ".join),
        date=df["date"].apply(lambda d: d.strftime("%Y-%m-%d") if pd.notna(d) else ""),
    )[["date", "companies", "headline", "label", "sentiment", "risk", "risk_score", "reason", "topics"]]
    news_df = news_df.sort_values("risk_score", ascending=False)

    return news_df, comp_df, topic_df


# ================================================================
# TOPLU GİRİŞ (metin / CSV)
# ================================================================

def _pick(cols, names):
    for n in names:
        if n in cols:
            return n
    return None


def batch_analyze(text, file):
    items = []

    path = file if isinstance(file, str) else getattr(file, "name", None)
    if path:
        df = pd.read_csv(path)
        df.columns = [c.strip().lower() for c in df.columns]
        h = _pick(df.columns, ["headline", "title", "text", "news", "haber", "baslik"])
        d = _pick(df.columns, ["date", "published", "pubdate", "tarih"])
        c = _pick(df.columns, ["company", "sirket", "şirket"])
        if h is None:
            raise gr.Error("CSV'de 'headline' (veya title/text/news) kolonu bulunamadı.")
        for _, r in df.iterrows():
            if pd.isna(r[h]) or not str(r[h]).strip():
                continue
            items.append({
                "headline": str(r[h]).strip(),
                "date": pd.to_datetime(r[d], utc=True, errors="coerce") if d else None,
                "company": str(r[c]).strip() if c and pd.notna(r[c]) else None,
            })

    for line in (text or "").splitlines():
        line = line.strip()
        if line:
            items.append({"headline": line, "date": None, "company": None})

    if not items:
        raise gr.Error("Haber girin (her satıra bir başlık) veya CSV yükleyin.")
    return analyze_items(items)


# ================================================================
# CANLI TARAMA (Google News RSS)
# ================================================================

def fetch_news(company, days, limit):
    q = quote(f'"{company}" insurance when:{int(days)}d')
    url = f"https://news.google.com/rss/search?q={q}&hl=en-US&gl=US&ceid=US:en"
    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
    with urllib.request.urlopen(req, timeout=15) as resp:
        root = ET.fromstring(resp.read())

    items = []
    for it in root.iter("item"):
        title = (it.findtext("title") or "").strip()
        title = re.sub(r"\s+-\s+[^-]+$", "", title)    # " - Kaynak" kısmını at
        try:
            date = parsedate_to_datetime(it.findtext("pubDate"))
        except Exception:
            date = None
        if title:
            items.append({"headline": title, "date": date, "company": company})
        if len(items) >= limit:
            break
    return items


def live_scan(company_text, days, limit):
    companies = [c.strip() for c in (company_text or "").split(",") if c.strip()]
    if not companies:
        raise gr.Error("En az bir şirket adı girin.")

    items, errors = [], []
    for c in companies:
        try:
            items += fetch_news(c, days, int(limit))
        except Exception as e:
            errors.append(f"{c}: {e}")

    if not items:
        raise gr.Error("Haber alınamadı. İnternet erişimini kontrol edin. " + " | ".join(errors))
    return analyze_items(items)


# ================================================================
# GRADIO UI
# ================================================================

EXAMPLES = [
    ["Insurance company reports record premium growth"],
    ["Allianz faces major claims increase after severe floods"],
    ["Zurich maintained stable operations in the third quarter."],
    ["AXA downgraded after reserve shortfall and fraud investigation"],
    ["Hurricane losses push reinsurance rates higher"],
]

BATCH_PLACEHOLDER = (
    "Allianz reports record premium growth\n"
    "Generali hit by wildfire claims surge\n"
    "Zurich faces lawsuit over auto claims"
)

NEWS_COLS = ["date", "companies", "headline", "label", "sentiment", "risk", "risk_score", "reason", "topics"]

BRAND = "Goshawk_insurance_bert"

LOGO_SVG = """
<svg width="64" height="64" viewBox="0 0 64 64" xmlns="http://www.w3.org/2000/svg" aria-label="Goshawk logo">
  <defs>
    <linearGradient id="gs" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#1E4F9C"/><stop offset="1" stop-color="#0B1F3A"/>
    </linearGradient>
  </defs>
  <path d="M32 3 L57 12 V32 C57 47 46 57 32 61 C18 57 7 47 7 32 V12 Z"
        fill="url(#gs)" stroke="#F2B134" stroke-width="2.5" stroke-linejoin="round"/>
  <path d="M15 37 C17 25 27 16 40 18 L51 24 L42 27 C45 32 43 38 38 41 L32 51 L28 43 C22 43 17 41 15 37 Z"
        fill="#FFFFFF"/>
  <path d="M51 24 L42 27 L45 33 Z" fill="#F2B134"/>
  <path d="M22 27 L36 23 L35 26 L24 30 Z" fill="#0B1F3A" opacity=".85"/>
  <circle cx="35" cy="29" r="2.2" fill="#F2B134"/>
  <circle cx="35.4" cy="29" r="1" fill="#0B1F3A"/>
</svg>
"""

CSS = """
.gradio-container {max-width: 1100px !important; margin: auto;
  font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;}
#brand-header {display:flex; align-items:center; gap:16px; padding:20px 22px;
  border-radius:18px; color:#fff; margin-bottom:10px;
  background: linear-gradient(135deg, #0B1F3A 0%, #1E4F9C 70%, #2B6CC4 100%);
  box-shadow: 0 6px 20px rgba(11,31,58,.25);}
#brand-header .logo {flex:0 0 auto; filter: drop-shadow(0 2px 6px rgba(0,0,0,.35));}
#brand-header .title {font-size:1.55rem; font-weight:800; letter-spacing:.2px; line-height:1.15;}
#brand-header .title span {color:#F2B134;}
#brand-header .sub {opacity:.85; font-size:.92rem; margin-top:4px;}
#brand-header .badges {margin-top:10px; display:flex; flex-wrap:wrap; gap:6px;}
#brand-header .badge {background:rgba(255,255,255,.14); border:1px solid rgba(255,255,255,.25);
  padding:3px 10px; border-radius:999px; font-size:.74rem; backdrop-filter: blur(4px);}
.legend {display:flex; flex-wrap:wrap; gap:8px; margin:4px 0 12px;}
.legend .chip {padding:4px 12px; border-radius:999px; font-size:.8rem; font-weight:600;
  border:1px solid rgba(128,128,128,.3);}
.chip.high {background:#fde8e8; color:#a31212;}
.chip.med {background:#fff1dc; color:#9a5a00;}
.chip.low {background:#e3f6e8; color:#14702d;}
button.primary {background: linear-gradient(135deg,#1E4F9C,#0B1F3A) !important;
  border:none !important; color:#fff !important; font-weight:700 !important;
  border-radius:12px !important;}
button.primary:hover {filter: brightness(1.15);}
.tab-nav button, button[role="tab"] {font-weight:600 !important;}
textarea, input[type="text"] {border-radius:10px !important;}
#footer {text-align:center; font-size:.82rem; opacity:.75; padding:10px 0 4px;}
@media (max-width: 640px) {
  #brand-header {padding:14px; gap:12px;}
  #brand-header .title {font-size:1.2rem;}
  #brand-header .logo svg {width:48px; height:48px;}
}
"""

HEADER_HTML = f"""
<div id="brand-header">
  <div class="logo">{LOGO_SVG}</div>
  <div>
    <div class="title">Goshawk<span>_</span>insurance<span>_</span>bert</div>
    <div class="sub">Sigorta haberleri için sentiment, risk sinyali ve erken uyarı sistemi</div>
    <div class="badges">
      <span class="badge">DistilBERT + LoRA</span>
      <span class="badge">Macro F1 78.06%</span>
      <span class="badge">Early Warning</span>
      <span class="badge">EN headlines</span>
    </div>
  </div>
</div>
<div class="legend">
  <span class="chip high">🔴 HIGH RISK</span>
  <span class="chip med">🟠 MEDIUM RISK</span>
  <span class="chip low">🟢 LOW RISK</span>
</div>
"""

with gr.Blocks(
    title=BRAND,
    theme=gr.themes.Soft(primary_hue="blue", secondary_hue="amber", neutral_hue="slate"),
    css=CSS,
) as demo:

    gr.HTML(HEADER_HTML)

    # ---------------- TAB 1 ----------------
    with gr.Tab("📰 Tek Haber"):
        with gr.Row():
            with gr.Column():
                text_input = gr.Textbox(label="Insurance News", lines=4,
                                        placeholder="Allianz faces major claims increase...")
                btn = gr.Button("🔍 Analiz Et", variant="primary")
            with gr.Column():
                out_pred = gr.Textbox(label="Sentiment", lines=3)
                out_prob = gr.Textbox(label="Olasılıklar", lines=3)
                out_risk = gr.Textbox(label="Risk Raporu", lines=5)
        outs = [out_pred, out_prob, out_risk]
        btn.click(analyze_news, text_input, outs)
        text_input.submit(analyze_news, text_input, outs)
        gr.Examples(EXAMPLES, inputs=text_input)

    # ---------------- TAB 2 ----------------
    with gr.Tab("📚 Toplu Analiz"):
        gr.Markdown(
            "Her satıra bir başlık yazın **veya** CSV yükleyin "
            "(kolonlar: `headline`, opsiyonel `date`, `company`). "
            "Tarih varsa **trend** hesaplanır."
        )
        b_text = gr.Textbox(label="Başlıklar (satır başına bir haber)", lines=6,
                            placeholder=BATCH_PLACEHOLDER)
        b_file = gr.File(label="CSV (opsiyonel)", file_types=[".csv"])
        b_btn = gr.Button("🚀 Toplu Analiz", variant="primary")
        gr.Markdown("### 🏢 Şirket Risk Tablosu (Underwriting / Rakip İstihbaratı)")
        b_comp = gr.Dataframe(wrap=True)
        gr.Markdown("### 📈 Sektör Konu Trendi")
        b_topic = gr.Dataframe(wrap=True)
        gr.Markdown("### 🗞️ Haber Detayı (riskten düşüğe)")
        b_news = gr.Dataframe(wrap=True)
        b_btn.click(batch_analyze, [b_text, b_file], [b_news, b_comp, b_topic])

    # ---------------- TAB 3 ----------------
    with gr.Tab("🌐 Canlı Tarama"):
        gr.Markdown(
            "Google News RSS'ten şirket haberlerini çekip otomatik analiz eder "
            "(sunucunun internet erişimi gerekir)."
        )
        l_comp = gr.Textbox(label="Şirketler (virgülle)", value="Allianz, AXA, Zurich, Generali")
        with gr.Row():
            l_days = gr.Slider(7, 90, value=30, step=1, label="Son kaç gün")
            l_limit = gr.Slider(5, 50, value=20, step=1, label="Şirket başına haber")
        l_btn = gr.Button("🔎 Haberleri Tara", variant="primary")
        gr.Markdown("### 🏢 Rakip Karşılaştırma")
        l_compdf = gr.Dataframe(wrap=True)
        gr.Markdown("### 📈 Konu Trendi")
        l_topic = gr.Dataframe(wrap=True)
        gr.Markdown("### 🗞️ Haberler")
        l_news = gr.Dataframe(wrap=True)
        l_btn.click(live_scan, [l_comp, l_days, l_limit], [l_news, l_compdf, l_topic])

    gr.HTML('<div id="footer">🦅 <b>Goshawk_insurance_bert</b> · Karar destek aracıdır</div>')
    gr.Markdown(
        "**Not:** Risk skoru = %65 model negatif olasılığı + %35 anahtar-kelime risk sinyali. "
        "Karar destek amaçlıdır; tek başına underwriting kararı için kullanılmamalıdır."
    )


if __name__ == "__main__":
    demo.launch(
        server_name="0.0.0.0",
        server_port=int(os.environ.get("PORT", 7860)),
        show_error=True,
    )