| import os |
| import re |
| import urllib.request |
| import xml.etree.ElementTree as ET |
| from email.utils import parsedate_to_datetime |
| from urllib.parse import quote |
|
|
| import gradio as gr |
| import pandas as pd |
| import torch |
| from peft import PeftModel |
| from transformers import AutoModelForSequenceClassification, AutoTokenizer |
|
|
| |
| |
| |
|
|
| BASE_MODEL = "distilbert-base-uncased" |
| ADAPTER_PATH = "." |
| MAX_LENGTH = 128 |
| BATCH_SIZE = 32 |
|
|
| LABELS = {0: "negative", 1: "neutral", 2: "positive"} |
|
|
| DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") |
|
|
| |
| |
| |
|
|
| print("Device:", DEVICE) |
|
|
| tokenizer = AutoTokenizer.from_pretrained(ADAPTER_PATH, use_fast=True) |
|
|
| base_model = AutoModelForSequenceClassification.from_pretrained( |
| BASE_MODEL, |
| num_labels=3, |
| id2label=LABELS, |
| label2id={v: k for k, v in LABELS.items()}, |
| ignore_mismatched_sizes=True, |
| ) |
|
|
| model = PeftModel.from_pretrained(base_model, ADAPTER_PATH) |
| try: |
| model = model.merge_and_unload() |
| except Exception as e: |
| print("Merge skipped:", e) |
|
|
| model = model.to(DEVICE) |
| model.eval() |
| print("Model ready.") |
|
|
|
|
| |
| |
| |
|
|
| def predict_batch(texts): |
| """Her metin için (p_neg, p_neu, p_pos) döndürür.""" |
| out = [] |
| for i in range(0, len(texts), BATCH_SIZE): |
| chunk = texts[i:i + BATCH_SIZE] |
| inputs = tokenizer( |
| chunk, |
| return_tensors="pt", |
| truncation=True, |
| max_length=MAX_LENGTH, |
| padding=True, |
| ) |
| inputs = {k: v.to(DEVICE) for k, v in inputs.items()} |
| with torch.inference_mode(): |
| probs = torch.softmax(model(**inputs).logits, dim=-1).cpu().tolist() |
| out.extend(probs) |
| return out |
|
|
|
|
| |
| |
| |
|
|
| RISK_SIGNALS = { |
| "Hasar / Claim": [ |
| "claim", "claims", "payout", "payouts", "loss", "losses", |
| "combined ratio", "reserve", "reserves", |
| ], |
| "Doğal afet": [ |
| "hurricane", "earthquake", "flood", "flooding", "wildfire", "storm", |
| "catastrophe", "cyclone", "typhoon", "tornado", "hailstorm", "drought", |
| ], |
| "Büyük kaza": [ |
| "crash", "explosion", "collapse", "derailment", "blast", "accident", |
| "fire", "sinking", "spill", |
| ], |
| "Dava / Fraud": [ |
| "lawsuit", "sued", "litigation", "fraud", "fine", "fined", "penalty", |
| "investigation", "probe", "settlement", "class action", "regulator", |
| ], |
| "Finansal stres": [ |
| "downgrade", "downgraded", "profit warning", "solvency", "writedown", |
| "write-down", "impairment", "insolvency", "bankruptcy", "capital shortfall", |
| "negative outlook", "slump", "plunge", |
| ], |
| "Siber / Operasyonel": [ |
| "cyberattack", "cyber attack", "data breach", "breach", "ransomware", |
| "outage", |
| ], |
| } |
|
|
| TOPICS = { |
| "Motor insurance": ["motor", "auto insurance", "car insurance", "vehicle", "automobile"], |
| "Health insurance": ["health insurance", "health", "medical", "hospital"], |
| "Catastrophe risk": ["catastrophe", "hurricane", "earthquake", "flood", "wildfire", |
| "storm", "cyclone", "typhoon", "tornado", "hailstorm"], |
| "Life insurance": ["life insurance", "life insurer", "annuity", "annuities", "pension", "mortality"], |
| "Reinsurance": ["reinsur", "retrocession", "cat bond", "catastrophe bond"], |
| "Property": ["property insurance", "homeowner", "home insurance", "real estate", "fire"], |
| "Cyber": ["cyber", "ransomware", "data breach"], |
| } |
|
|
| COMPANIES = [ |
| "Allianz", "AXA", "Zurich", "Generali", "Aviva", "Munich Re", "Swiss Re", |
| "Hannover Re", "Chubb", "AIG", "MetLife", "Prudential", "Allstate", |
| "Progressive", "State Farm", "Mapfre", "Ageas", "Lloyd's", "Travelers", |
| "Liberty Mutual", "Tokio Marine", "Ping An", |
| ] |
|
|
|
|
| def _compile(words): |
| return re.compile(r"\b(?:%s)" % "|".join(re.escape(w) for w in words), re.I) |
|
|
|
|
| SIGNAL_RE = {cat: _compile(words) for cat, words in RISK_SIGNALS.items()} |
| TOPIC_RE = {t: _compile(words) for t, words in TOPICS.items()} |
| COMPANY_RE = {c: re.compile(r"\b%s\b" % re.escape(c), re.I) for c in COMPANIES} |
|
|
|
|
| def find_signals(text): |
| hits = [] |
| for cat, rx in SIGNAL_RE.items(): |
| m = rx.findall(text) |
| if m: |
| hits.append((cat, sorted({x.lower() for x in m})[:2])) |
| return hits |
|
|
|
|
| def find_topics(text): |
| return [t for t, rx in TOPIC_RE.items() if rx.search(text)] |
|
|
|
|
| def find_companies(text): |
| return [c for c, rx in COMPANY_RE.items() if rx.search(text)] |
|
|
|
|
| def risk_level(score, high=60, medium=35): |
| if score >= high: |
| return "🔴 HIGH" |
| if score >= medium: |
| return "🟠 MEDIUM" |
| return "🟢 LOW" |
|
|
|
|
| |
| |
| |
|
|
| def score_item(text, probs): |
| p_neg, p_neu, p_pos = probs |
| sentiment = p_pos - p_neg |
| signals = find_signals(text) |
| n_sig = len(signals) |
| |
| risk = 100 * (0.65 * p_neg + 0.35 * min(n_sig, 3) / 3) |
| label = LABELS[max(range(3), key=lambda i: probs[i])] |
| reason = "; ".join(f"{c} ({', '.join(k)})" for c, k in signals) or "-" |
| return { |
| "label": label, |
| "sentiment": round(sentiment, 3), |
| "signals": n_sig, |
| "risk_score": round(risk, 1), |
| "risk": risk_level(risk), |
| "reason": reason, |
| } |
|
|
|
|
| def analyze_news(text): |
| text = (text or "").strip() |
| if not text: |
| return "Lütfen bir sigorta haber başlığı girin.", "", "" |
|
|
| probs = predict_batch([text])[0] |
| s = score_item(text, probs) |
|
|
| companies = find_companies(text) or ["-"] |
| topics = find_topics(text) or ["-"] |
|
|
| result = ( |
| f"Sentiment: {s['label'].upper()}\n" |
| f"Skor: {s['sentiment']:+.2f}\n" |
| f"Güven: {max(probs) * 100:.1f}%" |
| ) |
| prob_text = ( |
| f"Negative: {probs[0] * 100:.1f}%\n" |
| f"Neutral: {probs[1] * 100:.1f}%\n" |
| f"Positive: {probs[2] * 100:.1f}%" |
| ) |
| risk_text = ( |
| f"Şirket: {', '.join(companies)}\n" |
| f"Risk: {s['risk']} ({s['risk_score']}/100)\n" |
| f"Risk sinyali: {s['signals']}\n" |
| f"Neden: {s['reason']}\n" |
| f"Konu: {', '.join(topics)}" |
| ) |
| return result, prob_text, risk_text |
|
|
|
|
| |
| |
| |
|
|
| def analyze_items(items): |
| """items: [{'headline':str, 'date':datetime|None, 'company':str|None}]""" |
| if not items: |
| return None, None, None |
|
|
| texts = [it["headline"] for it in items] |
| probs = predict_batch(texts) |
|
|
| rows = [] |
| for it, p in zip(items, probs): |
| s = score_item(it["headline"], p) |
| companies = [it["company"]] if it.get("company") else find_companies(it["headline"]) |
| rows.append({ |
| "date": it.get("date"), |
| "headline": it["headline"], |
| "companies": companies or ["(Sektör)"], |
| "topics": find_topics(it["headline"]), |
| **s, |
| }) |
| df = pd.DataFrame(rows) |
|
|
| has_dates = df["date"].notna().sum() >= 4 |
| if has_dates: |
| df["date"] = pd.to_datetime(df["date"], utc=True, errors="coerce") |
| mid = df["date"].dropna().sort_values().iloc[len(df["date"].dropna()) // 2] |
| df["recent"] = df["date"] >= mid |
| else: |
| df["recent"] = False |
|
|
| |
| ex = df.explode("companies") |
| comp_rows = [] |
| for comp, g in ex.groupby("companies"): |
| risk = g["risk_score"].mean() |
| trend = "—" |
| if has_dates: |
| r, e = g[g["recent"]], g[~g["recent"]] |
| if len(r) and len(e): |
| d = r["sentiment"].mean() - e["sentiment"].mean() |
| trend = ("Improving ↑" if d > 0.15 |
| else "Deteriorating ↓" if d < -0.15 else "Stable →") |
| comp_rows.append({ |
| "Company": comp, |
| "News": len(g), |
| "Avg sentiment": round(g["sentiment"].mean(), 2), |
| "Risk signals": int(g["signals"].sum()), |
| "High-risk news": int((g["risk_score"] >= 60).sum()), |
| "Risk score": round(risk, 1), |
| "Risk": risk_level(risk, high=55, medium=30), |
| "Trend": trend, |
| }) |
| comp_df = pd.DataFrame(comp_rows).sort_values("Risk score", ascending=False) |
|
|
| |
| topic_rows = [] |
| n_recent = max(int(df["recent"].sum()), 1) |
| n_early = max(int((~df["recent"]).sum()), 1) |
| for topic in TOPICS: |
| mask = df["topics"].apply(lambda t: topic in t) |
| if not mask.any(): |
| continue |
| arrow = "—" |
| if has_dates: |
| share_r = (mask & df["recent"]).sum() / n_recent |
| share_e = (mask & ~df["recent"]).sum() / n_early |
| if share_e == 0: |
| arrow = "↑↑" |
| else: |
| ratio = share_r / share_e |
| arrow = ("↑↑" if ratio >= 1.5 else "↑" if ratio >= 1.15 |
| else "↓" if ratio <= 0.8 else "→") |
| topic_rows.append({ |
| "Topic": topic, |
| "News": int(mask.sum()), |
| "Avg sentiment": round(df.loc[mask, "sentiment"].mean(), 2), |
| "Trend": arrow, |
| }) |
| topic_df = pd.DataFrame(topic_rows).sort_values("News", ascending=False) \ |
| if topic_rows else pd.DataFrame(columns=["Topic", "News", "Avg sentiment", "Trend"]) |
|
|
| |
| news_df = df.assign( |
| companies=df["companies"].apply(", ".join), |
| topics=df["topics"].apply(", ".join), |
| date=df["date"].apply(lambda d: d.strftime("%Y-%m-%d") if pd.notna(d) else ""), |
| )[["date", "companies", "headline", "label", "sentiment", "risk", "risk_score", "reason", "topics"]] |
| news_df = news_df.sort_values("risk_score", ascending=False) |
|
|
| return news_df, comp_df, topic_df |
|
|
|
|
| |
| |
| |
|
|
| def _pick(cols, names): |
| for n in names: |
| if n in cols: |
| return n |
| return None |
|
|
|
|
| def batch_analyze(text, file): |
| items = [] |
|
|
| path = file if isinstance(file, str) else getattr(file, "name", None) |
| if path: |
| df = pd.read_csv(path) |
| df.columns = [c.strip().lower() for c in df.columns] |
| h = _pick(df.columns, ["headline", "title", "text", "news", "haber", "baslik"]) |
| d = _pick(df.columns, ["date", "published", "pubdate", "tarih"]) |
| c = _pick(df.columns, ["company", "sirket", "şirket"]) |
| if h is None: |
| raise gr.Error("CSV'de 'headline' (veya title/text/news) kolonu bulunamadı.") |
| for _, r in df.iterrows(): |
| if pd.isna(r[h]) or not str(r[h]).strip(): |
| continue |
| items.append({ |
| "headline": str(r[h]).strip(), |
| "date": pd.to_datetime(r[d], utc=True, errors="coerce") if d else None, |
| "company": str(r[c]).strip() if c and pd.notna(r[c]) else None, |
| }) |
|
|
| for line in (text or "").splitlines(): |
| line = line.strip() |
| if line: |
| items.append({"headline": line, "date": None, "company": None}) |
|
|
| if not items: |
| raise gr.Error("Haber girin (her satıra bir başlık) veya CSV yükleyin.") |
| return analyze_items(items) |
|
|
|
|
| |
| |
| |
|
|
| def fetch_news(company, days, limit): |
| q = quote(f'"{company}" insurance when:{int(days)}d') |
| url = f"https://news.google.com/rss/search?q={q}&hl=en-US&gl=US&ceid=US:en" |
| req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) |
| with urllib.request.urlopen(req, timeout=15) as resp: |
| root = ET.fromstring(resp.read()) |
|
|
| items = [] |
| for it in root.iter("item"): |
| title = (it.findtext("title") or "").strip() |
| title = re.sub(r"\s+-\s+[^-]+$", "", title) |
| try: |
| date = parsedate_to_datetime(it.findtext("pubDate")) |
| except Exception: |
| date = None |
| if title: |
| items.append({"headline": title, "date": date, "company": company}) |
| if len(items) >= limit: |
| break |
| return items |
|
|
|
|
| def live_scan(company_text, days, limit): |
| companies = [c.strip() for c in (company_text or "").split(",") if c.strip()] |
| if not companies: |
| raise gr.Error("En az bir şirket adı girin.") |
|
|
| items, errors = [], [] |
| for c in companies: |
| try: |
| items += fetch_news(c, days, int(limit)) |
| except Exception as e: |
| errors.append(f"{c}: {e}") |
|
|
| if not items: |
| raise gr.Error("Haber alınamadı. İnternet erişimini kontrol edin. " + " | ".join(errors)) |
| return analyze_items(items) |
|
|
|
|
| |
| |
| |
|
|
| EXAMPLES = [ |
| ["Insurance company reports record premium growth"], |
| ["Allianz faces major claims increase after severe floods"], |
| ["Zurich maintained stable operations in the third quarter."], |
| ["AXA downgraded after reserve shortfall and fraud investigation"], |
| ["Hurricane losses push reinsurance rates higher"], |
| ] |
|
|
| BATCH_PLACEHOLDER = ( |
| "Allianz reports record premium growth\n" |
| "Generali hit by wildfire claims surge\n" |
| "Zurich faces lawsuit over auto claims" |
| ) |
|
|
| NEWS_COLS = ["date", "companies", "headline", "label", "sentiment", "risk", "risk_score", "reason", "topics"] |
|
|
| BRAND = "Goshawk_insurance_bert" |
|
|
| LOGO_SVG = """ |
| <svg width="64" height="64" viewBox="0 0 64 64" xmlns="http://www.w3.org/2000/svg" aria-label="Goshawk logo"> |
| <defs> |
| <linearGradient id="gs" x1="0" y1="0" x2="0" y2="1"> |
| <stop offset="0" stop-color="#1E4F9C"/><stop offset="1" stop-color="#0B1F3A"/> |
| </linearGradient> |
| </defs> |
| <path d="M32 3 L57 12 V32 C57 47 46 57 32 61 C18 57 7 47 7 32 V12 Z" |
| fill="url(#gs)" stroke="#F2B134" stroke-width="2.5" stroke-linejoin="round"/> |
| <path d="M15 37 C17 25 27 16 40 18 L51 24 L42 27 C45 32 43 38 38 41 L32 51 L28 43 C22 43 17 41 15 37 Z" |
| fill="#FFFFFF"/> |
| <path d="M51 24 L42 27 L45 33 Z" fill="#F2B134"/> |
| <path d="M22 27 L36 23 L35 26 L24 30 Z" fill="#0B1F3A" opacity=".85"/> |
| <circle cx="35" cy="29" r="2.2" fill="#F2B134"/> |
| <circle cx="35.4" cy="29" r="1" fill="#0B1F3A"/> |
| </svg> |
| """ |
|
|
| CSS = """ |
| .gradio-container {max-width: 1100px !important; margin: auto; |
| font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;} |
| #brand-header {display:flex; align-items:center; gap:16px; padding:20px 22px; |
| border-radius:18px; color:#fff; margin-bottom:10px; |
| background: linear-gradient(135deg, #0B1F3A 0%, #1E4F9C 70%, #2B6CC4 100%); |
| box-shadow: 0 6px 20px rgba(11,31,58,.25);} |
| #brand-header .logo {flex:0 0 auto; filter: drop-shadow(0 2px 6px rgba(0,0,0,.35));} |
| #brand-header .title {font-size:1.55rem; font-weight:800; letter-spacing:.2px; line-height:1.15;} |
| #brand-header .title span {color:#F2B134;} |
| #brand-header .sub {opacity:.85; font-size:.92rem; margin-top:4px;} |
| #brand-header .badges {margin-top:10px; display:flex; flex-wrap:wrap; gap:6px;} |
| #brand-header .badge {background:rgba(255,255,255,.14); border:1px solid rgba(255,255,255,.25); |
| padding:3px 10px; border-radius:999px; font-size:.74rem; backdrop-filter: blur(4px);} |
| .legend {display:flex; flex-wrap:wrap; gap:8px; margin:4px 0 12px;} |
| .legend .chip {padding:4px 12px; border-radius:999px; font-size:.8rem; font-weight:600; |
| border:1px solid rgba(128,128,128,.3);} |
| .chip.high {background:#fde8e8; color:#a31212;} |
| .chip.med {background:#fff1dc; color:#9a5a00;} |
| .chip.low {background:#e3f6e8; color:#14702d;} |
| button.primary {background: linear-gradient(135deg,#1E4F9C,#0B1F3A) !important; |
| border:none !important; color:#fff !important; font-weight:700 !important; |
| border-radius:12px !important;} |
| button.primary:hover {filter: brightness(1.15);} |
| .tab-nav button, button[role="tab"] {font-weight:600 !important;} |
| textarea, input[type="text"] {border-radius:10px !important;} |
| #footer {text-align:center; font-size:.82rem; opacity:.75; padding:10px 0 4px;} |
| @media (max-width: 640px) { |
| #brand-header {padding:14px; gap:12px;} |
| #brand-header .title {font-size:1.2rem;} |
| #brand-header .logo svg {width:48px; height:48px;} |
| } |
| """ |
|
|
| HEADER_HTML = f""" |
| <div id="brand-header"> |
| <div class="logo">{LOGO_SVG}</div> |
| <div> |
| <div class="title">Goshawk<span>_</span>insurance<span>_</span>bert</div> |
| <div class="sub">Sigorta haberleri için sentiment, risk sinyali ve erken uyarı sistemi</div> |
| <div class="badges"> |
| <span class="badge">DistilBERT + LoRA</span> |
| <span class="badge">Macro F1 78.06%</span> |
| <span class="badge">Early Warning</span> |
| <span class="badge">EN headlines</span> |
| </div> |
| </div> |
| </div> |
| <div class="legend"> |
| <span class="chip high">🔴 HIGH RISK</span> |
| <span class="chip med">🟠 MEDIUM RISK</span> |
| <span class="chip low">🟢 LOW RISK</span> |
| </div> |
| """ |
|
|
| with gr.Blocks( |
| title=BRAND, |
| theme=gr.themes.Soft(primary_hue="blue", secondary_hue="amber", neutral_hue="slate"), |
| css=CSS, |
| ) as demo: |
|
|
| gr.HTML(HEADER_HTML) |
|
|
| |
| with gr.Tab("📰 Tek Haber"): |
| with gr.Row(): |
| with gr.Column(): |
| text_input = gr.Textbox(label="Insurance News", lines=4, |
| placeholder="Allianz faces major claims increase...") |
| btn = gr.Button("🔍 Analiz Et", variant="primary") |
| with gr.Column(): |
| out_pred = gr.Textbox(label="Sentiment", lines=3) |
| out_prob = gr.Textbox(label="Olasılıklar", lines=3) |
| out_risk = gr.Textbox(label="Risk Raporu", lines=5) |
| outs = [out_pred, out_prob, out_risk] |
| btn.click(analyze_news, text_input, outs) |
| text_input.submit(analyze_news, text_input, outs) |
| gr.Examples(EXAMPLES, inputs=text_input) |
|
|
| |
| with gr.Tab("📚 Toplu Analiz"): |
| gr.Markdown( |
| "Her satıra bir başlık yazın **veya** CSV yükleyin " |
| "(kolonlar: `headline`, opsiyonel `date`, `company`). " |
| "Tarih varsa **trend** hesaplanır." |
| ) |
| b_text = gr.Textbox(label="Başlıklar (satır başına bir haber)", lines=6, |
| placeholder=BATCH_PLACEHOLDER) |
| b_file = gr.File(label="CSV (opsiyonel)", file_types=[".csv"]) |
| b_btn = gr.Button("🚀 Toplu Analiz", variant="primary") |
| gr.Markdown("### 🏢 Şirket Risk Tablosu (Underwriting / Rakip İstihbaratı)") |
| b_comp = gr.Dataframe(wrap=True) |
| gr.Markdown("### 📈 Sektör Konu Trendi") |
| b_topic = gr.Dataframe(wrap=True) |
| gr.Markdown("### 🗞️ Haber Detayı (riskten düşüğe)") |
| b_news = gr.Dataframe(wrap=True) |
| b_btn.click(batch_analyze, [b_text, b_file], [b_news, b_comp, b_topic]) |
|
|
| |
| with gr.Tab("🌐 Canlı Tarama"): |
| gr.Markdown( |
| "Google News RSS'ten şirket haberlerini çekip otomatik analiz eder " |
| "(sunucunun internet erişimi gerekir)." |
| ) |
| l_comp = gr.Textbox(label="Şirketler (virgülle)", value="Allianz, AXA, Zurich, Generali") |
| with gr.Row(): |
| l_days = gr.Slider(7, 90, value=30, step=1, label="Son kaç gün") |
| l_limit = gr.Slider(5, 50, value=20, step=1, label="Şirket başına haber") |
| l_btn = gr.Button("🔎 Haberleri Tara", variant="primary") |
| gr.Markdown("### 🏢 Rakip Karşılaştırma") |
| l_compdf = gr.Dataframe(wrap=True) |
| gr.Markdown("### 📈 Konu Trendi") |
| l_topic = gr.Dataframe(wrap=True) |
| gr.Markdown("### 🗞️ Haberler") |
| l_news = gr.Dataframe(wrap=True) |
| l_btn.click(live_scan, [l_comp, l_days, l_limit], [l_news, l_compdf, l_topic]) |
|
|
| gr.HTML('<div id="footer">🦅 <b>Goshawk_insurance_bert</b> · Karar destek aracıdır</div>') |
| gr.Markdown( |
| "**Not:** Risk skoru = %65 model negatif olasılığı + %35 anahtar-kelime risk sinyali. " |
| "Karar destek amaçlıdır; tek başına underwriting kararı için kullanılmamalıdır." |
| ) |
|
|
|
|
| if __name__ == "__main__": |
| demo.launch( |
| server_name="0.0.0.0", |
| server_port=int(os.environ.get("PORT", 7860)), |
| show_error=True, |
| ) |
|
|
