| """Lapse & Surrender Study 2023–25 · scelo — Gradio Space. |
| |
| Interactive companion to the Hugging Face dataset |
| ``intelligentactuaries/synthetic-lapse-surrender-study-2023-2025``: 10,000 |
| synthetic policyholders answering the Society of Actuaries (SOA) Research |
| Institute and LIMRA 2023–2025 Term and Whole Life Lapse and Surrender |
| Experience Study data request, expanded into VM-51 policy-year records. |
| |
| Every figure is computed live with the ``scelo`` Python library (``study.py``): |
| experience rates and credibility, the lapse GLM and its lift, and the BasicTerm |
| projection, Solvency II life SCR and IFRS 17 CSM of the in-force term book on a |
| basis set with the controls. The charts are scelo's own chart family |
| (``sc.plot_*``). |
| |
| It runs as a Gradio Space on the Hub, and the organisation's Space |
| ``intelligentactuaries/lapse-surrender-study-scelo`` embeds it; ``python app.py`` |
| runs it locally. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import base64 |
| import tempfile |
| from dataclasses import asdict |
| from pathlib import Path |
|
|
| import gradio as gr |
| import numpy as np |
| import pandas as pd |
| import scelo as sc |
| from scelo import BasicTermAssumptions |
|
|
| import study |
| from study import ( |
| DATASET, |
| DATASET_URL, |
| DECREMENTS, |
| EXPERIENCE_BY, |
| FILES, |
| GLM_COVARIATES, |
| MARKETS, |
| PRODUCTS, |
| PUBLISHED_COVARIATES, |
| ROWS, |
| SCELO_DOCS_URL, |
| SOA_2015_22_URL, |
| SOA_REQUEST_URL, |
| YEARS, |
| ) |
|
|
| IA_ACCENT = "#A04A1F" |
|
|
| CSS = f""" |
| .gradio-container {{ max-width: 1320px !important; margin: 0 auto !important; }} |
| .ia-head h1 {{ font-size: 2.1rem; font-weight: 700; letter-spacing: -0.02em; margin: 0.2rem 0 0.35rem; }} |
| .ia-tagline {{ |
| display: inline-flex; align-items: center; gap: 0.5rem; padding: 0.25rem 0.7rem; |
| border: 1px solid rgba(160, 74, 31, 0.28); border-radius: 999px; background: rgba(160, 74, 31, 0.06); |
| font-size: 0.82rem; color: {IA_ACCENT}; font-weight: 600; |
| }} |
| .ia-tagline .dot {{ width: 6px; height: 6px; border-radius: 50%; background: {IA_ACCENT}; display: inline-block; }} |
| .ia-sub {{ font-size: 0.9rem; opacity: 0.8; margin-top: 0.5rem; }} |
| .ia-links {{ float: right; font-size: 0.9rem; }} |
| .ia-links a {{ margin-left: 1rem; text-decoration: none; opacity: 0.85; }} |
| .ia-synthetic {{ |
| border-left: 3px solid {IA_ACCENT}; background: rgba(160, 74, 31, 0.06); |
| padding: 0.6rem 0.9rem; border-radius: 0 8px 8px 0; font-size: 0.92rem; |
| }} |
| .ia-cards {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 0.75rem; }} |
| .ia-card {{ |
| background: rgba(160, 74, 31, 0.06); border: 1px solid rgba(160, 74, 31, 0.20); |
| border-radius: 10px; padding: 0.65rem 0.85rem; |
| }} |
| .ia-card-label {{ font-size: 0.7rem; letter-spacing: 0.08em; text-transform: uppercase; font-weight: 600; opacity: 0.8; }} |
| .ia-card-value {{ font-size: 1.45rem; font-weight: 700; margin-top: 0.15rem; }} |
| .ia-card-sub {{ font-size: 0.78rem; opacity: 0.75; margin-top: 0.1rem; }} |
| .ia-card-sub.up {{ color: #1f8f5c; opacity: 1; }} |
| .ia-card-sub.down {{ color: #b43939; opacity: 1; }} |
| .ia-ack {{ |
| border: 1px solid rgba(160, 74, 31, 0.22); border-radius: 10px; |
| padding: 0.9rem 1.1rem; font-size: 0.92rem; line-height: 1.5; |
| }} |
| .ia-footer {{ font-size: 0.8rem; opacity: 0.72; line-height: 1.5; }} |
| .ia-note {{ font-size: 0.85rem; opacity: 0.85; }} |
| .ia-fig {{ margin: 0; background: #ffffff; border: 1px solid rgba(58, 51, 45, 0.12); border-radius: 10px; padding: 0.5rem; }} |
| .ia-fig img {{ width: 100%; height: auto; display: block; }} |
| .ia-cap {{ font-size: 0.8rem; opacity: 0.75; margin-top: 0.35rem; }} |
| """ |
|
|
| |
| SIENNA = gr.themes.Color( |
| c50="#fbf3ee", c100="#f6e4d8", c200="#ecc6ae", c300="#dfa07c", c400="#cf7a4f", c500="#b85f31", |
| c600="#a04a1f", c700="#843c19", c800="#6b3216", c900="#582b14", c950="#301508", name="sienna", |
| ) |
| THEME = gr.themes.Default( |
| primary_hue=SIENNA, |
| neutral_hue="stone", |
| font=[gr.themes.GoogleFont("Source Sans 3"), "ui-sans-serif", "system-ui", "sans-serif"], |
| font_mono=[gr.themes.GoogleFont("IBM Plex Mono"), "ui-monospace", "monospace"], |
| ).set( |
| link_text_color="*primary_600", |
| link_text_color_hover="*primary_700", |
| link_text_color_visited="*primary_600", |
| link_text_color_active="*primary_700", |
| ) |
|
|
|
|
| |
| |
| |
|
|
| def cards(items) -> str: |
| """Metric cards: (label, value, sub, tone) with tone in {"", "up", "down"}.""" |
| cells = [] |
| for label, val, sub, *tone in items: |
| tone_cls = f" {tone[0]}" if tone and tone[0] else "" |
| sub_html = f'<div class="ia-card-sub{tone_cls}">{sub}</div>' if sub else "" |
| cells.append(f'<div class="ia-card"><div class="ia-card-label">{label}</div>' |
| f'<div class="ia-card-value">{val}</div>{sub_html}</div>') |
| return f'<div class="ia-cards">{"".join(cells)}</div>' |
|
|
|
|
| def chart(fig, caption: str = "") -> str: |
| """A scelo figure as an inline PNG (``sc.save_figure``), sized to its column.""" |
| with tempfile.TemporaryDirectory() as tmp: |
| path = sc.save_figure(fig, Path(tmp) / "figure.png", dpi=150) |
| data = base64.b64encode(Path(path).read_bytes()).decode("ascii") |
| cap = f'<figcaption class="ia-cap">{caption}</figcaption>' if caption else "" |
| return f'<figure class="ia-fig"><img alt="{caption}" src="data:image/png;base64,{data}"/>{cap}</figure>' |
|
|
|
|
| def notes_md(notes, basis: str | None = None, extra: list[str] | None = None) -> str: |
| lines = ([f"**Basis:** {basis}"] if basis else []) + [f"· {n}" for n in (notes or []) + (extra or [])] |
| return " \n".join(lines) |
|
|
|
|
| def text_table(df: pd.DataFrame, formats: dict[str, str]) -> pd.DataFrame: |
| """Format numeric columns as text, leaving missing cells empty and avoiding "-0".""" |
| def fmt(v, f_): |
| if v is None or (isinstance(v, float) and np.isnan(v)): |
| return "" |
| text = f_.format(v) |
| return text[1:] if text.startswith("-") and not text.strip("-0.,%") else text |
|
|
| out = pd.DataFrame(df).copy() |
| for c_, f_ in formats.items(): |
| if c_ in out.columns: |
| out[c_] = out[c_].map(lambda v, f_=f_: fmt(v, f_)) |
| return out |
|
|
|
|
| def as_text(df: pd.DataFrame) -> pd.DataFrame: |
| """Render mixed-type columns (profile summaries, cleaning plans) as text.""" |
| out = pd.DataFrame(df).copy() |
| for c_ in out.columns: |
| if not pd.api.types.is_numeric_dtype(out[c_]) and not pd.api.types.is_bool_dtype(out[c_]): |
| out[c_] = out[c_].map(lambda v: "" if v is None or (isinstance(v, float) and np.isnan(v)) else str(v)) |
| return out |
|
|
|
|
| def fmt_int(x: float) -> str: |
| return f"{x:,.0f}" |
|
|
|
|
| def delta(x: float, x0: float, good_up: bool = True) -> tuple[str, str]: |
| d = x - x0 |
| if abs(d) < 0.5: |
| return "published basis", "" |
| tone = "up" if (d > 0) == good_up else "down" |
| return f"{d:+,.0f} vs published basis", tone |
|
|
|
|
| |
| |
| |
|
|
| def overview(): |
| vm = study.experience() |
| status = vm["status"].value_counts() |
| html = cards([ |
| ("Policies", fmt_int(vm["agent_id"].nunique()), "one per synthetic policyholder"), |
| ("VM-51 records", fmt_int(len(vm)), "policy × observation year"), |
| ("Exposure", fmt_int(vm["exposure_years"].sum()), "policy-years, 2023–25"), |
| ("Lapses", fmt_int(status.get("lapsed", 0)), None), |
| ("Surrenders", fmt_int(status.get("surrendered", 0)), None), |
| ("Deaths", fmt_int(status.get("died", 0)), "imposed overlay"), |
| ]) |
| fig = chart(sc.plot_rates(vm.assign(lapsed=(vm["status"] == "lapsed").astype(int)), "country", "lapsed", |
| exposure="exposure_years", title="Lapses per exposure-year by market, 2023–25"), |
| "sc.plot_rates · all products · VM-51 policy-years · 95 % Poisson interval") |
| mix = pd.DataFrame(sc.tab(vm, "status", "observation_year")).reset_index() |
| mix.columns = [str(c) for c in mix.columns] |
| products = pd.DataFrame(sc.tab(vm.drop_duplicates("agent_id"), "product")).reset_index() |
| return html, fig, mix, products |
|
|
|
|
| def experience_view(dec_label, by_label, markets, products, years, cred): |
| by = EXPERIENCE_BY[by_label] |
| statuses = DECREMENTS[dec_label] |
| vm = study.experience() |
| df = vm[vm["country"].isin(markets or []) & vm["product"].isin(products or []) |
| & vm["observation_year"].isin(years or [])] |
| if df.empty: |
| msg = cards([("Records in view", "0", "widen the filters")]) |
| return msg, "", pd.DataFrame(), "No policy-years match the filters.", gr.Dataframe(visible=False), "" |
|
|
| events = int(df["status"].isin(statuses).sum()) |
| expo = float(df["exposure_years"].sum()) |
| html = cards([ |
| ("Records in view", fmt_int(len(df)), "VM-51 policy-years after filters"), |
| ("Exposure", fmt_int(expo), "policy-years"), |
| ("Events", fmt_int(events), dec_label), |
| ("Rate", f"{events / expo:.4f}" if expo else "–", "per exposure-year"), |
| ]) |
| fig = chart(sc.plot_rates(df.assign(event=df["status"].isin(statuses).astype(int)), by, "event", |
| exposure="exposure_years", title=f"{dec_label} per exposure-year by {by_label.lower()}"), |
| "sc.plot_rates · bars sorted by rate · 95 % Poisson interval · the vertical line is the rate over " |
| "everything in view") |
|
|
| table, notes = study.rates(df, by, statuses) |
| table = text_table(table, {"exposure": "{:,.1f}", "events": "{:,.0f}", "rate": "{:.4f}", |
| "lo_95": "{:.4f}", "hi_95": "{:.4f}"}) |
| table = table.rename(columns={by: by_label, "exposure": "Exposure (policy-years)", "events": "Events", |
| "rate": "Rate", "lo_95": "95 % lower", "hi_95": "95 % upper"}) |
| extra = ["Intervals are exact (Garwood) Poisson intervals on the event count."] |
| if by == "mbti": |
| extra.append("The MBTI spread is a **designed** property of the synthetic population, " |
| "not an empirical finding about personality and lapse.") |
| if statuses == ("died",): |
| extra.append("Deaths come from an imposed Gompertz overlay, q = 0.00004·e^(0.088·age) with sex and " |
| "market factors: they carry no information.") |
| md = notes_md(notes, "sc.freq_sev · central exposure from the VM-51 policy-years", extra) |
|
|
| if cred: |
| ct, basis, cnotes = study.credibility(df, by, statuses) |
| ct = text_table(ct, {"weight": "{:,.1f}", "mean": "{:.4f}", "Z": "{:.4f}", "credibility_premium": "{:.4f}"}) |
| return html, fig, table, md, gr.Dataframe(value=ct, visible=True), notes_md(cnotes, basis) |
| return html, fig, table, md, gr.Dataframe(visible=False), "" |
|
|
|
|
| def glm_view(population, cov_labels, base): |
| covariates = tuple(GLM_COVARIATES[c] for c in (cov_labels or [])) |
| if not covariates: |
| empty = pd.DataFrame() |
| return (cards([("Covariates", "0", "pick at least one")]), "Pick at least one covariate.", |
| "", empty, "", empty, empty, "") |
| model, lift, gini = study.fit_glm(population, covariates, base) |
| base_rate = float(np.exp(model.params["Intercept"])) |
| html = cards([ |
| ("Policies", fmt_int(model.n), "term book" if population == "term" else "all products"), |
| ("AIC", f"{model.aic:,.1f}", None), |
| ("Deviance", f"{model.deviance:,.1f}", f"null {model.null_deviance:,.1f}"), |
| ("Deviance explained", f"{1 - model.deviance / model.null_deviance:.1%}", None), |
| ("Gini", f"{gini:.3f}", "lift of the fitted rate"), |
| ]) |
| lines = [] |
| if population == "term" and base == "first" and sorted(covariates) == sorted(PUBLISHED_COVARIATES): |
| f = study.facts() |
| lines.append( |
| f"✅ **This is the published model.** AIC {model.aic:,.1f} and base rate {base_rate:.4f} match the " |
| f"Scelo IDE run (AIC {f['glm']['ide']['aic']:,.1f}, base {f['glm']['ide']['baseline']:.4f}) and " |
| f"scelo for R (AIC {f['glm']['r']['aic']:,.1f})." |
| ) |
| lines.append(f"`{model.formula}` · {model.family} / {model.link} · offset log(exposure) · engine " |
| f"{model.engine} · base rate exp(β₀) = {base_rate:.4f} lapses per exposure-year") |
| forest = chart(sc.plot_relativities(model), |
| "sc.plot_relativities · exp(β) with 95 % intervals · hollow marker = reference level") |
| rel = text_table(pd.DataFrame(model.relativities()), {"relativity": "{:.3f}", "estimate": "{:.4f}"}) |
| lift_fig = chart(sc.plot_lines(lift.assign(actual=lift["actual"] * 100, predicted=lift["predicted"] * 100), |
| "band", ["actual", "predicted"], |
| title="Lapses per 100 exposure-years, by decile of the fitted rate", |
| subtitle=f"equal-exposure deciles · Gini {gini:.3f}"), |
| "sc.lift · sc.plot_lines · rising = the model ranks lapse; actual ≈ predicted = calibrated") |
| lift_t = text_table(lift, {"exposure": "{:,.1f}", "actual": "{:.4f}", "predicted": "{:.4f}", |
| "lift": "{:.3f}", "a/e": "{:.3f}"}) |
| coef = text_table(pd.DataFrame(model.coef), {"estimate": "{:.4f}", "std_err": "{:.4f}", "z": "{:.2f}", |
| "p_value": "{:.2e}", "exp": "{:.4f}"}) |
| caveat = "" |
| if "mbti" in covariates: |
| caveat = ("ℹ️ The MBTI relativities span about two orders of magnitude, far wider than any real book. " |
| "That spread was **designed into** the synthetic population: do not read it as evidence about " |
| "personality and lapse, and do not use personality (or `community`, `sex`) as a rating factor.") |
| return html, "\n\n".join(lines), forest, rel, lift_fig, lift_t, coef, caveat |
|
|
|
|
| def value_view(lapse_mode, lapse_custom, disc, mort, acq, maint, prem, ra): |
| crude = study.crude_lapse_rate() |
| lapse = {"illustrative": 0.05, "experience": crude, "custom": lapse_custom}[lapse_mode] |
| basis_label = {"illustrative": "scelo illustrative 5.0 % p.a.", |
| "experience": f"term-book experience 2023–25, {crude:.2%} p.a.", |
| "custom": f"custom {lapse_custom:.2%} p.a."}[lapse_mode] |
| a = BasicTermAssumptions(lapse_rate=lapse, disc_rate=disc, mort_mult=mort, |
| expense_acq_pp=acq, expense_maint_pp_mth=maint, premium_mult=prem) |
| res = study.value(tuple(sorted(asdict(a).items())), ra) |
| ref = study.value(tuple(sorted(asdict(BasicTermAssumptions()).items())), 0.05) |
| pv, pv0 = res["bt"].attrs["pv"], ref["bt"].attrs["pv"] |
| scr, scr0 = res["scr"].attrs["scr"], ref["scr"].attrs["scr"] |
| csm0, csm00 = res["csm"].attrs["csm0"], ref["csm"].attrs["csm0"] |
| loss = res["csm"].attrs["loss_component"] |
| outgo, outgo0 = pv["claims"] + pv["expenses"], pv0["claims"] + pv0["expenses"] |
| items = [ |
| ("PV premiums", fmt_int(pv["premiums"]), *delta(pv["premiums"], pv0["premiums"])), |
| ("PV outgo", fmt_int(outgo), *delta(outgo, outgo0, good_up=False)), |
| ("PV net cash flow", fmt_int(pv["net"]), *delta(pv["net"], pv0["net"])), |
| ("Life SCR", fmt_int(scr), *delta(scr, scr0, good_up=False)), |
| ("Loss component", fmt_int(loss), "IFRS 17: onerous on this basis", "down") if loss > 0 |
| else ("IFRS 17 CSM₀", fmt_int(csm0), *delta(csm0, csm00)), |
| ] |
| caption = (f"Lapse basis: **{basis_label}** · USD · the published basis is scelo's illustrative one " |
| "(5 % lapse, 3 % discount, RA 5 %), on which the Python and R libraries agree to 6.6 × 10⁻¹⁰.") |
| proj = chart(sc.plot_projection(res["bt"], title="BasicTerm projection of the in-force term book"), |
| "sc.basicterm · sc.plot_projection · monthly projection shown by year · the book's own premiums") |
| scr_fig = chart(sc.plot_scr(res["scr"], title="Life SCR build-up"), "sc.scr_life · sc.plot_scr") |
| scr_tab = text_table(pd.DataFrame(res["scr"]).rename_axis("sub-risk").reset_index(), |
| {"charge": "{:,.0f}", "marginal": "{:,.0f}", "share": "{:.1%}"}) |
| csm_fig = chart(sc.plot_csm(res["csm"], title="CSM roll-forward"), "sc.csm · sc.plot_csm") |
| csm_tab = text_table(pd.DataFrame(res["csm"]).head(12), |
| {c_: "{:,.0f}" for c_ in ["coverage_units", "csm_open", "release", "accretion", "csm_close"]}) |
| pack = study.board_pack(res, a, ra, basis_label) |
|
|
| f = study.facts()["engines"]["B"] |
| engines = pd.DataFrame( |
| [ |
| ("PV net cash flow · lifelib BasicTerm_ME (lifelib's own basis)", f["ide_lifelib_pv_net"], |
| f["lifelib_pv_net"], "Scelo IDE 0.1.6 and scelo lifelib_run: identical"), |
| ("PV net cash flow · scelo BasicTerm port (illustrative basis)", None, f["port_pv_net"], |
| "Python and R agree"), |
| ("Life SCR", f["ide_scr"], f["lib_scr"], |
| "IDE: lifelib TradLife_A_EX1 on a 579-policy sample · scelo: standard formula, full book"), |
| ("IFRS 17 CSM at initial recognition", f["ide_csm"], f["lib_csm"], |
| "IDE: ifrs17sim on a 392-policy sample, RA 0 · scelo: general model, RA 5 %, full book"), |
| ], |
| columns=["figure", "Scelo IDE", "scelo (Python / R)", "note"], |
| ) |
| engines = text_table(engines, {"Scelo IDE": "{:,.0f}", "scelo (Python / R)": "{:,.0f}"}) |
| return (cards(items), caption, proj, notes_md(res["bt"].notes, res["bt"].basis), |
| scr_fig, scr_tab, notes_md(res["scr"].notes, res["scr"].basis), |
| csm_fig, csm_tab, notes_md(res["csm"].notes, res["csm"].basis), |
| pack, engines) |
|
|
|
|
| def data_view(name): |
| df = study.load(name) |
| head = (f"### `{name}` · {len(df):,} rows × {df.shape[1]} columns\n" |
| f"`{FILES[name]}` in [{DATASET}]({DATASET_URL}/viewer/{name}) · read with `sc.load`, " |
| "which types columns the way the Scelo IDE types an import. First 200 rows below.") |
| prof = pd.DataFrame(sc.profile(df)) |
| cols = [c_ for c_ in ["column", "type", "count", "missing", "unique", "min", "median", "max", "top_values"] |
| if c_ in prof.columns] |
| prof = prof[cols].copy() |
| for c_ in ["min", "median", "max"]: |
| if c_ in prof.columns: |
| prof[c_] = prof[c_].map(lambda v: "" if v is None or (isinstance(v, float) and np.isnan(v)) else |
| (f"{v:,.4g}" if isinstance(v, float) else str(v))) |
| if name == "term_book": |
| plan_md = ("#### Cleaning plan (`sc.suggest`)\nThe analysts declined all of these: *ENTP → ENTJ* is not " |
| "a typo, capping `face_amount_usd` would destroy the sum-assured tail, and imputing " |
| "`termination_year` onto in-force policies would fabricate lapses.") |
| plan = gr.Dataframe(value=as_text(pd.DataFrame(sc.suggest(df))), visible=True) |
| else: |
| plan_md, plan = "", gr.Dataframe(visible=False) |
| return head, as_text(df.head(200)), as_text(prof), plan_md, plan |
|
|
|
|
| |
| |
| |
|
|
| HEADER = f""" |
| <div class="ia-head"> |
| <div class="ia-links"> |
| <a href="{DATASET_URL}" target="_blank">Dataset ↗</a> |
| <a href="{SCELO_DOCS_URL}" target="_blank">scelo docs ↗</a> |
| <a href="{SOA_REQUEST_URL}" target="_blank">SOA request ↗</a> |
| </div> |
| <h1>Lapse & Surrender Study 2023–25</h1> |
| <span class="ia-tagline"><span class="dot"></span>10,000 synthetic policyholders · computed live with scelo</span> |
| <div class="ia-sub">A synthetic answer to the Society of Actuaries (SOA) Research Institute and LIMRA |
| <a href="{SOA_REQUEST_URL}" target="_blank">2023–2025 Term and Whole Life Lapse and Surrender Experience Study |
| data request</a> · Intelligent Actuaries</div> |
| </div> |
| <div class="ia-synthetic" style="margin-top:0.8rem"><b>Synthetic data.</b> Every record was generated by |
| language-model agents, not extracted from any insurer. It describes the generator, not a market: no valuation, |
| pricing or reserving assumption should be set from it. Not SOA or LIMRA data.</div> |
| """ |
|
|
| WHAT = f""" |
| ### What this is |
| The SOA Research Institute and LIMRA asked life insurers for **VM-51 policy-level records** of their individual |
| term and whole life business for 2023–2025. Intelligent Actuaries has no such book, so it generated one: |
| **10,000 synthetic policyholders** in ten markets, each an agent in the Scelo swarm (Claude Code, |
| `claude-sonnet-5`), described the one policy they hold and what happened to it in 2023, 2024 and 2025. A |
| deterministic driver validated every answer, overlaid mortality and wrote the **27,692 policy-year records** in |
| the request's Appendix B layout. |
| |
| This Space loads that dataset from the Hub and recomputes everything with the [`scelo`]({SCELO_DOCS_URL}) |
| Python library: experience rates and credibility, the lapse GLM, and the projection, SCR and CSM of the in-force |
| term book on a basis you choose. |
| """ |
|
|
| ACK_CARD = f""" |
| <div class="ia-ack">We gratefully acknowledge the <b>Society of Actuaries Research Institute</b>, the research arm |
| of the Society of Actuaries (SOA), and <b>LIMRA</b>. Their <a href="{SOA_REQUEST_URL}" target="_blank">2023–2025 |
| data request</a> defines the scope, the study window and the VM-51 record layout this dataset imitates, and their |
| <a href="{SOA_2015_22_URL}" target="_blank">2015–2022 study</a> is the published reference for this experience. |
| This is independent work: not SOA or LIMRA data, and not affiliated with or endorsed by either organisation.</div> |
| """ |
|
|
| ABOUT_LEFT = f""" |
| ### Acknowledgements |
| We gratefully acknowledge the **Society of Actuaries Research Institute**, the research arm of the Society of |
| Actuaries (SOA), and **LIMRA**, who run the Term and Whole Life Lapse and Surrender Experience Study together. |
| |
| - Their [**2023–2025 data request**]({SOA_REQUEST_URL}) (July 2026) defines what this dataset imitates: the scope |
| (level-premium term, annually renewable term, participating and non-participating whole life), the study window |
| (calendar years 2023 to 2025) and the policy-level record layout (the VM-51 format, the request's Appendix B). |
| - Their [**2015–2022 Term and Whole Life Lapse and Surrender Experience Study**]({SOA_2015_22_URL}) (2024), the |
| edition the request updates, is the published reference for this experience. |
| - The record format is the **NAIC Valuation Manual's VM-51**, which the request adopts. |
| |
| **Independence.** This is independent work by Intelligent Actuaries. It is not an SOA or LIMRA publication or |
| dataset, and it is not affiliated with, sponsored by or endorsed by the SOA, the SOA Research Institute, LIMRA or |
| the NAIC. It contains no SOA, LIMRA or contributing-company experience, and the request document (© 2026 Society |
| of Actuaries Research Institute and LIMRA) is not redistributed. Any errors are ours. |
| |
| ### How the data was made |
| 1. **Inversion.** The request asks insurers for VM-51 records; here each agent is one *policyholder*, answering a |
| questionnaire derived from Appendix B. |
| 2. **Design.** 10 markets × 6 attained-age bands × 16 MBTI types × 2 sexes, balanced, seed `20260822`; |
| socio-economic attributes from policyholder-population priors. |
| 3. **Conditioning.** Each agent sees its persona and a dated context block for its own market (August 2023 to |
| August 2026). |
| 4. **Generation.** 10,000 calls to `claude-sonnet-5` through Claude Code on the Scelo swarm's society tier, |
| 22 August 2026. |
| 5. **Validation.** Product-contract rules with every repair logged (79 policies); 11 premiums estimated and flagged. |
| 6. **Mortality overlay.** An imposed Gompertz decrement, so the exposure and cause-04 terminations are complete. |
| 7. **Expansion.** 27,692 VM-51 policy-year records, fixed-width and CSV. |
| |
| ### Limitations |
| - Every statistic describes the generator, not a market. |
| - The MBTI signal is designed in; the mortality is imposed; the design is balanced, not census-weighted |
| (post-stratify before reading anything as a market figure). |
| - Calendar-year timing is narrative-biased: estimate lapse as one force over 2023–25 with an exposure offset. |
| - 94.5 % of agents said they had considered lapsing: an artefact of asking, not a rate. |
| - The free text is model-written and may carry stereotypes about markets and communities. |
| """ |
|
|
| ABOUT_RIGHT = f""" |
| ### scelo in this Space |
| Every number is a [`scelo`]({SCELO_DOCS_URL}) call on the published files, computed when you move a |
| control; nothing is precomputed except the cross-engine reference figures. |
| |
| | Stage | Calls | |
| |---|---| |
| | Soft data | `sc.load` · `sc.profile` · `sc.tab` · `sc.suggest` | |
| | Experience | `sc.freq_sev` · `sc.credibility` · `sc.plot_rates` | |
| | Pricing | `sc.glm` · `.relativities()` · `sc.lift` · `sc.gini` · `sc.plot_relativities` · `sc.plot_lines` | |
| | Valuation | `sc.basicterm` · `sc.scr_life` · `sc.csm` · `sc.plot_projection` · `sc.plot_scr` · `sc.plot_csm` | |
| | Hard data | `sc.hard` · `sc.report` | |
| |
| ```bash |
| pip install scelo |
| ``` |
| |
| ### Cite |
| ```bibtex |
| @misc{{denewade2026syntheticlapse, |
| author = {{Denewade, Ali}}, |
| title = {{Synthetic Term and Whole Life Lapse and Surrender Study, |
| 2023--2025: 10,000 synthetic policyholders in the VM-51 layout}}, |
| year = {{2026}}, |
| publisher = {{Hugging Face}}, |
| howpublished = {{\\url{{{DATASET_URL}}}}}, |
| note = {{Synthetic data, generated in response to the SOA Research |
| Institute and LIMRA 2023--2025 Term and Whole Life Lapse and |
| Surrender Experience Study data request. Not SOA or LIMRA data.}} |
| }} |
| ``` |
| |
| Contact: [research@intelligentactuaries.com](mailto:research@intelligentactuaries.com) · |
| [intelligentactuaries.com](https://intelligentactuaries.com) |
| """ |
|
|
| FOOTER = f""" |
| <div class="ia-footer">Synthetic data generated by Intelligent Actuaries in response to the |
| <a href="{SOA_REQUEST_URL}" target="_blank">SOA Research Institute and LIMRA 2023–2025 Term and Whole Life Lapse and |
| Surrender Experience Study data request</a>, with thanks to both organisations. Not SOA or LIMRA data; not |
| affiliated with or endorsed by the SOA or LIMRA. Computed with scelo {sc.__version__}. Code MIT · data CC BY 4.0.</div> |
| """ |
|
|
| MARKET_CHOICES = [(f"{c} · {n}", c) for c, n in MARKETS.items()] |
|
|
|
|
| |
| |
| |
|
|
| with gr.Blocks(title="Lapse & Surrender Study · scelo", theme=THEME, css=CSS, delete_cache=(3600, 3600)) as demo: |
| gr.HTML(HEADER) |
|
|
| with gr.Tabs(): |
| |
| with gr.Tab("🧭 Overview"): |
| ov_cards = gr.HTML(cards([("Loading", "…", "reading the dataset from the Hub")])) |
| with gr.Row(): |
| with gr.Column(scale=3): |
| ov_plot = gr.HTML() |
| ov_mix = gr.Dataframe(label="Status by observation year · sc.tab", interactive=False) |
| with gr.Column(scale=2): |
| gr.Markdown(WHAT) |
| gr.HTML("<h3 style='margin:0.6rem 0 0.4rem'>Acknowledgement</h3>" + ACK_CARD) |
| ov_products = gr.Dataframe(label="Products · sc.tab", interactive=False) |
|
|
| |
| with gr.Tab("📉 Experience") as tab_exp: |
| with gr.Row(): |
| with gr.Column(scale=1, min_width=270): |
| x_dec = gr.Dropdown(list(DECREMENTS), value="Lapse", label="Decrement") |
| x_by = gr.Dropdown(list(EXPERIENCE_BY), value="Market", label="Group by") |
| x_markets = gr.CheckboxGroup(MARKET_CHOICES, value=list(MARKETS), label="Markets") |
| x_products = gr.CheckboxGroup(PRODUCTS, value=PRODUCTS, label="Products") |
| x_years = gr.CheckboxGroup(YEARS, value=YEARS, label="Observation years") |
| x_cred = gr.Checkbox(False, label="Bühlmann–Straub credibility (sc.credibility)") |
| with gr.Column(scale=3): |
| x_cards = gr.HTML() |
| x_plot = gr.HTML() |
| x_table = gr.Dataframe(label="Occurrence / exposure · sc.freq_sev", interactive=False) |
| x_notes = gr.Markdown(elem_classes="ia-note") |
| x_cred_table = gr.Dataframe(label="Credibility-weighted rates", interactive=False, visible=False) |
| x_cred_notes = gr.Markdown(elem_classes="ia-note") |
|
|
| |
| with gr.Tab("🧮 Lapse GLM") as tab_glm: |
| with gr.Row(): |
| with gr.Column(scale=1, min_width=270): |
| g_pop = gr.Radio([("Term book 2023–25 (4,254) · the published model", "term"), |
| ("Every policy, all products (10,000)", "all")], |
| value="term", label="Population") |
| g_cov = gr.CheckboxGroup(list(GLM_COVARIATES), value=["Market", "Product", "MBTI type"], |
| label="Covariates") |
| g_base = gr.Radio([("Alphabetically first (as published)", "first"), |
| ("Most frequent level", "frequent")], value="first", label="Reference level") |
| gr.Markdown("Poisson GLM, log link, offset log(exposure) · `sc.glm`", elem_classes="ia-note") |
| with gr.Column(scale=3): |
| g_cards = gr.HTML() |
| g_tie = gr.Markdown(elem_classes="ia-note") |
| g_forest = gr.HTML() |
| with gr.Row(): |
| g_rel = gr.Dataframe(label="Relativities", interactive=False, max_height=460) |
| g_lift = gr.HTML() |
| with gr.Accordion("Lift table and coefficient table", open=False): |
| g_lift_t = gr.Dataframe(label="Lift (equal-exposure deciles)", interactive=False) |
| g_coef = gr.Dataframe(label="Coefficients", interactive=False) |
| g_caveat = gr.Markdown() |
|
|
| |
| with gr.Tab("💰 Pricing & valuation") as tab_val: |
| with gr.Row(): |
| with gr.Column(scale=1, min_width=270): |
| v_mode = gr.Radio([("scelo illustrative · 5.0 % p.a.", "illustrative"), |
| ("Term-book experience 2023–25 · 12.97 % p.a.", "experience"), |
| ("Custom", "custom")], value="illustrative", label="Lapse basis") |
| v_custom = gr.Slider(0.0, 0.40, value=0.08, step=0.005, label="Custom annual lapse rate", |
| interactive=False) |
| v_disc = gr.Slider(0.0, 0.08, value=0.03, step=0.0025, label="Discount rate (p.a.)") |
| v_mort = gr.Slider(0.5, 2.0, value=1.0, step=0.05, label="Mortality multiplier") |
| v_acq = gr.Slider(0.0, 500.0, value=100.0, step=10.0, label="Acquisition expense per policy") |
| v_maint = gr.Slider(0.0, 20.0, value=5.0, step=0.5, label="Maintenance expense per policy-month") |
| v_prem = gr.Slider(0.5, 1.5, value=1.0, step=0.05, label="Premium multiplier") |
| v_ra = gr.Slider(0.0, 0.20, value=0.05, step=0.01, label="IFRS 17 risk adjustment (% of PV claims)") |
| gr.Markdown("In-force term book at 31 Dec 2025 · 2,698 model points", elem_classes="ia-note") |
| with gr.Column(scale=3): |
| v_cards = gr.HTML() |
| v_caption = gr.Markdown(elem_classes="ia-note") |
| v_proj = gr.HTML() |
| v_proj_notes = gr.Markdown(elem_classes="ia-note") |
| with gr.Row(): |
| with gr.Column(): |
| v_scr = gr.HTML() |
| v_scr_t = gr.Dataframe(interactive=False, show_label=False) |
| v_scr_notes = gr.Markdown(elem_classes="ia-note") |
| with gr.Column(): |
| v_csm = gr.HTML() |
| v_csm_t = gr.Dataframe(interactive=False, show_label=False) |
| v_csm_notes = gr.Markdown(elem_classes="ia-note") |
| gr.Markdown("#### Board pack\n`sc.hard` stamps each table with a content hash and its audit " |
| "entries; `sc.report` assembles them, with the acknowledgement and the " |
| "synthetic-data statement, into a printable board pack.") |
| v_pack = gr.DownloadButton("⬇️ Download the board pack (HTML)", variant="secondary") |
| with gr.Accordion("The same book on the other engines (published results)", open=False): |
| v_engines = gr.Dataframe(interactive=False, show_label=False) |
| gr.Markdown("From `analysis/facts.json` in the dataset. The sampled IDE figures are " |
| "comparable for scale, not for identity.", elem_classes="ia-note") |
|
|
| |
| with gr.Tab("🗂️ Data") as tab_data: |
| with gr.Row(): |
| with gr.Column(scale=1, min_width=270): |
| d_name = gr.Radio([(f"{n} · {ROWS[n]:,}", n) for n in FILES], value="term_book", |
| label="Config") |
| gr.Markdown(f"[Open the dataset on Hugging Face ↗]({DATASET_URL})") |
| with gr.Column(scale=3): |
| d_head = gr.Markdown() |
| d_preview = gr.Dataframe(interactive=False, max_height=380, show_label=False) |
| gr.Markdown("#### Column profile (`sc.profile`)") |
| d_profile = gr.Dataframe(interactive=False, max_height=420, show_label=False) |
| d_plan_md = gr.Markdown() |
| d_plan = gr.Dataframe(interactive=False, visible=False, show_label=False) |
|
|
| |
| with gr.Tab("📜 About"): |
| with gr.Row(): |
| with gr.Column(scale=3): |
| gr.Markdown(ABOUT_LEFT) |
| with gr.Column(scale=2): |
| gr.Markdown(ABOUT_RIGHT) |
|
|
| gr.HTML(FOOTER) |
|
|
| |
| demo.load(overview, outputs=[ov_cards, ov_plot, ov_mix, ov_products]) |
|
|
| x_in = [x_dec, x_by, x_markets, x_products, x_years, x_cred] |
| x_out = [x_cards, x_plot, x_table, x_notes, x_cred_table, x_cred_notes] |
| gr.on([tab_exp.select] + [c.change for c in x_in], experience_view, x_in, x_out) |
|
|
| g_in = [g_pop, g_cov, g_base] |
| g_out = [g_cards, g_tie, g_forest, g_rel, g_lift, g_lift_t, g_coef, g_caveat] |
| gr.on([tab_glm.select] + [c.change for c in g_in], glm_view, g_in, g_out) |
|
|
| v_in = [v_mode, v_custom, v_disc, v_mort, v_acq, v_maint, v_prem, v_ra] |
| v_out = [v_cards, v_caption, v_proj, v_proj_notes, v_scr, v_scr_t, v_scr_notes, |
| v_csm, v_csm_t, v_csm_notes, v_pack, v_engines] |
| v_mode.change(lambda m: gr.Slider(interactive=m == "custom"), v_mode, v_custom) |
| v_sliders = [v_custom, v_disc, v_mort, v_acq, v_maint, v_prem, v_ra] |
| gr.on([tab_val.select, v_mode.change] + [s.release for s in v_sliders], value_view, v_in, v_out) |
|
|
| d_out = [d_head, d_preview, d_profile, d_plan_md, d_plan] |
| gr.on([tab_data.select, d_name.change], data_view, [d_name], d_out) |
|
|
| demo.launch() |
|
|
