Update app.py

app.py

@@ -1,77 +1,58 @@
 # app.py
-import os, io, math, json, warnings
+# Efficient Portfolio Advisor — CAPM-on-CML plot + 1,000-mix dataset + 3x3 suggestions
+# - X axis: historical sigma (from covariances over lookback)
+# - Y axis: CAPM E[r] = rf + beta * ERP
+# - Plot includes two efficient CML mixes: same-σ and same-μ as the user portfolio
+# - Dataset: 1,000 long-only candidate mixes from *current* universe (incl. VOO)
+# - Suggestions: Tabs Low/Medium/High, 3 picks each, chosen by exposure+embedding sim with MMR
+# - Embeddings: FinLang/finance-embeddings-investopedia
+# - Score = α * exposure_similarity + (1-α) * embedding_similarity (α=0.6); MMR λ=0.7
+# - CSV of dataset downloadable.
+import os, io, math, time, json, warnings
 warnings.filterwarnings("ignore")
 
-# --- make common caches writable even on locked-down containers ---
-APP_ROOT = os.path.abspath(os.path.dirname(__file__))
-DATA_DIR = os.path.join(APP_ROOT, "data")
-os.makedirs(DATA_DIR, exist_ok=True)
-
-# Matplotlib cache
-os.environ.setdefault("MPLCONFIGDIR", os.path.join(DATA_DIR, ".mplconfig"))
-os.makedirs(os.environ["MPLCONFIGDIR"], exist_ok=True)
-
-# Hugging Face / Sentence Transformers caches
-os.environ.setdefault("HF_HOME", os.path.join(DATA_DIR, ".huggingface"))
-os.environ.setdefault("HUGGINGFACE_HUB_CACHE", os.path.join(DATA_DIR, ".huggingface", "hub"))
-os.environ.setdefault("SENTENCE_TRANSFORMERS_HOME", os.path.join(DATA_DIR, ".sentencetransformers"))
-for d in [os.environ["HF_HOME"], os.environ["HUGGINGFACE_HUB_CACHE"], os.environ["SENTENCE_TRANSFORMERS_HOME"]]:
-    os.makedirs(d, exist_ok=True)
-
 from typing import List, Tuple, Dict, Optional
 
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
 from PIL import Image
-import gradio as gr
 import requests
 import yfinance as yf
+import gradio as gr
 
-from sentence_transformers import SentenceTransformer, util as st_util
-from sklearn.preprocessing import StandardScaler
-from sklearn.neighbors import KNeighborsRegressor
+# ---------------- config ----------------
+DATA_DIR = "data"
+os.makedirs(DATA_DIR, exist_ok=True)
 
-# =========================
-# Config
-# =========================
-DEFAULT_LOOKBACK_YEARS = 5
 MAX_TICKERS = 30
-MARKET_TICKER = "VOO"        # proxy for market portfolio
-BILLS_TICKER  = "BILLS"      # synthetic cash / T-Bills bucket
-
-EMBED_MODEL_NAME = "BAAI/bge-base-en-v1.5"  # fully local, no API keys
-
-POS_COLS  = ["ticker", "amount_usd", "weight_exposure", "beta"]
-SUG_COLS  = ["ticker", "weight_%", "amount_$"]
-EFF_COLS  = ["asset", "weight_%", "amount_$"]
-
-N_SYNTH   = 1000             # size of synthetic dataset per run
-MMR_K     = 40               # shortlist size before MMR
-MMR_LAMBDA = 0.65            # similarity vs diversity tradeoff
-
-# ---------------- FRED mapping (risk-free source) ----------------
-FRED_MAP = [
-    (1,  "DGS1"),
-    (2,  "DGS2"),
-    (3,  "DGS3"),
-    (5,  "DGS5"),
-    (7,  "DGS7"),
-    (10, "DGS10"),
-    (20, "DGS20"),
-    (30, "DGS30"),
-    (100, "DGS30"),
-]
-
+DEFAULT_LOOKBACK_YEARS = 10
+MARKET_TICKER = "VOO"       # market proxy on CML
+BILLS_LABEL  = "Bills"      # label for risk-free leg in efficient mixes (display only)
+
+SYNTH_ROWS = 1000           # dataset size for suggestions
+EMB_MODEL = "FinLang/finance-embeddings-investopedia"
+ALPHA = 0.60                # exposure-vs-embedding blend
+MMR_LAMBDA = 0.70           # MMR diversity strength
+SHORTLIST_K = 40            # shortlist before MMR per band
+
+# Globals updated with horizon changes
+HORIZON_YEARS = 10
+RF_CODE = "DGS10"
+RF_ANN = 0.0375  # initialized at launch
+
+# ---------------- helpers ----------------
 def fred_series_for_horizon(years: float) -> str:
     y = max(1.0, min(100.0, float(years)))
-    for cutoff, code in FRED_MAP:
-        if y <= cutoff:
-            return code
+    if y <= 2: return "DGS2"
+    if y <= 3: return "DGS3"
+    if y <= 5: return "DGS5"
+    if y <= 7: return "DGS7"
+    if y <= 10: return "DGS10"
+    if y <= 20: return "DGS20"
     return "DGS30"
 
 def fetch_fred_yield_annual(code: str) -> float:
-    # FRED CSV endpoint (no API key required). Fallback to 3% if it fails.
     url = f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={code}"
     try:
         r = requests.get(url, timeout=10)
@@ -82,96 +63,95 @@ def fetch_fred_yield_annual(code: str) -> float:
     except Exception:
         return 0.03
 
-# =========================
-# Data helpers
-# =========================
-def _to_cols_close(df: pd.DataFrame) -> pd.DataFrame:
-    """Coerce yfinance download to a single-level columns DataFrame of adjusted closes."""
-    if df is None or df.empty:
-        return pd.DataFrame()
-    if isinstance(df, pd.Series):
-        df = df.to_frame("Close")
-    if isinstance(df.columns, pd.MultiIndex):
-        level0 = df.columns.get_level_values(0).unique().tolist()
-        fields = df.columns.get_level_values(1).unique().tolist()
-        field = "Adj Close" if "Adj Close" in fields else ("Close" if "Close" in fields else fields[0])
-        out = {}
-        for t in level0:
-            col = (t, field)
-            if col in df.columns:
-                out[t] = df[col]
-        out_df = pd.DataFrame(out)
-        return out_df
-    else:
-        if "Adj Close" in df.columns:
-            return df[["Adj Close"]].rename(columns={"Adj Close": "SINGLE"})
-        if "Close" in df.columns:
-            return df[["Close"]].rename(columns={"Close": "SINGLE"})
-        num_cols = [c for c in df.columns if pd.api.types.is_numeric_dtype(df[c])]
-        if num_cols:
-            return df[[num_cols[0]]].rename(columns={num_cols[0]: "SINGLE"})
-        return pd.DataFrame()
-
 def fetch_prices_monthly(tickers: List[str], years: int) -> pd.DataFrame:
-    start = (pd.Timestamp.today(tz="UTC") - pd.DateOffset(years=int(years), days=7)).date()
-    end   = pd.Timestamp.today(tz="UTC").date()
-    df_raw = yf.download(
-        list(dict.fromkeys(tickers)),
+    tickers = list(dict.fromkeys([t.upper().strip() for t in tickers]))
+    start = (pd.Timestamp.today(tz="UTC") - pd.DateOffset(years=years, days=7)).date()
+    end = pd.Timestamp.today(tz="UTC").date()
+
+    df = yf.download(
+        tickers,
         start=start, end=end,
-        interval="1mo", auto_adjust=True, progress=False, group_by="ticker",
-        threads=True,
+        interval="1mo",
+        auto_adjust=True,
+        actions=False,
+        progress=False,
+        group_by="column",
+        threads=False,
     )
-    df = _to_cols_close(df_raw).copy()
-    if df.empty:
-        return df
-    if df.shape[1] == 1 and "SINGLE" in df.columns:
-        df.columns = [tickers[0]]
-    df = df.dropna(how="all").fillna(method="ffill")
-    return df
+
+    # Normalize to: columns = tickers, values = prices
+    if isinstance(df, pd.Series):
+        df = df.to_frame()
+    if isinstance(df.columns, pd.MultiIndex):
+        lvl0 = [str(x) for x in df.columns.get_level_values(0).unique()]
+        if "Close" in lvl0:
+            df = df["Close"]
+        elif "Adj Close" in lvl0:
+            df = df["Adj Close"]
+        else:
+            df = df.xs(df.columns.levels[0][-1], axis=1, level=0, drop_level=True)
+
+    cols = [c for c in tickers if c in df.columns]
+    out = df[cols].dropna(how="all").fillna(method="ffill")
+    return out
 
 def monthly_returns(prices: pd.DataFrame) -> pd.DataFrame:
     return prices.pct_change().dropna()
 
+def yahoo_search(query: str):
+    if not query or not str(query).strip():
+        return []
+    url = "https://query1.finance.yahoo.com/v1/finance/search"
+    params = {"q": query.strip(), "quotesCount": 10, "newsCount": 0}
+    headers = {"User-Agent": "Mozilla/5.0"}
+    try:
+        r = requests.get(url, params=params, headers=headers, timeout=10)
+        r.raise_for_status()
+        data = r.json()
+        out = []
+        for q in data.get("quotes", []):
+            sym = q.get("symbol")
+            name = q.get("shortname") or q.get("longname") or ""
+            exch = q.get("exchDisp") or ""
+            if sym and sym.isascii():
+                out.append(f"{sym}  |  {name}  |  {exch}")
+        if not out:
+            out = [f"{query.strip().upper()}  |  typed symbol  |  n/a"]
+        return out[:10]
+    except Exception:
+        return [f"{query.strip().upper()}  |  typed symbol  |  n/a"]
+
 def validate_tickers(symbols: List[str], years: int) -> List[str]:
-    """Return subset of symbols that have enough data over lookback."""
-    symbols = [s.strip().upper() for s in symbols if s and isinstance(s, str)]
-    base = [s for s in symbols if s != MARKET_TICKER]
+    base = [s for s in dict.fromkeys([t.upper().strip() for t in symbols]) if s]
     px = fetch_prices_monthly(base + [MARKET_TICKER], years)
-    ok = []
-    for s in symbols:
-        if s in px.columns:
-            ok.append(s)
+    ok = [s for s in base if s in px.columns]
+    if MARKET_TICKER not in px.columns:
+        return []  # need market for aligned CAPM
     return ok
 
-# =========================
-# Moments & CAPM
-# =========================
-def annualize_mean(m):   return np.asarray(m, dtype=float) * 12.0
-def annualize_sigma(s):  return np.asarray(s, dtype=float) * math.sqrt(12.0)
-
+# -------------- aligned moments --------------
 def get_aligned_monthly_returns(symbols: List[str], years: int) -> pd.DataFrame:
-    uniq = [c for c in dict.fromkeys(symbols)]
-    if MARKET_TICKER not in uniq:
-        uniq.append(MARKET_TICKER)
-    px = fetch_prices_monthly(uniq, years)
+    uniq = [c for c in dict.fromkeys(symbols) if c != MARKET_TICKER]
+    tickers = uniq + [MARKET_TICKER]
+    px = fetch_prices_monthly(tickers, years)
     rets = monthly_returns(px)
-    cols = [c for c in uniq if c in rets.columns]
+    cols = [c for c in uniq if c in rets.columns] + ([MARKET_TICKER] if MARKET_TICKER in rets.columns else [])
     R = rets[cols].dropna(how="any")
     return R.loc[:, ~R.columns.duplicated()]
 
 def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
-    R = get_aligned_monthly_returns(symbols + [MARKET_TICKER], years)
-    if MARKET_TICKER not in R.columns or R.shape[0] < 3:
-        raise ValueError("Not enough aligned data to estimate moments.")
+    R = get_aligned_monthly_returns(symbols, years)
+    if MARKET_TICKER not in R.columns or len(R) < 3:
+        raise ValueError("Not enough aligned data with market proxy.")
     rf_m = rf_ann / 12.0
 
     m = R[MARKET_TICKER]
     if isinstance(m, pd.DataFrame):
         m = m.iloc[:, 0].squeeze()
 
-    mu_m_ann    = float(annualize_mean(m.mean()))
-    sigma_m_ann = float(annualize_sigma(m.std(ddof=1)))
-    erp_ann     = float(mu_m_ann - rf_ann)
+    mu_m_ann = float(m.mean() * 12.0)
+    sigma_m_ann = float(m.std(ddof=1) * math.sqrt(12.0))
+    erp_ann = float(mu_m_ann - rf_ann)
 
     ex_m = m - rf_m
     var_m = float(np.var(ex_m.values, ddof=1))
@@ -183,7 +163,7 @@ def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
         cov_sm = float(np.cov(ex_s.values, ex_m.values, ddof=1)[0, 1])
         betas[s] = cov_sm / var_m
 
-    betas[MARKET_TICKER] = 1.0  # by definition
+    betas[MARKET_TICKER] = 1.0
 
     asset_cols = [c for c in R.columns if c != MARKET_TICKER]
     cov_m = np.cov(R[asset_cols].values.T, ddof=1) if asset_cols else np.zeros((0, 0))
@@ -206,77 +186,47 @@ def portfolio_stats(weights: Dict[str, float],
         return 0.0, rf_ann, 0.0
     w_expo = w / gross
     beta_p = float(np.dot([betas.get(t, 0.0) for t in tickers], w_expo))
-    er_capm = capm_er(beta_p, rf_ann, erp_ann)
+    mu_capm = capm_er(beta_p, rf_ann, erp_ann)
     cov = cov_ann.reindex(index=tickers, columns=tickers).fillna(0.0).to_numpy()
-    sigma_p = math.sqrt(max(float(w_expo.T @ cov @ w_expo), 0.0))
-    return beta_p, er_capm, sigma_p
+    sigma_hist = float(max(w_expo.T @ cov @ w_expo, 0.0)) ** 0.5
+    return beta_p, mu_capm, sigma_hist
 
-# =========================
-# Efficient (CML) alternatives
-# =========================
+# -------------- efficient CML mixes --------------
 def efficient_same_sigma(sigma_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
-    """Weights (a on Market, b on Bills) and expected return on CML with same sigma."""
     if sigma_mkt <= 1e-12:
         return 0.0, 1.0, rf_ann
     a = sigma_target / sigma_mkt
     return a, 1.0 - a, rf_ann + a * erp_ann
 
 def efficient_same_return(mu_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
-    """Weights (a on Market, b on Bills) and sigma on CML with same expected return."""
     if abs(erp_ann) <= 1e-12:
-        return 0.0, 1.0, 0.0
+        return 0.0, 1.0, rf_ann
     a = (mu_target - rf_ann) / erp_ann
     return a, 1.0 - a, abs(a) * sigma_mkt
 
-# =========================
-# Plot
-# =========================
-def _pct_arr(x):
-    x = np.asarray(x, dtype=float)
-    return x * 100.0
-
-def plot_cml(
-    rf_ann, erp_ann, sigma_mkt,
-    pt_sigma_hist, pt_mu_capm,
-    same_sigma_sigma, same_sigma_mu,
-    same_mu_sigma, same_mu_mu,
-) -> Image.Image:
-    fig = plt.figure(figsize=(6.6, 4.4), dpi=130)
-
-    xmax = max(
-        0.3,
-        sigma_mkt * 2.0,
-        pt_sigma_hist * 1.4,
-        same_mu_sigma * 1.4,
-        same_sigma_sigma * 1.4,
-    )
+# -------------- plotting (CAPM on CML) --------------
+def _pct(x): return np.asarray(x, dtype=float) * 100.0
 
-    xs = np.linspace(0, xmax, 160)
-    slope = erp_ann / max(sigma_mkt, 1e-12)
-    cml = rf_ann + slope * xs
+def plot_cml(rf_ann, erp_ann, sigma_mkt,
+             sigma_hist, mu_capm,
+             sugg_mu=None, sugg_sigma=None) -> Image.Image:
+    fig = plt.figure(figsize=(6, 4), dpi=120)
 
-    plt.plot(_pct_arr(xs), _pct_arr(cml), label="CML via VOO", linewidth=1.8)
-    plt.scatter([0.0], [_pct_arr(rf_ann)], label="Risk-free", zorder=5)
-    plt.scatter([_pct_arr(sigma_mkt)], [_pct_arr(rf_ann + erp_ann)], label="Market (VOO)", zorder=5)
+    xmax = max(0.3, sigma_mkt * 2.2, (sigma_hist or 0.0) * 1.6, (sugg_sigma or 0.0) * 1.6)
+    xs = np.linspace(0, xmax, 200)
+    cml = rf_ann + (erp_ann / max(sigma_mkt, 1e-9)) * xs
 
-    # Your portfolio point uses CAPM expected return + historical sigma
-    plt.scatter([_pct_arr(pt_sigma_hist)], [_pct_arr(pt_mu_capm)], label="Your portfolio (CAPM)", zorder=6)
+    plt.plot(_pct(xs), _pct(cml), label="CML via Market", linewidth=1.8)
+    plt.scatter([_pct(0)], [_pct(rf_ann)], label="Risk-free")
+    plt.scatter([_pct(sigma_mkt)], [_pct(rf_ann + erp_ann)], label="Market (VOO)")
+    plt.scatter([_pct(sigma_hist)], [_pct(mu_capm)], label="Your CAPM point", marker="o")
 
-    # Efficient matches
-    plt.scatter([_pct_arr(same_sigma_sigma)], [_pct_arr(same_sigma_mu)], label="Efficient: same σ", zorder=5)
-    plt.scatter([_pct_arr(same_mu_sigma)],    [_pct_arr(same_mu_mu)], label="Efficient: same μ", zorder=5)
+    if sugg_mu is not None and sugg_sigma is not None:
+        plt.scatter([_pct(sugg_sigma)], [_pct(sugg_mu)], label="Selected Suggestion", marker="X", s=60)
 
-    # helper guides
-    plt.plot([_pct_arr(pt_sigma_hist), _pct_arr(same_sigma_sigma)],
-             [_pct_arr(pt_mu_capm),    _pct_arr(same_sigma_mu)],
-             ls="--", lw=1.1, alpha=0.7, color="gray")
-    plt.plot([_pct_arr(pt_sigma_hist), _pct_arr(same_mu_sigma)],
-             [_pct_arr(pt_mu_capm),    _pct_arr(same_mu_mu)],
-             ls="--", lw=1.1, alpha=0.7, color="gray")
-
-    plt.xlabel("σ (annual, %)")
-    plt.ylabel("E[return] (annual, %)")
-    plt.legend(loc="best", fontsize=8)
+    plt.xlabel("σ (annualized, %)")
+    plt.ylabel("Expected return (annual, %)")
+    plt.legend(loc="best")
     plt.tight_layout()
 
     buf = io.BytesIO()
@@ -285,180 +235,229 @@ def plot_cml(
     buf.seek(0)
     return Image.open(buf)
 
-# =========================
-# Synthetic dataset (for recommendations)
-# =========================
-def dirichlet_signed(k, rng):
-    signs = rng.choice([-1.0, 1.0], size=k, p=[0.25, 0.75])
-    raw = rng.dirichlet(np.ones(k))
-    gross = 1.0 + float(rng.gamma(2.0, 0.5))
-    return gross * signs * raw
-
-def build_synth_dataset(universe: List[str],
-                        cov_ann: pd.DataFrame,
-                        betas: Dict[str, float],
-                        rf_ann: float, erp_ann: float,
-                        n_rows: int = N_SYNTH,
-                        seed: int = 123) -> pd.DataFrame:
-    rng = np.random.default_rng(seed)
-    U = [u for u in universe if u != MARKET_TICKER] + [MARKET_TICKER]
+# -------------- synthetic dataset (from current universe) --------------
+def build_synthetic_dataset(universe: List[str],
+                            covA: pd.DataFrame,
+                            betas: Dict[str, float],
+                            rf_ann: float,
+                            erp_ann: float,
+                            sigma_mkt: float,
+                            n_rows: int = SYNTH_ROWS) -> pd.DataFrame:
+    rng = np.random.default_rng(12345)
+    U = list(universe)
+    if not U:
+        U = [MARKET_TICKER]
+
     rows = []
     for i in range(n_rows):
-        k = rng.integers(low=min(2, len(U)), high=min(8, len(U)) + 1)
+        k = int(rng.integers(low=2, high=min(8, len(U)) + 1))
         picks = list(rng.choice(U, size=k, replace=False))
-        w = dirichlet_signed(k, rng)  # exposure weights (can include short)
-        gross = float(np.sum(np.abs(w)))
-        if gross <= 1e-12:
-            continue
-        w_expo = w / gross
-        weights = {picks[j]: float(w_expo[j]) for j in range(k)}
-        beta_i, er_capm_i, sigma_i = portfolio_stats(weights, cov_ann, betas, rf_ann, erp_ann)
+        w = rng.dirichlet(np.ones(k))           # long-only, sums to 1
+
+        beta_p = float(np.dot([betas.get(t, 0.0) for t in picks], w))
+        mu_capm = capm_er(beta_p, rf_ann, erp_ann)
+
+        sub = covA.reindex(index=picks, columns=picks).fillna(0.0).to_numpy()
+        sigma_hist = float(max(w.T @ sub @ w, 0.0)) ** 0.5
+
+        # CAPM "equivalent" sigma on CML for the same expected return
+        sigma_capm = abs(beta_p) * sigma_mkt
+
         rows.append({
-            "id": int(i),
             "tickers": ",".join(picks),
-            "weights": ",".join(f"{x:.6f}" for x in w_expo),
-            "beta": float(beta_i),
-            "er_capm": float(er_capm_i),
-            "sigma": float(sigma_i),
+            "weights": ",".join(f"{x:.6f}" for x in w),
+            "beta": beta_p,
+            "mu_capm": mu_capm,
+            "sigma_hist": sigma_hist,
+            "sigma_capm": sigma_capm
         })
-    df = pd.DataFrame(rows)
-    return df
+    return pd.DataFrame(rows)
+
+# -------------- banding by σ (CAPM) --------------
+def _band_bounds(sigma_mkt: float, band: str) -> Tuple[float, float]:
+    b = (band or "Medium").strip().lower()
+    if b.startswith("low"):   return 0.0, 0.8 * sigma_mkt
+    if b.startswith("high"):  return 1.2 * sigma_mkt, 3.0 * sigma_mkt
+    return 0.8 * sigma_mkt, 1.2 * sigma_mkt
 
-# =========================
-# Embeddings + MMR selection
-# =========================
+def slice_band(df: pd.DataFrame, band: str, sigma_mkt: float) -> pd.DataFrame:
+    lo, hi = _band_bounds(sigma_mkt, band)
+    pick = df[(df["sigma_capm"] >= lo) & (df["sigma_capm"] <= hi)].copy()
+    return pick if not pick.empty else df.copy()
+
+# -------------- embeddings + exposure similarity + MMR --------------
 _embedder = None
 def get_embedder():
     global _embedder
     if _embedder is None:
-        _embedder = SentenceTransformer(EMBED_MODEL_NAME)
+        from sentence_transformers import SentenceTransformer
+        _embedder = SentenceTransformer(EMB_MODEL)
     return _embedder
 
-def row_to_sentence(row: pd.Series) -> str:
-    try:
-        ts = row["tickers"].split(",")
-        ws = [float(x) for x in row["weights"].split(",")]
-        pairs = ", ".join([f"{ts[i]} {ws[i]:+.2f}" for i in range(min(len(ts), len(ws)))])
-    except Exception:
-        pairs = ""
-    return (f"portfolio with sigma {row['sigma']:.4f}, "
-            f"capm_return {row['er_capm']:.4f}, "
-            f"beta {row['beta']:.3f}, "
-            f"exposures {pairs}")
-
-def mmr_select(query_emb: np.ndarray,
-               cand_embs: np.ndarray,
-               k: int = 3,
-               lambda_param: float = MMR_LAMBDA) -> List[int]:
-    """
-    Maximal Marginal Relevance: pick k diverse-yet-relevant indices.
-    """
-    if cand_embs.shape[0] <= k:
-        return list(range(cand_embs.shape[0]))
-    sim_to_query = st_util.cos_sim(query_emb, cand_embs).cpu().numpy().reshape(-1)
-    chosen = []
-    candidate_indices = list(range(cand_embs.shape[0]))
-    first = int(np.argmax(sim_to_query))
-    chosen.append(first)
-    candidate_indices.remove(first)
-    while len(chosen) < k and candidate_indices:
-        max_score = -1e9
-        max_idx = candidate_indices[0]
-        for idx in candidate_indices:
-            sim_q = sim_to_query[idx]
-            sim_d = max(st_util.cos_sim(cand_embs[idx], cand_embs[chosen]).cpu().numpy().reshape(-1))
-            mmr_score = lambda_param * sim_q - (1.0 - lambda_param) * sim_d
-            if mmr_score > max_score:
-                max_score = mmr_score
-                max_idx = idx
-        chosen.append(max_idx)
-        candidate_indices.remove(max_idx)
+def _weights_dict_from_row(r: pd.Series) -> Dict[str, float]:
+    ts = [t.strip().upper() for t in str(r["tickers"]).split(",")]
+    ws = [float(x) for x in str(r["weights"]).split(",")]
+    wmap = {ts[i]: ws[i] for i in range(min(len(ts), len(ws)))}
+    s = sum(wmap.values()) or 1.0
+    return {k: max(0.0, v) / s for k, v in wmap.items()}  # ensure long-only normalized
+
+def _aligned_vec(universe: List[str], wmap: Dict[str, float]) -> np.ndarray:
+    # vector in the same order
+    return np.array([float(wmap.get(t, 0.0)) for t in universe], dtype=float)
+
+def cosine_sim(a: np.ndarray, b: np.ndarray) -> float:
+    na = np.linalg.norm(a); nb = np.linalg.norm(b)
+    if na == 0 or nb == 0: return 0.0
+    return float(np.dot(a, b) / (na * nb))
+
+def portfolio_embedding(weights: Dict[str, float]) -> np.ndarray:
+    # weighted average of ticker embeddings
+    model = get_embedder()
+    toks = list(weights.keys())
+    if not toks: return np.zeros((get_embedder().get_sentence_embedding_dimension(),), dtype=float)
+    embs = model.encode(toks, convert_to_numpy=True, normalize_embeddings=True)
+    w = np.array([weights[t] for t in toks], dtype=float)
+    w = w / (w.sum() or 1.0)
+    vec = (embs * w[:, None]).sum(axis=0)
+    # normalize
+    n = np.linalg.norm(vec)
+    return vec / (n if n else 1.0)
+
+def mmr(query_vec: np.ndarray, cand_vecs: np.ndarray, k: int, lam: float) -> List[int]:
+    # classic MMR on cosine sim
+    if len(cand_vecs) <= k: return list(range(len(cand_vecs)))
+    sims_q = cand_vecs @ query_vec
+    chosen = [int(np.argmax(sims_q))]
+    candidates = set(range(len(cand_vecs))) - set(chosen)
+    while len(chosen) < k and candidates:
+        best_i, best_score = None, -1e9
+        for i in list(candidates):
+            sim_q = sims_q[i]
+            sim_d = max(float(cand_vecs[i] @ cand_vecs[j]) for j in chosen)
+            score = lam * sim_q - (1.0 - lam) * sim_d
+            if score > best_score:
+                best_score = score; best_i = i
+        chosen.append(best_i); candidates.remove(best_i)
     return chosen
 
-# =========================
-# Yahoo symbol search (for UX)
-# =========================
-def yahoo_search(query: str):
-    if not query or len(query.strip()) == 0:
-        return []
-    url = "https://query1.finance.yahoo.com/v1/finance/search"
-    params = {"q": query.strip(), "quotesCount": 10, "newsCount": 0}
-    headers = {"User-Agent": "Mozilla/5.0"}
-    try:
-        r = requests.get(url, params=params, headers=headers, timeout=10)
-        r.raise_for_status()
-        data = r.json()
-        out = []
-        for q in data.get("quotes", []):
-            sym = q.get("symbol")
-            name = q.get("shortname") or q.get("longname") or ""
-            exch = q.get("exchDisp") or ""
-            if sym and sym.isascii():
-                out.append(f"{sym}  |  {name}  |  {exch}")
-        if not out:
-            out = [f"{query.strip().upper()}  |  typed symbol  |  n/a"]
-        return out[:10]
-    except Exception:
-        return [f"{query.strip().upper()}  |  typed symbol  |  n/a"]
-
-_last_matches = []  # updated on each search
+def pick_3_for_band(synth: pd.DataFrame,
+                    band: str,
+                    sigma_mkt: float,
+                    uni: List[str],
+                    user_w: Dict[str, float]) -> Tuple[List[Dict], List[pd.DataFrame]]:
+    # shortlist by top CAPM returns within band
+    band_df = slice_band(synth, band, sigma_mkt)
+    band_df = band_df.sort_values("mu_capm", ascending=False).head(SHORTLIST_K).reset_index(drop=True)
+    if band_df.empty:
+        return [], []
+
+    # exposure vectors
+    user_vec = _aligned_vec(uni, user_w)
+
+    # portfolio embedding
+    q_emb = portfolio_embedding(user_w)
+
+    # candidate embeddings (weighted avg of ticker embeddings)
+    c_wmaps = [ _weights_dict_from_row(r) for _, r in band_df.iterrows() ]
+    toks_list = [list(wm.keys()) for wm in c_wmaps]
+    # flatten encode unique tokens once
+    tok_set = sorted(set(t for toks in toks_list for t in toks))
+    model = get_embedder()
+    tok_embs = model.encode(tok_set, convert_to_numpy=True, normalize_embeddings=True)
+    tok_idx = {t:i for i,t in enumerate(tok_set)}
+
+    cand_vecs = []
+    expo_sims = []
+    for wm in c_wmaps:
+        # exposure sim (cosine on aligned vectors)
+        c_vec = _aligned_vec(uni, wm)
+        expo_sims.append(cosine_sim(user_vec, c_vec))
+        # weighted-avg ticker embedding
+        if wm:
+            w = np.array([wm[t] for t in wm.keys()], dtype=float)
+            w = w / (w.sum() or 1.0)
+            e = np.vstack([tok_embs[tok_idx[t]] for t in wm.keys()])
+            v = (e * w[:,None]).sum(axis=0)
+            v = v / (np.linalg.norm(v) or 1.0)
+            cand_vecs.append(v)
+        else:
+            cand_vecs.append(np.zeros_like(tok_embs[0]))
+
+    cand_vecs = np.vstack(cand_vecs)
+    # embedding sim: dot with q_emb (already normalized)
+    emb_sims = cand_vecs @ q_emb
+
+    # blended score
+    scores = ALPHA * np.array(expo_sims) + (1.0 - ALPHA) * np.array(emb_sims)
+    short_idx = np.argsort(-scores)[:min(12, len(scores))]
+
+    # MMR on the short list to get 3 diverse
+    mmr_idx_local = mmr(q_emb, cand_vecs[short_idx], k=3, lam=MMR_LAMBDA)
+    chosen = [int(short_idx[i]) for i in mmr_idx_local]
+    picks = band_df.iloc[chosen].reset_index(drop=True)
+
+    # tables (% and $) for each pick
+    gross_amt = sum(abs(v) for v in user_w.values()) or 1.0
+    tbls = []
+    metas = []
+    for _, r in picks.iterrows():
+        wm = _weights_dict_from_row(r)
+        rows = [{"ticker": t, "weight_%": round(w*100.0, 2), "amount_$": round(w*gross_amt, 2)} for t, w in wm.items()]
+        df = pd.DataFrame(rows, columns=["ticker", "weight_%", "amount_$"]).sort_values("weight_%", ascending=False)
+        tbls.append(df.reset_index(drop=True))
+        metas.append({"mu": float(r["mu_capm"]), "sigma": float(r["sigma_capm"])})
+    return metas, tbls
+
+# -------------- UI helpers --------------
+def empty_positions_df():
+    return pd.DataFrame(columns=["ticker", "amount_usd", "weight_exposure", "beta"])
+
+def empty_suggestion_df():
+    return pd.DataFrame(columns=["ticker", "weight_%", "amount_$"])
 
-# =========================
-# Formatting helpers
-# =========================
-def fmt_pct(x: float) -> str:
-    return f"{x*100:.2f}%"
+def set_horizon(years: float):
+    y = max(1.0, min(100.0, float(years)))
+    code = fred_series_for_horizon(y)
+    rf = fetch_fred_yield_annual(code)
+    global HORIZON_YEARS, RF_CODE, RF_ANN
+    HORIZON_YEARS = y
+    RF_CODE = code
+    RF_ANN = rf
+    return f"Risk-free series {code}. Latest annual rate {rf:.2%}."
 
-def fmt_money(x: float) -> str:
-    return f"${x:,.0f}"
+def search_tickers_cb(q: str):
+    opts = yahoo_search(q)
+    note = "Select a symbol and click 'Add selected to portfolio'." if opts else "No matches."
+    return note, gr.update(choices=opts, value=None)
 
-# =========================
-# Gradio callbacks
-# =========================
-HORIZON_YEARS = 5.0
-RF_CODE = fred_series_for_horizon(HORIZON_YEARS)
-RF_ANN  = fetch_fred_yield_annual(RF_CODE)
-
-def do_search(query):
-    global _last_matches
-    _last_matches = yahoo_search(query)
-    note = "Select a symbol from Matches, then click Add."
-    return note, gr.update(choices=_last_matches, value=None)
-
-def add_symbol(selection: str, table: pd.DataFrame):
-    if selection and "  |  " in selection:
-        symbol = selection.split("  |  ")[0].strip().upper()
-    elif isinstance(selection, str) and selection.strip():
-        symbol = selection.strip().upper()
-    else:
-        return table, "Pick a row from Matches first."
+def add_symbol(selection: str, table: Optional[pd.DataFrame]):
+    if not selection:
+        return table if isinstance(table, pd.DataFrame) else pd.DataFrame(columns=["ticker","amount_usd"]), "Pick a row in Matches first."
+    symbol = selection.split("|")[0].strip().upper()
 
     current = []
-    if table is not None and len(table) > 0:
+    if isinstance(table, pd.DataFrame) and not table.empty:
         current = [str(x).upper() for x in table["ticker"].tolist() if str(x) != "nan"]
-
     tickers = current if symbol in current else current + [symbol]
-    tickers = [t for t in tickers if t]
 
     val = validate_tickers(tickers, years=DEFAULT_LOOKBACK_YEARS)
     tickers = [t for t in tickers if t in val]
 
     amt_map = {}
-    if table is not None and len(table) > 0:
+    if isinstance(table, pd.DataFrame) and not table.empty:
         for _, r in table.iterrows():
             t = str(r.get("ticker", "")).upper()
             if t in tickers:
                 amt_map[t] = float(pd.to_numeric(r.get("amount_usd", 0.0), errors="coerce") or 0.0)
 
     new_table = pd.DataFrame({"ticker": tickers, "amount_usd": [amt_map.get(t, 0.0) for t in tickers]})
-    msg = f"Added {symbol}" if symbol in tickers else f"{symbol} not valid or no data"
     if len(new_table) > MAX_TICKERS:
         new_table = new_table.iloc[:MAX_TICKERS]
-        msg = f"Reached max of {MAX_TICKERS}"
-    return new_table, msg
+        return new_table, f"Reached max of {MAX_TICKERS}."
+    return new_table, f"Added {symbol}."
 
-def lock_ticker_column(tb: pd.DataFrame):
-    if tb is None or len(tb) == 0:
+def lock_ticker_column(tb: Optional[pd.DataFrame]):
+    if not isinstance(tb, pd.DataFrame) or tb.empty:
         return pd.DataFrame(columns=["ticker", "amount_usd"])
     tickers = [str(x).upper() for x in tb["ticker"].tolist()]
     amounts = pd.to_numeric(tb["amount_usd"], errors="coerce").fillna(0.0).tolist()
@@ -467,335 +466,289 @@ def lock_ticker_column(tb: pd.DataFrame):
     amounts = amounts[:len(tickers)] + [0.0] * max(0, len(tickers) - len(amounts))
     return pd.DataFrame({"ticker": tickers, "amount_usd": amounts})
 
-def set_horizon(years: float):
-    y = max(1.0, min(100.0, float(years)))
-    code = fred_series_for_horizon(y)
-    rf = fetch_fred_yield_annual(code)
-    global HORIZON_YEARS, RF_CODE, RF_ANN
-    HORIZON_YEARS = y
-    RF_CODE = code
-    RF_ANN = rf
-    return f"Risk-free series {code}. Latest annual rate {rf:.2%}. Computations will use this.", rf
-
-def _table_from_weights(weights: Dict[str, float], gross_amt: float) -> pd.DataFrame:
-    items = []
-    for t, w in weights.items():
-        pct = float(w)
-        amt = float(w) * gross_amt
-        items.append({"ticker": t, "weight_%": round(pct * 100.0, 2), "amount_$": round(amt, 2)})
-    df = pd.DataFrame(items, columns=SUG_COLS)
-    df["absw"] = df["weight_%"].abs()
-    df = df.sort_values("absw", ascending=False).drop(columns=["absw"])
-    return df
-
-def _weights_dict_from_row(r: pd.Series) -> Dict[str, float]:
-    ts = [t.strip().upper() for t in str(r["tickers"]).split(",")]
-    ws = [float(x) for x in str(r["weights"]).split(",")]
-    wmap = {}
-    for i in range(min(len(ts), len(ws))):
-        wmap[ts[i]] = ws[i]
-    gross = sum(abs(v) for v in wmap.values())
+# -------------- main compute --------------
+UNIVERSE: List[str] = [MARKET_TICKER, "QQQ", "VTI", "SOXX", "IBIT"]
+
+def compute(
+    years_lookback: int,
+    table: Optional[pd.DataFrame],
+    pick_low: int,
+    pick_med: int,
+    pick_high: int
+):
+    # sanitize table
+    if isinstance(table, pd.DataFrame):
+        df = table.copy()
+    else:
+        df = pd.DataFrame(columns=["ticker", "amount_usd"])
+    df = df.dropna(how="all")
+    for col in ("ticker","amount_usd"):
+        if col not in df.columns: df[col] = []
+    df["ticker"] = df["ticker"].astype(str).str.upper().str.strip()
+    df["amount_usd"] = pd.to_numeric(df["amount_usd"], errors="coerce").fillna(0.0)
+
+    symbols = [t for t in df["ticker"].tolist() if t]
+    if len(symbols) == 0:
+        empty = empty_positions_df()
+        e = "Add at least one ticker."
+        return None, e, "Universe empty.", empty, empty_suggestion_df(), empty_suggestion_df(), empty_suggestion_df(), empty_suggestion_df(), empty_suggestion_df(), empty_suggestion_df(), json.dumps({}), e
+
+    symbols = validate_tickers(symbols, years_lookback)
+    if len(symbols) == 0:
+        empty = empty_positions_df()
+        e = "Could not validate any tickers."
+        return None, e, "Universe invalid.", empty, empty_suggestion_df(), empty_suggestion_df(), empty_suggestion_df(), empty_suggestion_df(), empty_suggestion_df(), empty_suggestion_df(), json.dumps({}), e
+
+    global UNIVERSE
+    UNIVERSE = list(sorted(set([s for s in symbols if s != MARKET_TICKER] + [MARKET_TICKER])))[:MAX_TICKERS]
+
+    df = df[df["ticker"].isin(symbols)].copy()
+    amounts = {r["ticker"]: float(r["amount_usd"]) for _, r in df.iterrows()}
+    gross = sum(abs(v) for v in amounts.values())
     if gross <= 1e-12:
-        return {}
-    return {k: v / gross for k, v in wmap.items()}
+        empty = empty_positions_df()
+        e = "All amounts are zero."
+        return None, e, "Universe ok.", empty, *(empty_suggestion_df() for _ in range(6)), json.dumps({}), e
 
-def compute(lookback_years: int,
-            table_input,
-            risk_bucket: str,
-            horizon_years: float):
+    weights = {k: v / gross for k, v in amounts.items()}
+    rf_ann = RF_ANN
 
-    try:
-        # --- coerce incoming table to DataFrame (Gradio 5 may pass list-like) ---
-        if table_input is None:
-            df = pd.DataFrame(columns=["ticker", "amount_usd"])
-        elif isinstance(table_input, pd.DataFrame):
-            df = table_input.copy()
-        else:
-            df = pd.DataFrame(table_input, columns=["ticker", "amount_usd"])
-
-        df = df.dropna(how="all")
-        if df.empty:
-            return (None, "Add at least one ticker", "", pd.DataFrame(columns=POS_COLS),
-                    pd.DataFrame(columns=SUG_COLS), pd.DataFrame(columns=SUG_COLS),
-                    pd.DataFrame(columns=SUG_COLS), pd.DataFrame(columns=EFF_COLS),
-                    pd.DataFrame(columns=EFF_COLS), json.dumps([]), 1, "No suggestions yet.")
-
-        # --- sanitize
-        df["ticker"] = df["ticker"].astype(str).str.upper().str.strip()
-        df["amount_usd"] = pd.to_numeric(df["amount_usd"], errors="coerce").fillna(0.0)
-
-        symbols = [t for t in df["ticker"].tolist() if t]
-        symbols = validate_tickers(symbols, int(lookback_years))
-        if len(symbols) == 0:
-            return (None, "Could not validate any tickers", "Universe invalid",
-                    pd.DataFrame(columns=POS_COLS),
-                    pd.DataFrame(columns=SUG_COLS), pd.DataFrame(columns=SUG_COLS),
-                    pd.DataFrame(columns=SUG_COLS), pd.DataFrame(columns=EFF_COLS),
-                    pd.DataFrame(columns=EFF_COLS), json.dumps([]), 1, "No suggestions.")
-
-        # --- universe & amounts
-        universe = sorted(set([s for s in symbols if s != MARKET_TICKER] + [MARKET_TICKER]))
-        df = df[df["ticker"].isin(symbols)].copy()
-        amounts = {r["ticker"]: float(r["amount_usd"]) for _, r in df.iterrows()}
-        gross_amt = sum(abs(v) for v in amounts.values())
-        if gross_amt <= 1e-9:
-            return (None, "All amounts are zero", "Universe ok", pd.DataFrame(columns=POS_COLS),
-                    pd.DataFrame(columns=SUG_COLS), pd.DataFrame(columns=SUG_COLS),
-                    pd.DataFrame(columns=SUG_COLS), pd.DataFrame(columns=EFF_COLS),
-                    pd.DataFrame(columns=EFF_COLS), json.dumps([]), 1, "No suggestions.")
-
-        weights = {k: v / gross_amt for k, v in amounts.items()}
-
-        # --- risk free & moments
-        rf_code = fred_series_for_horizon(horizon_years)
-        rf_ann  = fetch_fred_yield_annual(rf_code)
-        moms    = estimate_all_moments_aligned(universe, int(lookback_years), rf_ann)
-        betas, covA, erp_ann, sigma_mkt = moms["betas"], moms["cov_ann"], moms["erp_ann"], moms["sigma_m_ann"]
-
-        # --- portfolio stats (CAPM return + historical sigma)
-        beta_p, er_capm_p, sigma_p = portfolio_stats(weights, covA, betas, rf_ann, erp_ann)
-
-        # --- efficient alternatives on CML
-        a_sigma, b_sigma, mu_eff_sigma = efficient_same_sigma(sigma_p, rf_ann, erp_ann, sigma_mkt)
-        a_mu, b_mu, sigma_eff_mu       = efficient_same_return(er_capm_p, rf_ann, erp_ann, sigma_mkt)
-
-        eff_same_sigma_tbl = _table_from_weights({MARKET_TICKER: a_sigma, BILLS_TICKER: b_sigma}, gross_amt)
-        eff_same_mu_tbl    = _table_from_weights({MARKET_TICKER: a_mu,    BILLS_TICKER: b_mu},   gross_amt)
-
-        # --- build synthetic dataset (based ONLY on this universe)
-        synth = build_synth_dataset(universe, covA, betas, rf_ann, erp_ann, n_rows=N_SYNTH, seed=777)
-
-        # --- risk buckets by sigma (absolute percentage points around median)
-        median_sigma = float(synth["sigma"].median()) if len(synth) else sigma_p
-        low_max  = max(float(synth["sigma"].min()), median_sigma - 0.05)   # 5% below median
-        high_min = median_sigma + 0.05
-
-        if risk_bucket == "Low":
-            cand_df = synth[synth["sigma"] <= low_max].copy()
-        elif risk_bucket == "High":
-            cand_df = synth[synth["sigma"] >= high_min].copy()
-        else:  # Medium
-            cand_df = synth[(synth["sigma"] > low_max) & (synth["sigma"] < high_min)].copy()
-
-        if len(cand_df) == 0:
-            cand_df = synth.copy()
-
-        # --- embed all candidates + query, and pick 3 via MMR for diversity
-        embed = get_embedder()
-        cand_sentences = cand_df.apply(row_to_sentence, axis=1).tolist()
-
-        cur_pairs = ", ".join([f"{k}:{v:+.2f}" for k, v in sorted(weights.items())])
-        q_sentence = f"user portfolio ({risk_bucket} risk); capm_target {er_capm_p:.4f}; sigma_hist {sigma_p:.4f}; exposures {cur_pairs}"
-
-        cand_embs = embed.encode(cand_sentences, convert_to_tensor=True, normalize_embeddings=True, batch_size=64, show_progress_bar=False)
-        q_emb     = embed.encode([q_sentence], convert_to_tensor=True, normalize_embeddings=True)[0]
-
-        sims = st_util.cos_sim(q_emb, cand_embs)[0]
-        top_idx = sims.topk(k=min(MMR_K, len(cand_df))).indices.cpu().numpy().tolist()
-        shortlist_embs = cand_embs[top_idx]
-        mmr_local = mmr_select(q_emb, shortlist_embs, k=3, lambda_param=MMR_LAMBDA)
-        chosen = [top_idx[i] for i in mmr_local]
-        recs = cand_df.iloc[chosen].reset_index(drop=True)
-
-        # --- suggestion tables for 3 picks
-        suggs = []
-        for _, r in recs.iterrows():
-            wmap = _weights_dict_from_row(r)
-            suggs.append({
-                "weights": wmap,
-                "er_capm": float(r["er_capm"]),
-                "sigma": float(r["sigma"]),
-                "beta": float(r["beta"]),
-                "table": _table_from_weights(wmap, gross_amt)
-            })
-
-        # --- plot
-        img = plot_cml(
-            rf_ann, erp_ann, sigma_mkt,
-            sigma_p, er_capm_p,
-            same_sigma_sigma=sigma_p, same_sigma_mu=mu_eff_sigma,
-            same_mu_sigma=sigma_eff_mu, same_mu_mu=er_capm_p
-        )
-
-        # --- positions table (computed)
-        rows = []
-        for t in universe:
-            if t == MARKET_TICKER:
-                continue
-            rows.append({
-                "ticker": t,
-                "amount_usd": round(amounts.get(t, 0.0), 2),
-                "weight_exposure": round(weights.get(t, 0.0), 6),
-                "beta": round(betas.get(t, np.nan), 4) if t != MARKET_TICKER else 1.0
-            })
-        pos_table = pd.DataFrame(rows, columns=POS_COLS)
-
-        # --- info summary
-        info_lines = []
-        info_lines.append("### Inputs")
-        info_lines.append(f"- Lookback years **{int(lookback_years)}**")
-        info_lines.append(f"- Horizon years **{int(round(horizon_years))}**")
-        info_lines.append(f"- Risk-free **{fmt_pct(rf_ann)}** from **{rf_code}**")
-        info_lines.append(f"- Market ERP **{fmt_pct(erp_ann)}**")
-        info_lines.append(f"- Market σ **{fmt_pct(sigma_mkt)}**")
-        info_lines.append("")
-        info_lines.append("### Your portfolio (plotted as CAPM return, historical σ)")
-        info_lines.append(f"- Beta **{beta_p:.2f}**")
-        info_lines.append(f"- σ (historical) **{fmt_pct(sigma_p)}**")
-        info_lines.append(f"- E[return] (CAPM / SML) **{fmt_pct(er_capm_p)}**")
-        info_lines.append("")
-        info_lines.append("### Efficient alternatives on CML")
-        info_lines.append(f"- Same σ → Market **{a_sigma:.2f}**, Bills **{b_sigma:.2f}**, Return **{fmt_pct(mu_eff_sigma)}**")
-        info_lines.append(f"- Same μ → Market **{a_mu:.2f}**, Bills **{b_mu:.2f}**, σ **{fmt_pct(sigma_eff_mu)}**")
-        info_lines.append("")
-        info_lines.append(f"### Dataset-based suggestions (risk: **{risk_bucket}**)")
-        info_lines.append("Use the selector to flip between **Pick #1 / #2 / #3**. Table shows % exposure and $ amounts.")
-
-        current_idx = 1
-        current = suggs[current_idx - 1] if suggs else None
-        current_tbl = current["table"] if current else pd.DataFrame(columns=SUG_COLS)
-        current_msg = ("Pick #1 — "
-                       f"E[μ] {fmt_pct(current['er_capm'])}, σ {fmt_pct(current['sigma'])}, β {current['beta']:.2f}"
-                       ) if current else "No suggestion."
-
-        return (img,
-                "\n".join(info_lines),
-                f"Universe set to {', '.join(universe)}",
-                pos_table,
-                suggs[0]["table"] if len(suggs) >= 1 else pd.DataFrame(columns=SUG_COLS),
-                suggs[1]["table"] if len(suggs) >= 2 else pd.DataFrame(columns=SUG_COLS),
-                suggs[2]["table"] if len(suggs) >= 3 else pd.DataFrame(columns=SUG_COLS),
-                eff_same_sigma_tbl,
-                eff_same_mu_tbl,
-                json.dumps([{
-                    "er_capm": s["er_capm"], "sigma": s["sigma"], "beta": s["beta"],
-                } for s in suggs]),
-                current_idx,
-                current_msg)
+    # Moments
+    moms = estimate_all_moments_aligned(symbols, years_lookback, rf_ann)
+    betas, covA, erp_ann, sigma_mkt = moms["betas"], moms["cov_ann"], moms["erp_ann"], moms["sigma_m_ann"]
 
-    except Exception as e:
-        msg = f"⚠️ Compute failed: {type(e).__name__}: {e}"
-        empty = pd.DataFrame()
-        return (None, msg, "Error", empty, empty, empty, empty, empty, empty, "[]", 1, msg)
+    # Portfolio CAPM stats (Y) vs historical σ (X)
+    beta_p, mu_capm, sigma_hist = portfolio_stats(weights, covA, betas, rf_ann, erp_ann)
+    sigma_capm = abs(beta_p) * sigma_mkt  # for info only
+
+    # Efficient alternatives on CML
+    a_sigma, b_sigma, mu_eff_sigma = efficient_same_sigma(sigma_hist, rf_ann, erp_ann, sigma_mkt)
+    a_mu, b_mu, sigma_eff_mu = efficient_same_return(mu_capm, rf_ann, erp_ann, sigma_mkt)
 
-def on_pick_change(idx: int, meta_json: str):
+    # Dataset (1,000 mixes) and save CSV
+    synth = build_synthetic_dataset(UNIVERSE, covA, betas, rf_ann, erp_ann, sigma_mkt, n_rows=SYNTH_ROWS)
+    csv_path = os.path.join(DATA_DIR, f"investor_profiles_{int(time.time())}.csv")
     try:
-        data = json.loads(meta_json)
+        synth.to_csv(csv_path, index=False)
     except Exception:
-        data = []
-    if not data:
-        return "No suggestion."
-    i = int(idx) - 1
-    i = max(0, min(i, len(data)-1))
-    s = data[i]
-    return f"Pick #{i+1} — E[μ] {fmt_pct(s['er_capm'])}, σ {fmt_pct(s['sigma'])}, β {s['beta']:.2f}"
-
-# =========================
-# UI
-# =========================
-with gr.Blocks(title="Efficient Portfolio Advisor", css="""
-#small-note {font-size: 12px; color:#666;}
-""") as demo:
-
-    gr.Markdown("## Efficient Portfolio Advisor\n"
-                "Search symbols, enter **$ amounts**, set your **horizon**. "
-                "The plot shows your **CAPM expected return** vs **historical σ**, alongside the **CML**. "
-                "Recommendations are generated from a **synthetic dataset (1000 portfolios)** and ranked with **local embeddings (BGE-base)** for relevance + diversity.")
-
-    with gr.Tab("Build Portfolio"):
+        csv_path = None
+
+    # Picks per band (Low/Medium/High)
+    meta_low,  tbls_low  = pick_3_for_band(synth, "Low",    sigma_mkt, UNIVERSE, weights)
+    meta_med,  tbls_med  = pick_3_for_band(synth, "Medium", sigma_mkt, UNIVERSE, weights)
+    meta_high, tbls_high = pick_3_for_band(synth, "High",   sigma_mkt, UNIVERSE, weights)
+
+    # fallbacks if any band empty
+    def ensure_three(meta, tbls):
+        while len(meta) < 3:
+            meta.append({"mu": mu_capm, "sigma": sigma_capm})
+            tbls.append(empty_suggestion_df())
+        return meta[:3], tbls[:3]
+
+    meta_low,  tbls_low  = ensure_three(meta_low,  tbls_low)
+    meta_med,  tbls_med  = ensure_three(meta_med,  tbls_med)
+    meta_high, tbls_high = ensure_three(meta_high, tbls_high)
+
+    # clamp pick indices to 1..3
+    pick_low  = int(max(1, min(3, pick_low or 1)))
+    pick_med  = int(max(1, min(3, pick_med or 1)))
+    pick_high = int(max(1, min(3, pick_high or 1)))
+
+    # default highlighted suggestion: Medium / chosen index
+    sel = meta_med[pick_med-1]
+    img = plot_cml(rf_ann, erp_ann, sigma_mkt, sigma_hist, mu_capm, sel["mu"], sel["sigma"])
+
+    # positions table (computed)
+    pos_table = pd.DataFrame(
+        [{
+            "ticker": t,
+            "amount_usd": amounts.get(t, 0.0),
+            "weight_exposure": weights.get(t, 0.0),
+            "beta": 1.0 if t == MARKET_TICKER else betas.get(t, np.nan)
+        } for t in symbols],
+        columns=["ticker", "amount_usd", "weight_exposure", "beta"]
+    )
+
+    # efficient mixes tables (display-only)
+    eff_same_sigma_tbl = pd.DataFrame([
+        {"ticker": MARKET_TICKER, "weight_%": round(a_sigma*100,2), "amount_$": round(a_sigma*gross,2)},
+        {"ticker": BILLS_LABEL,   "weight_%": round(b_sigma*100,2), "amount_$": round(b_sigma*gross,2)},
+    ])
+    eff_same_mu_tbl = pd.DataFrame([
+        {"ticker": MARKET_TICKER, "weight_%": round(a_mu*100,2), "amount_$": round(a_mu*gross,2)},
+        {"ticker": BILLS_LABEL,   "weight_%": round(b_mu*100,2), "amount_$": round(b_mu*gross,2)},
+    ])
+
+    # info summary
+    info = "\n".join([
+        "### Inputs",
+        f"- Lookback years {years_lookback}",
+        f"- Horizon years {int(round(HORIZON_YEARS))}",
+        f"- Risk-free {rf_ann:.2%} from {RF_CODE}",
+        f"- Market ERP {erp_ann:.2%}",
+        f"- Market σ {sigma_mkt:.2%}",
+        "",
+        "### Your portfolio (CAPM on CML plot)",
+        f"- Beta {beta_p:.2f}",
+        f"- Expected return (CAPM / SML) {mu_capm:.2%}",
+        f"- σ (historical) {sigma_hist:.2%}",
+        "",
+        "### Efficient alternatives on CML",
+        f"- Same σ: Market {a_sigma:.2f}, Bills {b_sigma:.2f}, E[r] {mu_eff_sigma:.2%}",
+        f"- Same μ: Market {a_mu:.2f}, Bills {b_mu:.2f}, σ {sigma_eff_mu:.2%}",
+        "",
+        "### Suggestions",
+        "Three tabs (Low/Medium/High). Select a pick to highlight it on the plot.",
+        "_Plot is **always** CAPM E[r] vs historical σ; your CAPM point will never exceed the CML._"
+    ])
+
+    # pack suggestion meta for quick plot refresh on band selection
+    meta = {
+        "low":   meta_low,
+        "med":   meta_med,
+        "high":  meta_high,
+        "plot":  {"rf": rf_ann, "erp": erp_ann, "sigma_mkt": sigma_mkt, "sigma_hist": sigma_hist, "mu_capm": mu_capm}
+    }
+
+    uni_msg = f"Universe set to: {', '.join(UNIVERSE)}"
+
+    # outputs:
+    # plot, summary, universe, positions,
+    # low tables (3), medium tables (3), high tables (3),
+    # efficient tables (same σ, same μ),
+    # meta json, status
+    return (
+        img, info, uni_msg, pos_table,
+        tbls_low[0], tbls_low[1], tbls_low[2],
+        tbls_med[0], tbls_med[1], tbls_med[2],
+        tbls_high[0], tbls_high[1], tbls_high[2],
+        eff_same_sigma_tbl, eff_same_mu_tbl,
+        json.dumps(meta), (csv_path or "")
+    )
+
+def highlight_from_pick(meta_json: str, band: str, pick_idx: int):
+    try:
+        meta = json.loads(meta_json)
+        plotp = meta.get("plot", {})
+        rf = float(plotp["rf"]); erp = float(plotp["erp"]); sigma_mkt = float(plotp["sigma_mkt"])
+        sigma_hist = float(plotp["sigma_hist"]); mu_capm = float(plotp["mu_capm"])
+        arr = meta["low" if band=="Low" else "med" if band=="Medium" else "high"]
+        i = int(max(1, min(3, pick_idx or 1))) - 1
+        sel = arr[i]
+        return plot_cml(rf, erp, sigma_mkt, sigma_hist, mu_capm, sel["mu"], sel["sigma"])
+    except Exception as e:
+        # if anything fails, fall back to no suggestion highlighted
+        return None
+
+# -------------- UI --------------
+def clamp13(i: int): return int(max(1, min(3, int(i or 1))))
+
+with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
+    gr.Markdown(
+        "## Efficient Portfolio Advisor\n"
+        "Search symbols, enter **dollar amounts**, set horizon. Data uses Yahoo monthly prices; risk-free from FRED.\n\n"
+        "**Plot:** CAPM E[r] vs historical σ on the **CML**.\n"
+        "**Efficient mixes:** CML portfolio with **same σ** and CML portfolio with **same E[r]** as yours.\n"
+        "**Suggestions:** 1,000 long-only mixes from your universe → 3 picks per risk band using exposure+embeddings with MMR diversity."
+    )
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            q = gr.Textbox(label="Search symbol")
+            search_note = gr.Markdown()
+            matches = gr.Dropdown(choices=[], label="Matches")
+            search_btn = gr.Button("Search")
+            add_btn = gr.Button("Add selected to portfolio")
+
+            gr.Markdown("### Portfolio positions (enter $ amounts; negatives allowed for your input)")
+            table = gr.Dataframe(
+                headers=["ticker", "amount_usd"],
+                datatype=["str", "number"],
+                row_count=0,
+                col_count=(2, "fixed"),
+                type="pandas"   # Gradio 5-friendly
+            )
+
+            horizon = gr.Number(label="Horizon in years (1–100)", value=HORIZON_YEARS, precision=0)
+            lookback = gr.Slider(1, 15, value=DEFAULT_LOOKBACK_YEARS, step=1, label="Lookback years for betas & covariances")
+
+            run_btn = gr.Button("Compute (build dataset & suggest)")
+        with gr.Column(scale=1):
+            plot = gr.Image(label="Capital Market Line (CAPM)", type="pil")
+            summary = gr.Markdown(label="Inputs & Results")
+            universe_msg = gr.Textbox(label="Universe status / Horizon", interactive=False)
+
+            positions = gr.Dataframe(
+                label="Computed positions",
+                headers=["ticker", "amount_usd", "weight_exposure", "beta"],
+                datatype=["str", "number", "number", "number"],
+                col_count=(4, "fixed"),
+                value=empty_positions_df(),
+                interactive=False,
+                type="pandas"
+            )
+
+    # Suggestions area: three tabs, each 3 picks
+    meta_box = gr.Textbox(value="{}", visible=False, label="meta")
+    csv_path = gr.File(label="Generated dataset CSV", value=None, visible=True)
+
+    with gr.Tab("Low"):
         with gr.Row():
-            with gr.Column(scale=1):
-                q = gr.Textbox(label="Search symbol")
-                search_note = gr.Markdown(elem_id="small-note")
-                matches = gr.Dropdown(choices=[], label="Matches", value=None)
-                search_btn = gr.Button("Search")
-                add_btn = gr.Button("Add selected to portfolio")
-
-                gr.Markdown("### Positions (enter dollars; negatives allowed for shorts)")
-                table = gr.Dataframe(
-                    headers=["ticker", "amount_usd"],
-                    datatype=["str", "number"],
-                    row_count=0,
-                    col_count=(2, "fixed"),
-                    wrap=True,
-                    type="pandas"    # important for Gradio 5
-                )
-
-                # Handy sample
-                sample_btn = gr.Button("Load sample portfolio")
-
-            with gr.Column(scale=1):
-                horizon = gr.Slider(1, 30, value=DEFAULT_LOOKBACK_YEARS, step=1, label="Investment horizon (years)")
-                lookback = gr.Slider(1, 10, value=DEFAULT_LOOKBACK_YEARS, step=1, label="Lookback (years) for β and σ")
-                risk_bucket = gr.Radio(["Low", "Medium", "High"], value="Medium", label="Recommendation risk level")
-                run_btn = gr.Button("Compute")
-
-                rf_msg = gr.Textbox(label="Risk-free source / status", interactive=False)
-                search_btn.click(fn=do_search, inputs=q, outputs=[search_note, matches])
-                add_btn.click(fn=add_symbol, inputs=[matches, table], outputs=[table, search_note])
-                table.change(fn=lock_ticker_column, inputs=table, outputs=table)
-                horizon.change(fn=set_horizon, inputs=horizon, outputs=[rf_msg, gr.State()])  # rf_msg + silent
-                sample_btn.click(lambda: pd.DataFrame({"ticker": ["AAPL","MSFT","VOO"], "amount_usd": [3000, 2000, 5000]}),
-                                 inputs=None, outputs=table)
-
-    with gr.Tab("Results"):
+            low1 = gr.Dataframe(label="Pick #1", interactive=False, type="pandas")
+            low2 = gr.Dataframe(label="Pick #2", interactive=False, type="pandas")
+            low3 = gr.Dataframe(label="Pick #3", interactive=False, type="pandas")
+        pick_low = gr.Slider(1, 3, value=1, step=1, label="Highlight pick")
+        low_btn = gr.Button("Show on plot")
+
+    with gr.Tab("Medium"):
+        with gr.Row():
+            med1 = gr.Dataframe(label="Pick #1", interactive=False, type="pandas")
+            med2 = gr.Dataframe(label="Pick #2", interactive=False, type="pandas")
+            med3 = gr.Dataframe(label="Pick #3", interactive=False, type="pandas")
+        pick_med = gr.Slider(1, 3, value=1, step=1, label="Highlight pick")
+        med_btn = gr.Button("Show on plot")
+
+    with gr.Tab("High"):
         with gr.Row():
-            with gr.Column(scale=1):
-                plot = gr.Image(label="Capital Market Line", type="pil")
-                summary = gr.Markdown()
-                universe_msg = gr.Textbox(label="Universe status", interactive=False)
-
-            with gr.Column(scale=1):
-                positions = gr.Dataframe(
-                    label="Computed positions",
-                    headers=POS_COLS,
-                    datatype=["str", "number", "number", "number"],
-                    col_count=(len(POS_COLS), "fixed"),
-                    interactive=False,
-                    type="pandas"
-                )
-
-                gr.Markdown("### Recommendations (always from embeddings)")
-                with gr.Row():
-                    sugg1 = gr.Dataframe(label="Pick #1", interactive=False, type="pandas")
-                    sugg2 = gr.Dataframe(label="Pick #2", interactive=False, type="pandas")
-                    sugg3 = gr.Dataframe(label="Pick #3", interactive=False, type="pandas")
-
-                with gr.Row():
-                    pick_idx = gr.Slider(1, 3, value=1, step=1, label="Carousel: show Pick #")
-                    pick_meta = gr.Textbox(value="[]", visible=False)
-                pick_msg = gr.Markdown("")
-
-                gr.Markdown("### Efficient alternatives on the CML")
-                eff_same_sigma_tbl = gr.Dataframe(label="Efficient: Same σ", interactive=False, type="pandas")
-                eff_same_mu_tbl    = gr.Dataframe(label="Efficient: Same μ", interactive=False, type="pandas")
-
-        run_btn.click(
-            fn=compute,
-            inputs=[lookback, table, risk_bucket, horizon],
-            outputs=[
-                plot, summary, universe_msg, positions,
-                sugg1, sugg2, sugg3,
-                eff_same_sigma_tbl, eff_same_mu_tbl,
-                pick_meta, pick_idx, pick_msg
-            ]
-        )
-        pick_idx.change(fn=on_pick_change, inputs=[pick_idx, pick_meta], outputs=pick_msg)
-
-    with gr.Tab("About"):
-        gr.Markdown(
-            "### Modality & Model\n"
-            "- **Modality**: Text (portfolio → text descriptions) powering **embeddings**\n"
-            "- **Embedding model**: `BAAI/bge-base-en-v1.5` (local, downloaded once; no API)\n\n"
-            "### Use case\n"
-            "Given a portfolio, we build a synthetic dataset of 1,000 alternative mixes **using the same tickers**, "
-            "compute each mix’s **CAPM return, σ, and β**, and rank candidates with embeddings to return **3 diverse, relevant suggestions** "
-            "for **Low / Medium / High** risk.\n\n"
-            "### Theory links\n"
-            "- Portfolio expected return in the plot uses **CAPM (SML)**, while σ is historical.\n"
-            "- The **CML** and the two **efficient alternatives** (same σ, same μ) use a mix of **Market (VOO)** and **Bills**."
-        )
+            high1 = gr.Dataframe(label="Pick #1", interactive=False, type="pandas")
+            high2 = gr.Dataframe(label="Pick #2", interactive=False, type="pandas")
+            high3 = gr.Dataframe(label="Pick #3", interactive=False, type="pandas")
+        pick_high = gr.Slider(1, 3, value=1, step=1, label="Highlight pick")
+        high_btn = gr.Button("Show on plot")
+
+    gr.Markdown("### Efficient alternatives on the CML")
+    eff_same_sigma_tbl = gr.Dataframe(label="Efficient: Same σ", interactive=False, type="pandas")
+    eff_same_mu_tbl    = gr.Dataframe(label="Efficient: Same μ", interactive=False, type="pandas")
+
+    # wire search / add / locking / horizon
+    search_btn.click(fn=search_tickers_cb, inputs=q, outputs=[search_note, matches])
+    add_btn.click(fn=add_symbol, inputs=[matches, table], outputs=[table, search_note])
+    table.change(fn=lock_ticker_column, inputs=table, outputs=table)
+    horizon.change(fn=set_horizon, inputs=horizon, outputs=universe_msg)
+
+    # main compute
+    run_btn.click(
+        fn=compute,
+        inputs=[lookback, table, gr.State(1), gr.State(1), gr.State(1)],
+        outputs=[
+            plot, summary, universe_msg, positions,
+            low1, low2, low3,
+            med1, med2, med3,
+            high1, high2, high3,
+            eff_same_sigma_tbl, eff_same_mu_tbl,
+            meta_box, csv_path
+        ]
+    )
+
+    # highlight buttons refresh plot with selected suggestion
+    low_btn.click(fn=highlight_from_pick, inputs=[meta_box, gr.State("Low"),  pick_low],  outputs=plot)
+    med_btn.click(fn=highlight_from_pick, inputs=[meta_box, gr.State("Medium"), pick_med], outputs=plot)
+    high_btn.click(fn=highlight_from_pick, inputs=[meta_box, gr.State("High"), pick_high], outputs=plot)
+
+# initialize risk-free at launch
+RF_CODE = fred_series_for_horizon(HORIZON_YEARS)
+RF_ANN = fetch_fred_yield_annual(RF_CODE)
 
 if __name__ == "__main__":
-    # On HF Spaces you don't need share=True; binding to 0.0.0.0 is enough.
+    # On Hugging Face Spaces you don't need share=True; binding to 0.0.0.0 is fine
     demo.launch(server_name="0.0.0.0", server_port=7860)