Update app.py

app.py

@@ -1,7 +1,6 @@
 # app.py
-# Efficient Portfolio Advisor — with dataset-based Low/Medium/High suggestions
-# Modality: Text. Models: yfinance (prices), FRED (risk-free), simple CAPM math,
-# optional reranking with sentence-transformers "FinLang/finance-embeddings-investopedia".
+# Efficient Portfolio Advisor — CML-consistent plotting + suggestion picker
+# Modality: Text. Optional reranking model: FinLang/finance-embeddings-investopedia
 
 import os
 import io
@@ -16,50 +15,31 @@ import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
 from PIL import Image
-import requests
 import gradio as gr
+import requests
 import yfinance as yf
 
-# Optional (lazy) import for embeddings
-_ST_MODEL = None
-
-# ---------- Config ----------
-DATA_DIR = "data"
-os.makedirs(DATA_DIR, exist_ok=True)
+_ST_MODEL = None  # lazy load for embeddings
 
+# ---------------- Config ----------------
+DATA_DIR = "data"; os.makedirs(DATA_DIR, exist_ok=True)
 MARKET_TICKER = "VOO"
 MAX_TICKERS = 30
 DEFAULT_LOOKBACK_YEARS = 10
 DATASET_ROWS = 1000
 
-# FRED mappings by horizon
 FRED_MAP = [
-    (1,  "DGS1"),
-    (2,  "DGS2"),
-    (3,  "DGS3"),
-    (5,  "DGS5"),
-    (7,  "DGS7"),
-    (10, "DGS10"),
-    (20, "DGS20"),
-    (30, "DGS30"),
-    (100, "DGS30"),
+    (1,  "DGS1"), (2,  "DGS2"), (3,  "DGS3"), (5,  "DGS5"),
+    (7,  "DGS7"), (10, "DGS10"), (20, "DGS20"), (30, "DGS30"), (100, "DGS30")
 ]
 
 POS_COLS = ["ticker", "amount_usd", "weight_exposure", "beta"]
 SUG_COLS_HOLD = ["pick", "ticker", "weight_%", "amount_$"]
 
-# ---------- Small helpers ----------
+# ---------------- Small helpers ----------------
 def fmt_pct(x: float, dec: int = 2) -> str:
-    try:
-        return f"{x*100:.{dec}f}%"
-    except Exception:
-        return "—"
-
-def fmt_usd(x: float) -> str:
-    try:
-        return f"${x:,.2f}"
-    except Exception:
-        return "—"
+    try: return f"{x*100:.{dec}f}%"
+    except: return "—"
 
 def ensure_dir(p: str):
     os.makedirs(os.path.dirname(p), exist_ok=True)
@@ -67,60 +47,45 @@ def ensure_dir(p: str):
 def fred_series_for_horizon(years: float) -> str:
     y = max(1.0, min(100.0, float(years)))
     for cutoff, code in FRED_MAP:
-        if y <= cutoff:
-            return code
+        if y <= cutoff: return code
     return "DGS30"
 
 def fetch_fred_yield_annual(code: str) -> float:
     url = f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={code}"
     try:
-        r = requests.get(url, timeout=10)
-        r.raise_for_status()
+        r = requests.get(url, timeout=10); r.raise_for_status()
         df = pd.read_csv(io.StringIO(r.text))
         s = pd.to_numeric(df.iloc[:, 1], errors="coerce").dropna()
         return float(s.iloc[-1] / 100.0) if len(s) else 0.03
-    except Exception:
-        return 0.03
+    except: return 0.03
 
-# ---------- Prices & returns (fix for 'Close' KeyError) ----------
+# ---------------- Prices & returns ----------------
 def fetch_prices_monthly(tickers: List[str], years: int) -> pd.DataFrame:
     start = pd.Timestamp.today(tz="UTC") - pd.DateOffset(years=years, days=7)
     end = pd.Timestamp.today(tz="UTC")
-
     raw = yf.download(
         list(dict.fromkeys(tickers)),
-        start=start.date(),
-        end=end.date(),
-        interval="1mo",
-        auto_adjust=False,         # prefer 'Adj Close' if present
-        progress=False,
-        group_by="ticker",
-        threads=False,
+        start=start.date(), end=end.date(),
+        interval="1mo", auto_adjust=False, progress=False,
+        group_by="ticker", threads=False
     )
     if raw is None or len(raw) == 0:
         return pd.DataFrame()
 
-    # MultiIndex (ticker, field) vs single-index
     if isinstance(raw.columns, pd.MultiIndex):
         price = None
         for field in ("Adj Close", "Close"):
             if field in raw.columns.get_level_values(-1):
-                price = raw.xs(field, axis=1, level=-1, drop_level=True)
-                break
+                price = raw.xs(field, axis=1, level=-1, drop_level=True); break
         if price is None:
             price = raw.copy()
             price.columns = [c[0] if isinstance(c, tuple) else c for c in price.columns]
     else:
-        if "Adj Close" in raw.columns:
-            price = raw["Adj Close"]
-        elif "Close" in raw.columns:
-            price = raw["Close"]
-        else:
-            price = raw
-
-    if isinstance(price, pd.Series):
-        price = price.to_frame()
+        if "Adj Close" in raw.columns: price = raw["Adj Close"]
+        elif "Close" in raw.columns:   price = raw["Close"]
+        else:                          price = raw
 
+    if isinstance(price, pd.Series): price = price.to_frame()
     price = price.dropna(how="all").fillna(method="ffill")
     price = price.loc[:, ~pd.Index(price.columns).duplicated()]
     return price
@@ -128,23 +93,18 @@ def fetch_prices_monthly(tickers: List[str], years: int) -> pd.DataFrame:
 def monthly_returns(prices: pd.DataFrame) -> pd.DataFrame:
     return prices.pct_change().dropna()
 
-def annualize_mean(m):
-    return np.asarray(m, dtype=float) * 12.0
-
-def annualize_sigma(s):
-    return np.asarray(s, dtype=float) * math.sqrt(12.0)
+def annualize_mean(m):  return np.asarray(m, dtype=float) * 12.0
+def annualize_sigma(s): return np.asarray(s, dtype=float) * math.sqrt(12.0)
 
-# ---------- Search & validation ----------
+# ---------------- Search & validation ----------------
 def yahoo_search(query: str):
-    if not query or not query.strip():
-        return []
+    if not query or not query.strip(): return []
     url = "https://query1.finance.yahoo.com/v1/finance/search"
     params = {"q": query.strip(), "quotesCount": 10, "newsCount": 0}
     headers = {"User-Agent": "Mozilla/5.0"}
     try:
         r = requests.get(url, params=params, headers=headers, timeout=10)
-        r.raise_for_status()
-        data = r.json()
+        r.raise_for_status(); data = r.json()
         out = []
         for q in data.get("quotes", []):
             sym = q.get("symbol")
@@ -155,55 +115,56 @@ def yahoo_search(query: str):
         if not out:
             out = [{"symbol": query.strip().upper(), "name": "typed symbol", "exchange": "—"}]
         return out[:10]
-    except Exception:
+    except:
         return [{"symbol": query.strip().upper(), "name": "typed symbol", "exchange": "—"}]
 
 def validate_tickers(symbols: List[str], years: int) -> List[str]:
-    # include market to keep alignment, but validate only user symbols
     base = list(dict.fromkeys([s.strip().upper() for s in symbols if s.strip()]))[:MAX_TICKERS]
     px = fetch_prices_monthly(base + [MARKET_TICKER], years)
     ok = [s for s in base if s in px.columns]
     return ok
 
-# ---------- Aligned CAPM moments ----------
+# ---------------- Aligned CAPM moments (now includes MARKET in cov & μ) ----------------
 def get_aligned_monthly_returns(symbols: List[str], years: int) -> pd.DataFrame:
-    uniq = [c for c in dict.fromkeys(symbols) if c != MARKET_TICKER]
-    tickers = uniq + [MARKET_TICKER]
-    px = fetch_prices_monthly(tickers, years)
+    uniq = [c for c in dict.fromkeys(symbols)]
+    px = fetch_prices_monthly(uniq, years)
     rets = monthly_returns(px)
-    cols = [c for c in uniq if c in rets.columns] + ([MARKET_TICKER] if MARKET_TICKER in rets.columns else [])
+    cols = [c for c in uniq if c in rets.columns]
     R = rets[cols].dropna(how="any")
     return R.loc[:, ~R.columns.duplicated()]
 
 def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
-    R = get_aligned_monthly_returns(symbols, years)
+    R = get_aligned_monthly_returns(symbols + [MARKET_TICKER], years)
     if MARKET_TICKER not in R.columns or R.shape[0] < 3:
         raise ValueError("Not enough aligned data to estimate moments.")
     rf_m = rf_ann / 12.0
 
-    m = R[MARKET_TICKER]
-    if isinstance(m, pd.DataFrame):
-        m = m.iloc[:, 0].squeeze()
-
-    mu_m_ann = float(annualize_mean(m.mean()))
-    sigma_m_ann = float(annualize_sigma(m.std(ddof=1)))
+    # Means
+    mu_m = R[MARKET_TICKER];  mu_m_ann = float(annualize_mean(mu_m.mean()))
+    mu_all_ann = annualize_mean(R.mean(axis=0))  # pandas Series across all cols
+    sigma_m_ann = float(annualize_sigma(mu_m.std(ddof=1)))
     erp_ann = float(mu_m_ann - rf_ann)
 
-    ex_m = m - rf_m
-    var_m = float(np.var(ex_m.values, ddof=1))
-    var_m = max(var_m, 1e-6)
-
+    # Betas vs market
+    ex_m = mu_m - rf_m
+    var_m = float(np.var(ex_m.values, ddof=1)); var_m = max(var_m, 1e-6)
     betas: Dict[str, float] = {}
-    for s in [c for c in R.columns if c != MARKET_TICKER]:
+    for s in R.columns:
         ex_s = R[s] - rf_m
         betas[s] = float(np.cov(ex_s.values, ex_m.values, ddof=1)[0, 1] / var_m)
     betas[MARKET_TICKER] = 1.0
 
-    asset_cols = [c for c in R.columns if c != MARKET_TICKER]
-    cov_m = np.cov(R[asset_cols].values.T, ddof=1) if asset_cols else np.zeros((0, 0))
-    covA = pd.DataFrame(cov_m * 12.0, index=asset_cols, columns=asset_cols)
+    # Covariance includes MARKET_TICKER too
+    cov_m = np.cov(R.values.T, ddof=1)
+    covA = pd.DataFrame(cov_m * 12.0, index=R.columns, columns=R.columns)
 
-    return {"betas": betas, "cov_ann": covA, "erp_ann": erp_ann, "sigma_m_ann": sigma_m_ann}
+    return {
+        "betas": betas,
+        "cov_ann": covA,
+        "erp_ann": erp_ann,
+        "sigma_m_ann": sigma_m_ann,
+        "mu_all_ann": pd.Series(mu_all_ann, index=R.columns)  # annualized means per asset incl. market
+    }
 
 def capm_er(beta: float, rf_ann: float, erp_ann: float) -> float:
     return float(rf_ann + beta * erp_ann)
@@ -214,99 +175,80 @@ def portfolio_stats(weights: Dict[str, float],
                     rf_ann: float,
                     erp_ann: float) -> Tuple[float, float, float]:
     tickers = list(weights.keys())
-    if len(tickers) == 0:
-        return 0.0, 0.0, 0.0
+    if len(tickers) == 0: return 0.0, 0.0, 0.0
     w = np.array([weights[t] for t in tickers], dtype=float)
-    gross = float(np.sum(np.abs(w)))
-    if gross <= 1e-12:
-        return 0.0, 0.0, 0.0
-    w_expo = w / gross
+    gross = float(np.sum(np.abs(w)));  w_expo = w / max(gross, 1e-12)
+
     beta_p = float(np.dot([betas.get(t, 0.0) for t in tickers], w_expo))
-    er_p = capm_er(beta_p, rf_ann, erp_ann)
+    er_p   = capm_er(beta_p, rf_ann, erp_ann)
+
     cov = cov_ann.reindex(index=tickers, columns=tickers).fillna(0.0).to_numpy()
     sigma_p = math.sqrt(float(max(w_expo.T @ cov @ w_expo, 0.0)))
     return beta_p, er_p, sigma_p
 
-# ---------- CML helpers (plot in %) ----------
+def portfolio_hist_return(weights: Dict[str, float], mu_all_ann: pd.Series) -> float:
+    tickers = list(weights.keys())
+    w = np.array([weights[t] for t in tickers], dtype=float)
+    gross = float(np.sum(np.abs(w)));  w_expo = w / max(gross, 1e-12)
+    mu = mu_all_ann.reindex(tickers).fillna(0.0).to_numpy()
+    return float(np.dot(mu, w_expo))
+
+# ---------------- CML plot (percent axes) ----------------
 def efficient_same_sigma(sigma_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
-    if sigma_mkt <= 1e-12:
-        return 0.0, 1.0, rf_ann
+    if sigma_mkt <= 1e-12: return 0.0, 1.0, rf_ann
     a = sigma_target / sigma_mkt
     return a, 1.0 - a, rf_ann + a * erp_ann
 
 def efficient_same_return(mu_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
-    if abs(erp_ann) <= 1e-12:
-        return 0.0, 1.0, rf_ann
+    if abs(erp_ann) <= 1e-12: return 0.0, 1.0, rf_ann
     a = (mu_target - rf_ann) / erp_ann
     return a, 1.0 - a, abs(a) * sigma_mkt
 
-def plot_cml_percent(
-    rf_ann, erp_ann, sigma_mkt,
-    pt_sigma, pt_mu,
-    same_sigma_sigma, same_sigma_mu,
-    same_mu_sigma, same_mu_mu,
-    suggestion_sigma=None, suggestion_mu=None
-) -> Image.Image:
-    fig = plt.figure(figsize=(6, 4), dpi=120)
-
-    xmax = max(
-        0.3,
-        sigma_mkt * 2.0,
-        pt_sigma * 1.4,
-        same_mu_sigma * 1.4,
-        same_sigma_sigma * 1.4,
-        (suggestion_sigma or 0.0) * 1.4,
-    )
+def plot_cml_percent(base, suggestion=None) -> Image.Image:
+    rf_ann = base["rf"]; erp = base["erp"]; sig_m = base["sigma_m"]
+    pt_s   = base["pt_sigma"]; pt_mu = base["pt_mu"]
+    sames_s_s = base["same_sigma_sigma"]; sames_s_mu = base["same_sigma_mu"]
+    same_mu_s = base["same_mu_sigma"];   same_mu_mu = base["same_mu_mu"]
+
+    fig = plt.figure(figsize=(6,4), dpi=120)
+    xmax = max(0.3, sig_m*2.0, pt_s*1.4, same_mu_s*1.4, sames_s_s*1.4, (suggestion["sigma"] if suggestion else 0.0)*1.4)
     xs = np.linspace(0, xmax, 160)
-    slope = erp_ann / max(sigma_mkt, 1e-12)
+    slope = erp / max(sig_m, 1e-12)
     cml = rf_ann + slope * xs
-    plt.plot(xs * 100, cml * 100, label="CML via Market")
-
-    # Points
-    plt.scatter([0.0], [rf_ann * 100], label="Risk-free (FRED)")
-    plt.scatter([sigma_mkt * 100], [(rf_ann + erp_ann) * 100], label="Market VOO")
-    plt.scatter([pt_sigma * 100], [pt_mu * 100], label="Your portfolio")
-    plt.scatter([same_sigma_sigma * 100], [same_sigma_mu * 100], label="Efficient same sigma")
-    plt.scatter([same_mu_sigma * 100], [same_mu_mu * 100], label="Efficient same return")
-    if suggestion_sigma is not None and suggestion_mu is not None:
-        plt.scatter([suggestion_sigma * 100], [suggestion_mu * 100], label="Suggestion")
-
-    # simple dotted guides
-    plt.plot([pt_sigma * 100, same_sigma_sigma * 100], [pt_mu * 100, same_sigma_mu * 100], linestyle="--", lw=1, alpha=0.7, color="gray")
-    plt.plot([pt_sigma * 100, same_mu_sigma * 100], [pt_mu * 100, same_mu_mu * 100], linestyle="--", lw=1, alpha=0.7, color="gray")
-
-    plt.xlabel("σ (annualized, %)")
-    plt.ylabel("Expected return (annual, %)")
-    plt.legend(loc="best", fontsize=8)
-    plt.tight_layout()
-
-    buf = io.BytesIO()
-    plt.savefig(buf, format="png")
-    plt.close(fig)
-    buf.seek(0)
+    plt.plot(xs*100, cml*100, label="CML via Market")
+
+    plt.scatter([0.0], [rf_ann*100], label="Risk-free (FRED)")
+    plt.scatter([sig_m*100], [(rf_ann+erp)*100], label="Market VOO")
+    plt.scatter([pt_s*100], [pt_mu*100], label="Your portfolio")
+    plt.scatter([sames_s_s*100], [sames_s_mu*100], label="Efficient same σ")
+    plt.scatter([same_mu_s*100], [same_mu_mu*100], label="Efficient same return")
+
+    if suggestion:
+        plt.scatter([suggestion["sigma"]*100], [suggestion["mu"]*100], label="Suggestion")
+
+    plt.xlabel("σ (annualized, %)"); plt.ylabel("Expected return (annual, %)")
+    plt.legend(loc="best", fontsize=8); plt.tight_layout()
+
+    buf = io.BytesIO(); plt.savefig(buf, format="png"); plt.close(fig); buf.seek(0)
     return Image.open(buf)
 
-# ---------- Synthetic dataset (built only from current universe) ----------
+# ---------------- Synthetic dataset (universe only) ----------------
 def _row_exposures(row: pd.Series, universe: List[str]) -> Optional[np.ndarray]:
     try:
         ts = [t.strip() for t in str(row["tickers"]).split(",")]
         ws = [float(x) for x in str(row["weights"]).split(",")]
         wmap = {t: ws[i] for i, t in enumerate(ts) if i < len(ws)}
         w = np.array([wmap.get(t, 0.0) for t in universe], dtype=float)
-        gross = float(np.sum(np.abs(w)))
-        if gross <= 1e-12:
-            return None
+        gross = float(np.sum(np.abs(w)));  
+        if gross <= 1e-12: return None
         return w / gross
-    except Exception:
-        return None
+    except: return None
 
 def build_synthetic_dataset(universe: List[str], years: int, rf_ann: float, erp_ann: float, n_rows: int = DATASET_ROWS) -> pd.DataFrame:
-    # require MARKET_TICKER present for moments; weights exclude it unless random pick includes
     moms = estimate_all_moments_aligned(universe, years, rf_ann)
     covA, betas = moms["cov_ann"], moms["betas"]
 
-    rng = np.random.default_rng(12345)
-    rows = []
+    rng = np.random.default_rng(12345); rows = []
     for i in range(n_rows):
         k = int(rng.integers(low=min(2, len(universe)), high=min(8, len(universe)) + 1))
         picks = list(rng.choice(universe, size=k, replace=False))
@@ -314,49 +256,34 @@ def build_synthetic_dataset(universe: List[str], years: int, rf_ann: float, erp_
         raw = rng.dirichlet(np.ones(k))
         gross = 1.0 + float(rng.gamma(2.0, 0.5))
         w = gross * signs * raw
-        # portfolio stats
         beta_p, er_p, sigma_p = portfolio_stats({picks[j]: w[j] for j in range(k)}, covA, betas, rf_ann, erp_ann)
         rows.append({
             "id": i,
             "tickers": ",".join(picks),
             "weights": ",".join(f"{x:.6f}" for x in w),
-            "er_p": er_p,
-            "sigma_p": sigma_p,
-            "beta_p": beta_p
+            "er_p": er_p, "sigma_p": sigma_p, "beta_p": beta_p
         })
     return pd.DataFrame(rows)
 
 def dataset_path_for_universe(universe: List[str]) -> str:
     key = ",".join(sorted(universe))
     h = abs(hash(key)) % (10**8)
-    p = os.path.join(DATA_DIR, f"investor_profiles_{h}.csv")
-    return p
+    return os.path.join(DATA_DIR, f"investor_profiles_{h}.csv")
 
-# ---------- Suggestion logic (Low / Medium / High) ----------
+# ---------------- Suggestions (build + picker) ----------------
 def _risk_targets(sigmas: np.ndarray) -> Dict[str, float]:
-    # choose targets by quantiles of dataset sigma
-    return {
-        "Low": float(np.quantile(sigmas, 0.15)),
-        "Medium": float(np.quantile(sigmas, 0.50)),
-        "High": float(np.quantile(sigmas, 0.85)),
-    }
+    return {"Low": float(np.quantile(sigmas, 0.15)),
+            "Medium": float(np.quantile(sigmas, 0.50)),
+            "High": float(np.quantile(sigmas, 0.85))}
 
 def _describe_row_for_embeddings(row: pd.Series, universe: List[str]) -> str:
-    # text description for semantic reranking
     parts = [f"sigma {row['sigma_p']:.4f}", f"beta {row['beta_p']:.2f}", f"expected return {row['er_p']:.4f}"]
     ex = _row_exposures(row, universe)
     if ex is not None:
         top = sorted([(universe[i], float(abs(ex[i]))) for i in range(len(universe))], key=lambda kv: -kv[1])[:4]
-        parts.append("focus on " + ", ".join([f"{t}:{w:.2f}" for t, w in top]))
+        parts.append("focus " + ", ".join([f"{t}:{w:.2f}" for t, w in top]))
     return " ".join(parts)
 
-def _get_prompt(risk_level: str) -> str:
-    if risk_level == "Low":
-        return "low risk, stable, conservative diversified portfolio"
-    if risk_level == "High":
-        return "high risk, growth oriented, aggressive portfolio"
-    return "balanced moderate risk diversified portfolio"
-
 def _maybe_load_st_model():
     global _ST_MODEL
     if _ST_MODEL is None:
@@ -364,83 +291,77 @@ def _maybe_load_st_model():
         _ST_MODEL = SentenceTransformer("FinLang/finance-embeddings-investopedia")
     return _ST_MODEL
 
-def suggest_from_dataset(csv_path: str,
-                         universe: List[str],
-                         total_amount: float,
-                         risk_level: str,
-                         use_embeddings: bool = False):
-    try:
-        df = pd.read_csv(csv_path)
-    except Exception:
-        return pd.DataFrame(columns=SUG_COLS_HOLD), None
-
-    if df.empty:
-        return pd.DataFrame(columns=SUG_COLS_HOLD), None
+def build_suggestions(csv_path: str,
+                      universe: List[str],
+                      total_amount: float,
+                      risk_level: str,
+                      use_embeddings: bool,
+                      covA: pd.DataFrame,
+                      betas: Dict[str, float],
+                      rf_ann: float,
+                      erp_ann: float,
+                      mu_all_ann: pd.Series):
+    try: df = pd.read_csv(csv_path)
+    except Exception: return [], pd.DataFrame(columns=SUG_COLS_HOLD)
+
+    if df.empty: return [], pd.DataFrame(columns=SUG_COLS_HOLD)
 
     sigmas = df["sigma_p"].to_numpy(dtype=float)
-    targets = _risk_targets(sigmas)
-    target_sigma = targets.get(risk_level, targets["Medium"])
-
-    # distance to target sigma
-    df = df.copy()
-    df["dist"] = (df["sigma_p"] - target_sigma).abs()
+    target_sigma = _risk_targets(sigmas).get(risk_level, float(np.median(sigmas)))
 
-    # Take a reasonable candidate pool
+    df = df.copy(); df["dist"] = (df["sigma_p"] - target_sigma).abs()
     cand = df.nsmallest(100, "dist").reset_index(drop=True)
 
-    # Optional semantic rerank
     if use_embeddings:
         model = _maybe_load_st_model()
-        prompt = _get_prompt(risk_level)
-        texts = [prompt] + [ _describe_row_for_embeddings(r, universe) for _, r in cand.iterrows() ]
+        prompt = {"Low":"low risk conservative mix","Medium":"balanced moderate risk","High":"aggressive growth high risk"}[risk_level]
+        texts = [prompt] + [_describe_row_for_embeddings(r, universe) for _, r in cand.iterrows()]
         embs = model.encode(texts)
-        S = model.similarity(embs[0:1], embs[1:]).flatten()  # cosine similarity
+        S = model.similarity(embs[0:1], embs[1:]).flatten()
         cand = cand.assign(sim=S).sort_values("sim", ascending=False).head(50).reset_index(drop=True)
 
-    # Now pick the top 3 by a combined score (distance, then ER desc)
-    cand["score"] = cand["dist"] - 0.2 * cand["er_p"]  # small bias toward higher ER
+    cand["score"] = cand["dist"] - 0.2*cand["er_p"]
     picks = cand.nsmallest(3, "score").reset_index(drop=True)
 
-    # Build a simple holdings table: percent and dollars
-    hold_rows = []
-    first_pick_mu = None
-    first_pick_sigma = None
+    suggestions = []
     for i, row in picks.iterrows():
         expo = _row_exposures(row, universe)
-        if expo is None:
-            continue
-        if first_pick_mu is None:
-            first_pick_mu = float(row["er_p"])
-            first_pick_sigma = float(row["sigma_p"])
+        if expo is None: continue
         wmap = {universe[j]: float(expo[j]) for j in range(len(universe)) if abs(float(expo[j])) > 1e-4}
-        for t, w in sorted(wmap.items(), key=lambda kv: -abs(kv[1]))[:12]:
-            hold_rows.append({
-                "pick": i + 1,
-                "ticker": t,
-                "weight_%": round(w * 100.0, 2),
-                "amount_$": round(w * total_amount, 2)
-            })
-
-    hold_df = pd.DataFrame(hold_rows, columns=SUG_COLS_HOLD)
-    return hold_df, (first_pick_mu, first_pick_sigma)
-
-# ---------- UI callbacks ----------
+        # recompute metrics using current moments (historical μ for plotting)
+        beta_s, er_capm_s, sigma_s = portfolio_stats(wmap, covA, betas, rf_ann, erp_ann)
+        mu_hist_s = portfolio_hist_return(wmap, mu_all_ann)
+        # holdings table for this pick
+        rows_hold = [{
+            "pick": i+1,
+            "ticker": t,
+            "weight_%": round(w*100.0, 2),
+            "amount_$": round(w*total_amount, 2)
+        } for t, w in sorted(wmap.items(), key=lambda kv: -abs(kv[1]))]
+        suggestions.append({
+            "pick": i+1,
+            "hold_df": pd.DataFrame(rows_hold, columns=SUG_COLS_HOLD),
+            "mu_hist": mu_hist_s, "sigma_hist": sigma_s,
+            "beta": beta_s, "er_capm": er_capm_s
+        })
+
+    first_table = suggestions[0]["hold_df"] if suggestions else pd.DataFrame(columns=SUG_COLS_HOLD)
+    return suggestions, first_table
+
+# ---------------- UI callbacks ----------------
 def search_tickers_cb(q: str):
     hits = yahoo_search(q)
-    if not hits:
-        return "No matches", []
+    if not hits: return "No matches", []
     opts = [f"{h['symbol']}  |  {h['name']}  |  {h['exchange']}" for h in hits]
     return "Select a symbol and click Add", opts
 
 def add_symbol(selection: str, table: pd.DataFrame):
-    if not selection:
-        return table, "Pick a row from Matches first."
+    if not selection: return table, "Pick a row from Matches first."
     symbol = selection.split("|")[0].strip().upper()
     current = [] if table is None or len(table) == 0 else [str(x).upper() for x in table["ticker"].tolist() if str(x) != "nan"]
     tickers = current if symbol in current else current + [symbol]
     val = validate_tickers(tickers, years=DEFAULT_LOOKBACK_YEARS)
     tickers = [t for t in tickers if t in val]
-    # preserve amounts
     amt_map = {}
     if table is not None and len(table) > 0:
         for _, r in table.iterrows():
@@ -450,8 +371,7 @@ def add_symbol(selection: str, table: pd.DataFrame):
     new_table = pd.DataFrame({"ticker": tickers, "amount_usd": [amt_map.get(t, 0.0) for t in tickers]})
     msg = f"Added {symbol}" if symbol in tickers else f"{symbol} not valid"
     if len(new_table) > MAX_TICKERS:
-        new_table = new_table.iloc[:MAX_TICKERS]
-        msg = f"Reached max of {MAX_TICKERS}"
+        new_table = new_table.iloc[:MAX_TICKERS]; msg = f"Reached max of {MAX_TICKERS}"
     return new_table, msg
 
 def lock_ticker_column(tb: pd.DataFrame):
@@ -464,138 +384,163 @@ def lock_ticker_column(tb: pd.DataFrame):
     amounts = amounts[:len(tickers)] + [0.0] * max(0, len(tickers) - len(amounts))
     return pd.DataFrame({"ticker": tickers, "amount_usd": amounts})
 
-# Global horizon & rf on change (persisted during session)
 HORIZON_YEARS = 10
 RF_CODE = fred_series_for_horizon(HORIZON_YEARS)
 RF_ANN = fetch_fred_yield_annual(RF_CODE)
 
 def set_horizon(years: float):
     y = max(1.0, min(100.0, float(years)))
-    code = fred_series_for_horizon(y)
-    rf = fetch_fred_yield_annual(code)
+    code = fred_series_for_horizon(y); rf = fetch_fred_yield_annual(code)
     global HORIZON_YEARS, RF_CODE, RF_ANN
-    HORIZON_YEARS = y
-    RF_CODE = code
-    RF_ANN = rf
+    HORIZON_YEARS = y; RF_CODE = code; RF_ANN = rf
     return f"Risk-free series {code}. Latest annual rate {fmt_pct(rf)}. Horizon set to {int(round(y))} years."
 
 def compute(lookback_years: int,
             table: pd.DataFrame,
             risk_level: str,
             use_embeddings: bool):
-    # ---- read table
     df = table.dropna()
     df["ticker"] = df["ticker"].astype(str).str.upper().str.strip()
     df["amount_usd"] = pd.to_numeric(df["amount_usd"], errors="coerce").fillna(0.0)
 
     symbols = [t for t in df["ticker"].tolist() if t]
     if len(symbols) == 0:
-        return None, "Add at least one ticker.", "—", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=SUG_COLS_HOLD), None
+        empty_hold = pd.DataFrame(columns=SUG_COLS_HOLD)
+        empty_pos = pd.DataFrame(columns=POS_COLS)
+        return None, "Add at least one ticker.", "—", empty_pos, empty_hold, None, [], {}
 
     symbols = validate_tickers(symbols, lookback_years)
     if len(symbols) == 0:
-        return None, "Could not validate any tickers.", "—", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=SUG_COLS_HOLD), None
+        empty_hold = pd.DataFrame(columns=SUG_COLS_HOLD)
+        empty_pos = pd.DataFrame(columns=POS_COLS)
+        return None, "Could not validate any tickers.", "—", empty_pos, empty_hold, None, [], {}
 
-    universe = list(sorted(set([s for s in symbols if s != MARKET_TICKER] + [MARKET_TICKER])))[:MAX_TICKERS]
+    universe = list(sorted(set(symbols + [MARKET_TICKER])))[:MAX_TICKERS]
 
     df = df[df["ticker"].isin(symbols)].copy()
     amounts = {r["ticker"]: float(r["amount_usd"]) for _, r in df.iterrows()}
     total_amt = float(sum(abs(v) for v in amounts.values()))
     if total_amt <= 1e-12:
-        return None, "All amounts are zero.", f"Universe set to {', '.join(universe)}", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=SUG_COLS_HOLD), None
+        empty_hold = pd.DataFrame(columns=SUG_COLS_HOLD)
+        empty_pos = pd.DataFrame(columns=POS_COLS)
+        return None, "All amounts are zero.", f"Universe set to {', '.join(universe)}", empty_pos, empty_hold, None, [], {}
+
     weights = {k: v / total_amt for k, v in amounts.items()}
 
-    # ---- moments & portfolio metrics
     moms = estimate_all_moments_aligned(universe, lookback_years, RF_ANN)
-    betas, covA, erp_ann, sigma_mkt = moms["betas"], moms["cov_ann"], moms["erp_ann"], moms["sigma_m_ann"]
-    beta_p, er_p, sigma_p = portfolio_stats(weights, covA, betas, RF_ANN, erp_ann)
+    betas, covA, erp_ann = moms["betas"], moms["cov_ann"], moms["erp_ann"]
+    sigma_mkt, mu_all_ann = moms["sigma_m_ann"], moms["mu_all_ann"]
+
+    beta_p, er_capm_p, sigma_p = portfolio_stats(weights, covA, betas, RF_ANN, erp_ann)
+    mu_hist_p = portfolio_hist_return(weights, mu_all_ann)  # use this for plotting
 
     a_sigma, b_sigma, mu_eff_sigma = efficient_same_sigma(sigma_p, RF_ANN, erp_ann, sigma_mkt)
-    a_mu, b_mu, sigma_eff_mu = efficient_same_return(er_p, RF_ANN, erp_ann, sigma_mkt)
+    a_mu, b_mu, sigma_eff_mu     = efficient_same_return(mu_hist_p, RF_ANN, erp_ann, sigma_mkt)
 
-    # ---- dataset build (only for current universe)
+    # dataset for this universe
     csv_path = dataset_path_for_universe(universe)
     if not os.path.exists(csv_path):
         synth = build_synthetic_dataset(universe, lookback_years, RF_ANN, erp_ann, n_rows=DATASET_ROWS)
-        ensure_dir(csv_path)
-        synth.to_csv(csv_path, index=False)
-
-    # ---- dataset-based suggestions (simple table: percent & dollars)
-    hold_df, first_pick_pt = suggest_from_dataset(csv_path, universe, total_amt, risk_level, use_embeddings)
-    sug_mu, sug_sigma = (first_pick_pt if first_pick_pt is not None else (None, None))
-
-    # ---- plot
-    img = plot_cml_percent(
-        RF_ANN, erp_ann, sigma_mkt,
-        sigma_p, er_p,
-        sigma_p, mu_eff_sigma,
-        sigma_eff_mu, er_p,
-        suggestion_sigma=sug_sigma, suggestion_mu=sug_mu
+        ensure_dir(csv_path); synth.to_csv(csv_path, index=False)
+
+    # suggestions list + first table
+    suggestions, first_table = build_suggestions(
+        csv_path, universe, total_amt, risk_level, use_embeddings,
+        covA, betas, RF_ANN, erp_ann, mu_all_ann
     )
 
-    # ---- summary (percent everywhere)
+    # plot state + initial image with first suggestion overlay
+    plot_state = {
+        "rf": RF_ANN, "erp": erp_ann, "sigma_m": sigma_mkt,
+        "pt_sigma": sigma_p, "pt_mu": mu_hist_p,
+        "same_sigma_sigma": sigma_p, "same_sigma_mu": mu_eff_sigma,
+        "same_mu_sigma": sigma_eff_mu, "same_mu_mu": mu_hist_p
+    }
+    sug_overlay = {"sigma": suggestions[0]["sigma_hist"], "mu": suggestions[0]["mu_hist"]} if suggestions else None
+    img = plot_cml_percent(plot_state, suggestion=sug_overlay)
+
+    # summary text (show both CAPM and historical for your portfolio)
     info_lines = []
-    info_lines.append("### Inputs")
-    info_lines.append(f"- Lookback years {int(lookback_years)}")
-    info_lines.append(f"- Horizon years {int(round(HORIZON_YEARS))}")
-    info_lines.append(f"- Risk-free {fmt_pct(RF_ANN)} from {RF_CODE}")
-    info_lines.append(f"- Market ERP {fmt_pct(erp_ann)}")
-    info_lines.append(f"- Market σ {fmt_pct(sigma_mkt)}")
-    info_lines.append("")
-    info_lines.append("### Your portfolio (CAPM expectations)")
-    info_lines.append(f"- Beta {beta_p:.2f}")
-    info_lines.append(f"- σ {fmt_pct(sigma_p)}")
-    info_lines.append(f"- Expected return {fmt_pct(er_p)}")
-    info_lines.append("")
-    info_lines.append("### Efficient alternatives on CML")
-    info_lines.append(f"- Same σ as your portfolio → Market weight {a_sigma:.2f}, Bills weight {b_sigma:.2f}, return {fmt_pct(mu_eff_sigma)}")
-    info_lines.append(f"- Same expected return → Market weight {a_mu:.2f}, Bills weight {b_mu:.2f}, σ {fmt_pct(sigma_eff_mu)}")
-    info_lines.append("")
-    info_lines.append(f"### Dataset-based suggestions (risk: {risk_level})")
-    info_lines.append("- Shown below as simple holdings: percent of exposure and dollars allocated.")
+    info_lines += [
+        "### Inputs",
+        f"- Lookback years {int(lookback_years)}",
+        f"- Horizon years {int(round(HORIZON_YEARS))}",
+        f"- Risk-free {fmt_pct(RF_ANN)} from {RF_CODE}",
+        f"- Market ERP {fmt_pct(erp_ann)}",
+        f"- Market σ {fmt_pct(sigma_mkt)}",
+        "",
+        "### Your portfolio",
+        f"- Beta {beta_p:.2f}",
+        f"- σ (historical) {fmt_pct(sigma_p)}",
+        f"- Expected return (historical) {fmt_pct(mu_hist_p)}",
+        f"- Expected return (CAPM / SML) {fmt_pct(er_capm_p)}",
+        "",
+        "### Efficient alternatives on CML",
+        f"- Same σ as your portfolio → Market {a_sigma:.2f}, Bills {b_sigma:.2f}, return {fmt_pct(mu_eff_sigma)}",
+        f"- Same return (historical) → Market {a_mu:.2f}, Bills {b_mu:.2f}, σ {fmt_pct(sigma_eff_mu)}",
+        "",
+        f"### Dataset-based suggestions (risk: {risk_level})",
+        "- Use the selector below to flip between Pick #1 / #2 / #3. Table shows % exposure and $ amounts."
+    ]
     if use_embeddings:
         info_lines.append("- Reranked with finance embeddings (FinLang/finance-embeddings-investopedia).")
-
     info = "\n".join(info_lines)
 
-    # ---- positions table for current inputs
+    # positions table
     rows = []
     for t in symbols:
-        beta_val = 1.0 if t == MARKET_TICKER else betas.get(t, np.nan)
         rows.append({
             "ticker": t,
             "amount_usd": round(amounts.get(t, 0.0), 2),
             "weight_exposure": round(weights.get(t, 0.0), 6),
-            "beta": round(beta_val, 6),
+            "beta": round(betas.get(t, np.nan), 6),
         })
     pos_table = pd.DataFrame(rows, columns=POS_COLS)
 
     uni_msg = f"Universe set to: {', '.join(universe)}"
-    return img, info, uni_msg, pos_table, hold_df, csv_path
-
-# ---------- UI ----------
+    # also return a short pick-info for pick #1
+    pick_info = ""
+    if suggestions:
+        s = suggestions[0]
+        pick_info = (f"**Pick #1** — σ {fmt_pct(s['sigma_hist'])}, "
+                     f"ER (hist) {fmt_pct(s['mu_hist'])}, "
+                     f"ER (CAPM) {fmt_pct(s['er_capm'])}, beta {s['beta']:.2f}")
+
+    return img, info, uni_msg, pos_table, first_table, csv_path, suggestions, plot_state, pick_info
+
+def change_pick(idx: int, suggestions, plot_state):
+    # idx is 1..3
+    if not suggestions or idx is None:
+        return pd.DataFrame(columns=SUG_COLS_HOLD), plot_cml_percent(plot_state), ""
+    i = int(idx) - 1
+    if i < 0 or i >= len(suggestions):
+        i = 0
+    s = suggestions[i]
+    img = plot_cml_percent(plot_state, suggestion={"sigma": s["sigma_hist"], "mu": s["mu_hist"]})
+    pick_info = (f"**Pick #{idx}** — σ {fmt_pct(s['sigma_hist'])}, "
+                 f"ER (hist) {fmt_pct(s['mu_hist'])}, "
+                 f"ER (CAPM) {fmt_pct(s['er_capm'])}, beta {s['beta']:.2f}")
+    return s["hold_df"], img, pick_info
+
+# ---------------- UI ----------------
 with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
     with gr.Accordion("About (assignment section 1)", open=False):
         gr.Markdown(
             "**Modality**: Text.\n\n"
-            "**Use case**: Given a user’s stock/ETF universe and current dollar amounts, the system recommends three "
-            "alternative mixes (Low / Medium / High risk) generated from a 1,000-row dataset of random portfolios built "
-            "only from the user’s current universe.\n\n"
-            "**System goal**: User provides text inputs (tickers and amounts). The system returns three similar items "
-            "(suggested mixes) from the dataset. Optional reranking uses the text-embedding model "
-            "`FinLang/finance-embeddings-investopedia`."
+            "**Use case**: Given a user’s stock/ETF universe and dollar amounts, the system recommends three "
+            "alternative mixes (Low / Medium / High risk) drawn from a 1,000-row dataset generated from the user’s current universe.\n\n"
+            "**System goal**: User inputs text (tickers & amounts). System returns three similar items (suggested mixes) from the dataset. "
+            "Optional reranking uses the text-embedding model `FinLang/finance-embeddings-investopedia`."
         )
 
     gr.Markdown(
         "## Efficient Portfolio Advisor\n"
-        "Search symbols, enter dollar amounts, set your horizon. Prices from Yahoo Finance. "
-        "Risk-free from FRED. Low/Medium/High suggestions are chosen only from a 1,000-row dataset generated from your current universe, "
-        "optionally refined with finance embeddings."
+        "Search symbols, enter dollar amounts, set your horizon. Prices from Yahoo Finance. Risk-free from FRED. "
+        "Suggestions are built only from your current universe and optionally refined with finance embeddings."
     )
 
     with gr.Row():
         with gr.Column(scale=1):
-            # search
             q = gr.Textbox(label="Search symbol")
             search_note = gr.Markdown(" ")
             matches = gr.Dropdown(choices=[], label="Matches")
@@ -603,25 +548,20 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
                 search_btn = gr.Button("Search")
                 add_btn = gr.Button("Add selected to portfolio")
 
-            # portfolio table
             gr.Markdown("### Portfolio positions — type dollar amounts (negatives allowed for shorts)")
             table = gr.Dataframe(
                 headers=["ticker", "amount_usd"],
                 datatype=["str", "number"],
-                row_count=0,
-                col_count=(2, "fixed"),
+                row_count=0, col_count=(2, "fixed"),
                 value=pd.DataFrame(columns=["ticker", "amount_usd"])
             )
 
-            # horizon & lookback
             horizon = gr.Number(label="Horizon in years (1–100)", value=HORIZON_YEARS, precision=0)
             lookback = gr.Slider(1, 10, value=DEFAULT_LOOKBACK_YEARS, step=1, label="Lookback years for beta & sigma")
 
-            # suggestions controls
             gr.Markdown("### Suggestions")
             risk = gr.Radio(["Low", "Medium", "High"], value="Medium", label="Risk tolerance")
             use_st = gr.Checkbox(label="Use finance embeddings to refine picks", value=False)
-
             run_btn = gr.Button("Compute (build dataset & suggest)")
 
         with gr.Column(scale=1):
@@ -636,8 +576,12 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
                 value=pd.DataFrame(columns=POS_COLS),
                 interactive=False
             )
-            suggestions = gr.Dataframe(
-                label="Dataset-based suggestions (top 3 — holdings shown as % and $)",
+
+            # Suggestion picker
+            pick_slider = gr.Slider(1, 3, value=1, step=1, label="View suggested mix #", interactive=True)
+            pick_info = gr.Markdown("")
+            suggestions_tbl = gr.Dataframe(
+                label="Holdings (for selected pick) — percent & dollars",
                 headers=SUG_COLS_HOLD,
                 datatype=["number", "str", "number", "number"],
                 col_count=(len(SUG_COLS_HOLD), "fixed"),
@@ -646,11 +590,12 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
             )
             dl = gr.File(label="Generated dataset CSV", value=None, visible=True)
 
-    # wiring
-    def do_search(query):
-        note, options = search_tickers_cb(query)
-        return note, gr.update(choices=options)
+    # States to support picker
+    sug_state = gr.State([])
+    plot_state = gr.State({})
 
+    # Wire up events
+    def do_search(query): note, options = search_tickers_cb(query); return note, gr.update(choices=options)
     search_btn.click(fn=do_search, inputs=q, outputs=[search_note, matches])
     add_btn.click(fn=add_symbol, inputs=[matches, table], outputs=[table, search_note])
     table.change(fn=lock_ticker_column, inputs=table, outputs=table)
@@ -659,7 +604,13 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
     run_btn.click(
         fn=compute,
         inputs=[lookback, table, risk, use_st],
-        outputs=[plot, summary, universe_msg, positions, suggestions, dl]
+        outputs=[plot, summary, universe_msg, positions, suggestions_tbl, dl, sug_state, plot_state, pick_info]
+    )
+
+    pick_slider.change(
+        fn=change_pick,
+        inputs=[pick_slider, sug_state, plot_state],
+        outputs=[suggestions_tbl, plot, pick_info]
     )
 
 if __name__ == "__main__":