Update app.py

app.py

@@ -1,72 +1,42 @@
-# app.py
-# Efficient Portfolio Advisor — CAPM on CML + Low/Medium/High suggestion carousel
-# - Search tickers, enter $ amounts (negatives allowed), pick horizon
-# - Plot shows CAPM point on the CML (not historical)
-# - Suggestions are sampled from a 1,000-row dataset generated from your universe
-# - Carousel lets you flip between 3 suggestions in the chosen risk band
-# - Optional: rerank suggestions with finance embeddings (FinLang) to be on-theme
-
-import io
-import os
-import math
-import json
-import time
-import warnings
-from typing import Dict, List, Optional, Tuple
-
+3# app.py
+import os, io, math, time, warnings
 warnings.filterwarnings("ignore")
 
+from typing import List, Tuple, Dict, Optional
+
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
 from PIL import Image
-
-import gradio as gr
 import requests
 import yfinance as yf
+import gradio as gr
 
-# Optional embeddings (won't break if missing GPU; loads once)
-_EMBED_MODEL = None
-def get_embed_model():
-    global _EMBED_MODEL
-    if _EMBED_MODEL is None:
-        try:
-            from sentence_transformers import SentenceTransformer
-            _EMBED_MODEL = SentenceTransformer("FinLang/finance-embeddings-investopedia")
-        except Exception:
-            _EMBED_MODEL = None
-    return _EMBED_MODEL
-
-
-# ---------------- Configuration ----------------
+# ---------------- config ----------------
 DATA_DIR = "data"
 os.makedirs(DATA_DIR, exist_ok=True)
 
-MARKET_TICKER = "VOO"         # proxy for market
 MAX_TICKERS = 30
 DEFAULT_LOOKBACK_YEARS = 10
-DEFAULT_HORIZON_YEARS = 10
-SYNTH_ROWS = 1000
-
-FRED_MAP = [
-    (1,  "DGS1"),
-    (2,  "DGS2"),
-    (3,  "DGS3"),
-    (5,  "DGS5"),
-    (7,  "DGS7"),
-    (10, "DGS10"),
-    (20, "DGS20"),
-    (30, "DGS30"),
-    (100,"DGS30"),
-]
-
-def ensure_dir(p): os.makedirs(p, exist_ok=True)
+MARKET_TICKER = "VOO"
+
+SYNTH_ROWS = 1000  # size of generated dataset for suggestions
 
+# Globals that update with horizon changes
+HORIZON_YEARS = 10
+RF_CODE = "DGS10"
+RF_ANN = 0.0375  # updated at launch
+
+# ---------------- helpers ----------------
 def fred_series_for_horizon(years: float) -> str:
+    # crude tenor map
     y = max(1.0, min(100.0, float(years)))
-    for cutoff, code in FRED_MAP:
-        if y <= cutoff:
-            return code
+    if y <= 2: return "DGS2"
+    if y <= 3: return "DGS3"
+    if y <= 5: return "DGS5"
+    if y <= 7: return "DGS7"
+    if y <= 10: return "DGS10"
+    if y <= 20: return "DGS20"
     return "DGS30"
 
 def fetch_fred_yield_annual(code: str) -> float:
@@ -81,52 +51,46 @@ def fetch_fred_yield_annual(code: str) -> float:
         return 0.03
 
 def fetch_prices_monthly(tickers: List[str], years: int) -> pd.DataFrame:
-    tickers = list(dict.fromkeys([t.upper().strip() for t in tickers if t]))
-    if not tickers:
-        return pd.DataFrame()
-    start = pd.Timestamp.today(tz="UTC") - pd.DateOffset(years=years, days=7)
-    end = pd.Timestamp.today(tz="UTC")
+    tickers = list(dict.fromkeys([t.upper().strip() for t in tickers]))
+    start = (pd.Timestamp.today(tz="UTC") - pd.DateOffset(years=years, days=7)).date()
+    end = pd.Timestamp.today(tz="UTC").date()
 
-    raw = yf.download(
+    df = yf.download(
         tickers,
-        start=start.date(),
-        end=end.date(),
+        start=start,
+        end=end,
         interval="1mo",
         auto_adjust=True,
+        actions=False,
         progress=False,
-        group_by="column"
+        group_by="column",
+        threads=False,
     )
-    if raw is None or len(raw) == 0:
-        return pd.DataFrame()
-
-    # Handle single or multi-index columns
-    if isinstance(raw.columns, pd.MultiIndex):
-        # level 0: OHLCV, level 1: ticker
-        if "Close" in raw.columns.levels[0]:
-            closes = raw.xs("Close", axis=1, level=0)
+
+    # Normalize to wide frame of prices (one column per ticker)
+    if isinstance(df, pd.Series):
+        df = df.to_frame()
+    if isinstance(df.columns, pd.MultiIndex):
+        # prefer Close; fall back to Adj Close if needed
+        lvl0 = [str(x) for x in df.columns.get_level_values(0).unique()]
+        if "Close" in lvl0:
+            df = df["Close"]
+        elif "Adj Close" in lvl0:
+            df = df["Adj Close"]
         else:
-            # fallback: try Adj Close else last level
-            level0 = raw.columns.levels[0].tolist()
-            col0 = "Adj Close" if "Adj Close" in level0 else level0[0]
-            closes = raw.xs(col0, axis=1, level=0)
+            # take last level if unknown shape
+            df = df.xs(df.columns.levels[0][-1], axis=1, level=0, drop_level=True)
     else:
-        if "Close" in raw.columns:
-            closes = raw[["Close"]]
-        elif "Adj Close" in raw.columns:
-            closes = raw[["Adj Close"]].rename(columns={"Adj Close":"Close"})
-        else:
-            closes = raw
-
-    if isinstance(closes, pd.Series):
-        closes = closes.to_frame()
+        # some yfinance versions already return simple columns per ticker
+        pass
 
-    # Normalize columns to uppercase tickers
-    closes.columns = [str(c).upper() for c in closes.columns]
-    closes = closes.dropna(how="all").fillna(method="ffill")
-    return closes
+    # keep only tickers we asked for, forward fill, drop all-NaN rows
+    cols = [c for c in tickers if c in df.columns]
+    out = df[cols].dropna(how="all").fillna(method="ffill")
+    return out
 
 def monthly_returns(prices: pd.DataFrame) -> pd.DataFrame:
-    return prices.pct_change().dropna(how="all")
+    return prices.pct_change().dropna()
 
 def yahoo_search(query: str):
     if not query or not str(query).strip():
@@ -152,28 +116,28 @@ def yahoo_search(query: str):
         return [f"{query.strip().upper()}  |  typed symbol  |  n/a"]
 
 def validate_tickers(symbols: List[str], years: int) -> List[str]:
-    base = [s for s in dict.fromkeys([t.upper().strip() for t in symbols if t])]
+    base = [s for s in dict.fromkeys([t.upper().strip() for t in symbols]) if s]
     px = fetch_prices_monthly(base + [MARKET_TICKER], years)
-    ok = [t for t in base if t in px.columns]
+    ok = [s for s in base if s in px.columns]
+    # Ensure market exists as well for aligned computation
+    if MARKET_TICKER not in px.columns:
+        return []  # without market we can't compute CAPM moments
     return ok
 
-
-# ---------------- Moments / CAPM ----------------
+# -------------- aligned moments --------------
 def get_aligned_monthly_returns(symbols: List[str], years: int) -> pd.DataFrame:
-    uniq = [c for c in dict.fromkeys([s.upper() for s in symbols if s])]
-    if MARKET_TICKER not in uniq:
-        uniq.append(MARKET_TICKER)
-    px = fetch_prices_monthly(uniq, years)
+    uniq = [c for c in dict.fromkeys(symbols) if c != MARKET_TICKER]
+    tickers = uniq + [MARKET_TICKER]
+    px = fetch_prices_monthly(tickers, years)
     rets = monthly_returns(px)
-    cols = [c for c in uniq if c in rets.columns]
+    cols = [c for c in uniq if c in rets.columns] + ([MARKET_TICKER] if MARKET_TICKER in rets.columns else [])
     R = rets[cols].dropna(how="any")
     return R.loc[:, ~R.columns.duplicated()]
 
 def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
     R = get_aligned_monthly_returns(symbols, years)
-    if MARKET_TICKER not in R.columns or R.shape[0] < 3:
-        raise ValueError("Not enough aligned data for market / assets")
-
+    if MARKET_TICKER not in R.columns or len(R) < 3:
+        raise ValueError("Not enough aligned data with market proxy.")
     rf_m = rf_ann / 12.0
 
     m = R[MARKET_TICKER]
@@ -186,19 +150,19 @@ def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
 
     ex_m = m - rf_m
     var_m = float(np.var(ex_m.values, ddof=1))
-    var_m = max(var_m, 1e-8)
+    var_m = max(var_m, 1e-9)
 
     betas: Dict[str, float] = {}
     for s in [c for c in R.columns if c != MARKET_TICKER]:
         ex_s = R[s] - rf_m
         cov_sm = float(np.cov(ex_s.values, ex_m.values, ddof=1)[0, 1])
         betas[s] = cov_sm / var_m
+
     betas[MARKET_TICKER] = 1.0
 
-    # IMPORTANT: include the market in covariance (fixes under-estimated sigma)
-    cov_cols = list(R.columns)
-    cov_m = np.cov(R[cov_cols].values.T, ddof=1)
-    covA = pd.DataFrame(cov_m * 12.0, index=cov_cols, columns=cov_cols)
+    asset_cols = [c for c in R.columns if c != MARKET_TICKER]
+    cov_m = np.cov(R[asset_cols].values.T, ddof=1) if asset_cols else np.zeros((0, 0))
+    covA = pd.DataFrame(cov_m * 12.0, index=asset_cols, columns=asset_cols)
 
     return {"betas": betas, "cov_ann": covA, "erp_ann": erp_ann, "sigma_m_ann": sigma_m_ann}
 
@@ -219,58 +183,41 @@ def portfolio_stats(weights: Dict[str, float],
     beta_p = float(np.dot([betas.get(t, 0.0) for t in tickers], w_expo))
     mu_capm = capm_er(beta_p, rf_ann, erp_ann)
     cov = cov_ann.reindex(index=tickers, columns=tickers).fillna(0.0).to_numpy()
-    sigma_hist = float(max(w_expo.T @ cov @ w_expo, 0.0)) ** 0.5  # annualized
+    sigma_hist = float(max(w_expo.T @ cov @ w_expo, 0.0)) ** 0.5
     return beta_p, mu_capm, sigma_hist
 
-
-# ---------------- Efficient points on the CML ----------------
-def efficient_same_sigma_on_cml(sigma_target: float, rf: float, erp: float, sigma_mkt: float) -> float:
-    # Expected return on CML at a given sigma
+def efficient_same_sigma(sigma_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
+    # weights on (Market, Bills) that achieve same sigma as target, on CML
     if sigma_mkt <= 1e-12:
-        return rf
+        return 0.0, 1.0, rf_ann
     a = sigma_target / sigma_mkt
-    return rf + a * erp
-
-def efficient_same_return_on_cml(mu_target: float, rf: float, erp: float, sigma_mkt: float) -> float:
-    # Sigma on CML needed to hit a target return
-    if abs(erp) <= 1e-12:
-        return 0.0
-    a = (mu_target - rf) / erp
-    return abs(a) * sigma_mkt
+    return a, 1.0 - a, rf_ann + a * erp_ann
 
+def efficient_same_return(mu_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
+    if abs(erp_ann) <= 1e-12:
+        return 0.0, 1.0, rf_ann
+    a = (mu_target - rf_ann) / erp_ann
+    return a, 1.0 - a, abs(a) * sigma_mkt
 
-# ---------------- Plot ----------------
+# -------------- plotting (CAPM on CML) --------------
 def _pct(x):
-    arr = np.asarray(x, dtype=float)
-    return arr * 100.0
-
-def plot_cml(
-    rf_ann: float,
-    erp_ann: float,
-    sigma_mkt: float,
-    port_beta: float,
-    port_mu_capm: float,
-    port_sigma_capm: float,
-    sugg_mu_capm: Optional[float],
-    sugg_sigma_capm: Optional[float],
-) -> Image.Image:
-    fig = plt.figure(figsize=(6.5, 4.2), dpi=120)
-
-    xmax = max(0.30, sigma_mkt * 2.1, port_sigma_capm * 1.35, (sugg_sigma_capm or 0) * 1.35)
-    xs = np.linspace(0.0, xmax, 160)
-    cml = rf_ann + (erp_ann / max(sigma_mkt, 1e-12)) * xs
-    plt.plot(_pct(xs), _pct(cml), label="CML via Market", linewidth=1.8)
+    return np.asarray(x, dtype=float) * 100.0
+
+def plot_cml(rf_ann, erp_ann, sigma_mkt, beta_p, mu_capm, sigma_capm, sugg_mu=None, sugg_sigma=None) -> Image.Image:
+    fig = plt.figure(figsize=(6, 4), dpi=120)
 
-    # key points
-    plt.scatter([_pct(0.0)], [_pct(rf_ann)], label="Risk-free", zorder=3)
-    plt.scatter([_pct(sigma_mkt)], [_pct(rf_ann + erp_ann)], label="Market", zorder=3)
+    xmax = max(0.3, sigma_mkt * 2.2, (sigma_capm or 0.0) * 1.6, (sugg_sigma or 0.0) * 1.6)
+    xs = np.linspace(0, xmax, 200)
+    cml = rf_ann + (erp_ann / max(sigma_mkt, 1e-9)) * xs
 
-    # Your CAPM point
-    plt.scatter([_pct(port_sigma_capm)], [_pct(port_mu_capm)], label="Your CAPM point", zorder=4)
+    plt.plot(_pct(xs), _pct(cml), label="CML via Market", linewidth=1.8)
+    # key points on CML (CAPM view)
+    plt.scatter([_pct(0)], [_pct(rf_ann)], label="Risk-free")
+    plt.scatter([_pct(sigma_mkt)], [_pct(rf_ann + erp_ann)], label="Market")
+    plt.scatter([_pct(sigma_capm)], [_pct(mu_capm)], label="Your CAPM point", marker="o")
 
-    # Selected suggestion (if any)
-    if sugg_mu_capm is not None and sugg_sigma_capm is not None:
-        plt.scatter([_pct(sugg_sigma_capm)], [_pct(sugg_mu_capm)], label="Selected Suggestion", zorder=4)
+    if sugg_mu is not None and sugg_sigma is not None:
+        plt.scatter([_pct(sugg_sigma)], [_pct(sugg_mu)], label="Selected Suggestion", marker="X", s=60)
 
     plt.xlabel("σ (annualized, %)")
     plt.ylabel("Expected return (annual, %)")
@@ -283,24 +230,33 @@ def plot_cml(
     buf.seek(0)
     return Image.open(buf)
 
-
-# ---------------- Synthetic dataset (universe-driven) ----------------
+# -------------- synthetic dataset (from current universe) --------------
 def build_synthetic_dataset(universe: List[str],
-                            cov_ann: pd.DataFrame,
+                            covA: pd.DataFrame,
                             betas: Dict[str, float],
-                            rf_ann: float, erp_ann: float,
+                            rf_ann: float,
+                            erp_ann: float,
+                            sigma_mkt: float,
                             n_rows: int = SYNTH_ROWS) -> pd.DataFrame:
     rng = np.random.default_rng(12345)
+    assets = [t for t in universe if t != MARKET_TICKER]
+    if not assets:
+        assets = [MARKET_TICKER]
+
     rows = []
-    tickers = list(dict.fromkeys([t for t in universe if t]))
     for i in range(n_rows):
-        k = int(rng.integers(low=max(2, min(2, len(tickers))), high=min(8, len(tickers)) + 1))
-        picks = list(rng.choice(tickers, size=k, replace=False))
-        w = rng.dirichlet(np.ones(k))  # long-only exposure
-        # stats
-        wmap = {picks[j]: float(w[j]) for j in range(k)}
-        beta_p, mu_capm, sigma_hist = portfolio_stats(wmap, cov_ann, betas, rf_ann, erp_ann)
-        sigma_capm = abs(beta_p) * (cov_ann.loc[MARKET_TICKER, MARKET_TICKER] ** 0.5) if MARKET_TICKER in cov_ann.index else 0.0
+        k = int(rng.integers(low=2, high=min(8, len(universe)) + 1))
+        picks = list(rng.choice(universe, size=k, replace=False))
+        # long-only exposures sum to 1 (cleaner for presentation)
+        w = rng.dirichlet(np.ones(k))
+        beta_p = float(np.dot([betas.get(t, 0.0) for t in picks], w))
+        mu_capm = capm_er(beta_p, rf_ann, erp_ann)
+        # historical sigma of that physical mix (not used on CML)
+        sub = covA.reindex(index=picks, columns=picks).fillna(0.0).to_numpy()
+        sigma_hist = float(max(w.T @ sub @ w, 0.0)) ** 0.5
+        # CAPM sigma on CML for same expected return
+        sigma_capm = abs(beta_p) * sigma_mkt
+
         rows.append({
             "tickers": ",".join(picks),
             "weights": ",".join(f"{x:.6f}" for x in w),
@@ -311,107 +267,101 @@ def build_synthetic_dataset(universe: List[str],
         })
     return pd.DataFrame(rows)
 
-def parse_row_exposures(row: pd.Series, universe: List[str]) -> Optional[np.ndarray]:
-    try:
-        ts = [t.strip() for t in str(row["tickers"]).split(",")]
-        ws = [float(x) for x in str(row["weights"]).split(",")]
-        mp = {ts[i]: ws[i] for i in range(min(len(ts), len(ws)))}
-        v = np.array([mp.get(t, 0.0) for t in universe], dtype=float)
-        s = float(v.sum())
-        if s <= 1e-12: return None
-        return v / s
-    except Exception:
-        return None
-
-def select_band(df: pd.DataFrame, band: str) -> pd.DataFrame:
-    if df.empty: return df
-    q = df["sigma_capm"].quantile
-    if band == "Low":
-        lo, hi = -1.0, q(0.25)
-    elif band == "Medium":
-        lo, hi = q(0.40), q(0.60)
-    else:  # High
-        lo, hi = q(0.75), float("inf")
-    cut = df[(df["sigma_capm"] >= lo) & (df["sigma_capm"] <= hi)].copy()
-    if cut.empty:
-        return df.nsmallest(3, "sigma_capm") if band == "Low" else df.nlargest(3, "sigma_capm")
-    return cut
-
-def top3_by_return_in_band(df: pd.DataFrame, band: str) -> pd.DataFrame:
-    band_df = select_band(df, band)
-    return band_df.sort_values("mu_capm", ascending=False).head(3).reset_index(drop=True)
-
-
-# ---------------- Embeddings rerank (optional) ----------------
-def rerank_with_embeddings(df3: pd.DataFrame, band: str) -> pd.DataFrame:
-    model = get_embed_model()
-    if model is None or df3.empty:
-        return df3
-
-    prompts = {
-        "Low"   : "low risk diversified ETF mix, low beta, low volatility",
-        "Medium": "balanced risk ETF mix, moderate beta, medium volatility",
-        "High"  : "high risk growth ETF mix, higher beta, higher volatility"
-    }
-    q = prompts.get(band, "balanced portfolio")
-
-    docs = []
-    for _, r in df3.iterrows():
-        docs.append(
-            f"tickers={r['tickers']} weights={r['weights']} "
-            f"beta={r['beta']:.3f} mu_capm={r['mu_capm']:.3f} sigma_capm={r['sigma_capm']:.3f}"
-        )
+def _band_bounds(sigma_mkt: float, band: str) -> Tuple[float, float]:
+    band = (band or "Medium").strip().lower()
+    if band.startswith("low"):
+        return 0.0, 0.8 * sigma_mkt
+    if band.startswith("high"):
+        return 1.2 * sigma_mkt, 3.0 * sigma_mkt
+    # medium
+    return 0.8 * sigma_mkt, 1.2 * sigma_mkt
+
+def top3_by_return_in_band(df: pd.DataFrame, band: str, sigma_mkt: float) -> pd.DataFrame:
+    lo, hi = _band_bounds(sigma_mkt, band)
+    pick = df[(df["sigma_capm"] >= lo) & (df["sigma_capm"] <= hi)].copy()
+    if pick.empty:
+        pick = df.copy()
+    pick = pick.sort_values("mu_capm", ascending=False).head(3).reset_index(drop=True)
+    pick.insert(0, "pick", [1, 2, 3][: len(pick)])
+    return pick
+
+# -------------- optional: embeddings rerank --------------
+def rerank_with_embeddings(top3: pd.DataFrame, band: str) -> pd.DataFrame:
     try:
-        E = model.encode([q] + docs, normalize_embeddings=True)
-        qv = E[0:1]
-        dv = E[1:]
-        sims = (qv @ dv.T).ravel()
-        order = np.argsort(-sims)
-        return df3.iloc[order].reset_index(drop=True)
-    except Exception:
-        return df3
+        from sentence_transformers import SentenceTransformer
+        model = SentenceTransformer("FinLang/finance-embeddings-investopedia")
+        prompt = {
+            "low": "low risk conservative portfolio stable diversified market exposure",
+            "medium": "balanced medium risk diversified portfolio",
+            "high": "high risk growth aggressive portfolio higher expected return"
+        }[(band or "medium").lower() if (band or "medium").lower() in {"low","medium","high"} else "medium"]
+
+        cand_texts = []
+        for _, r in top3.iterrows():
+            cand_texts.append(
+                f"portfolio with tickers {r['tickers']} having beta {float(r['beta']):.2f}, "
+                f"expected return {float(r['mu_capm']):.3f}, sigma {float(r['sigma_capm']):.3f}"
+            )
 
+        q = model.encode([prompt])
+        c = model.encode(cand_texts)
+        # cosine similarity
+        sims = (q @ c.T) / (np.linalg.norm(q) * np.linalg.norm(c, axis=1, keepdims=False))
+        order = np.argsort(-sims.ravel())
+        return top3.iloc[order].reset_index(drop=True)
+    except Exception:
+        return top3
 
-# ---------------- Gradio helpers ----------------
+# -------------- UI helpers --------------
 def empty_positions_df():
     return pd.DataFrame(columns=["ticker", "amount_usd", "weight_exposure", "beta"])
 
 def empty_suggestion_df():
     return pd.DataFrame(columns=["ticker", "weight_%", "amount_$"])
 
-UNIVERSE: List[str] = [MARKET_TICKER]
-HORIZON_YEARS = DEFAULT_HORIZON_YEARS
-RF_CODE = fred_series_for_horizon(HORIZON_YEARS)
-RF_ANN = fetch_fred_yield_annual(RF_CODE)
+def set_horizon(years: float):
+    y = max(1.0, min(100.0, float(years)))
+    code = fred_series_for_horizon(y)
+    rf = fetch_fred_yield_annual(code)
+    global HORIZON_YEARS, RF_CODE, RF_ANN
+    HORIZON_YEARS = y
+    RF_CODE = code
+    RF_ANN = rf
+    return f"Risk-free series {code}. Latest annual rate {rf:.2%}."
 
-def search_cb(q: str):
+def search_tickers_cb(q: str):
     opts = yahoo_search(q)
-    note = "Select a row and click 'Add selected to portfolio'." if opts else "No matches."
+    note = "Select a symbol and click 'Add selected to portfolio'." if opts else "No matches."
     return note, gr.update(choices=opts, value=None)
 
-def add_symbol(selection: str, table: pd.DataFrame):
-    if not selection or "|" not in selection:
-        return table, "Pick a symbol from Matches first."
+def add_symbol(selection: str, table: Optional[pd.DataFrame]):
+    if not selection:
+        return table if isinstance(table, pd.DataFrame) else pd.DataFrame(columns=["ticker","amount_usd"]), "Pick a row in Matches first."
     symbol = selection.split("|")[0].strip().upper()
-    current = [] if table is None or len(table) == 0 else [str(x).upper() for x in table["ticker"].tolist() if str(x) != "nan"]
+
+    current = []
+    if isinstance(table, pd.DataFrame) and not table.empty:
+        current = [str(x).upper() for x in table["ticker"].tolist() if str(x) != "nan"]
     tickers = current if symbol in current else current + [symbol]
+
     val = validate_tickers(tickers, years=DEFAULT_LOOKBACK_YEARS)
     tickers = [t for t in tickers if t in val]
+
     amt_map = {}
-    if table is not None and len(table) > 0:
+    if isinstance(table, pd.DataFrame) and not table.empty:
         for _, r in table.iterrows():
             t = str(r.get("ticker", "")).upper()
             if t in tickers:
                 amt_map[t] = float(pd.to_numeric(r.get("amount_usd", 0.0), errors="coerce") or 0.0)
+
     new_table = pd.DataFrame({"ticker": tickers, "amount_usd": [amt_map.get(t, 0.0) for t in tickers]})
-    msg = f"Added {symbol}" if symbol in tickers else f"{symbol} not valid"
     if len(new_table) > MAX_TICKERS:
         new_table = new_table.iloc[:MAX_TICKERS]
-        msg = f"Reached max of {MAX_TICKERS}"
-    return new_table, msg
+        return new_table, f"Reached max of {MAX_TICKERS}."
+    return new_table, f"Added {symbol}."
 
-def lock_table(tb: pd.DataFrame):
-    if tb is None or len(tb) == 0:
+def lock_ticker_column(tb: Optional[pd.DataFrame]):
+    if not isinstance(tb, pd.DataFrame) or tb.empty:
         return pd.DataFrame(columns=["ticker", "amount_usd"])
     tickers = [str(x).upper() for x in tb["ticker"].tolist()]
     amounts = pd.to_numeric(tb["amount_usd"], errors="coerce").fillna(0.0).tolist()
@@ -420,27 +370,24 @@ def lock_table(tb: pd.DataFrame):
     amounts = amounts[:len(tickers)] + [0.0] * max(0, len(tickers) - len(amounts))
     return pd.DataFrame({"ticker": tickers, "amount_usd": amounts})
 
-def set_horizon(years: float):
-    y = max(1.0, min(100.0, float(years)))
-    code = fred_series_for_horizon(y)
-    rf = fetch_fred_yield_annual(code)
-    global HORIZON_YEARS, RF_CODE, RF_ANN
-    HORIZON_YEARS = y
-    RF_CODE = code
-    RF_ANN = rf
-    return f"Risk-free series {code}. Latest annual rate {rf:.2%}."
-
-def to_pct_str(x): return f"{x*100:.2f}%"
+# -------------- main compute --------------
+UNIVERSE: List[str] = [MARKET_TICKER, "QQQ", "VTI", "SOXX", "IBIT"]
 
 def compute(
     years_lookback: int,
-    table: pd.DataFrame,
+    table: Optional[pd.DataFrame],
     risk_band: str,
     use_embeddings: bool,
     pick_idx: int
 ):
-    # --- inputs
-    df = (table or pd.DataFrame(columns=["ticker","amount_usd"])).dropna()
+    # sanitize table
+    if isinstance(table, pd.DataFrame):
+        df = table.copy()
+    else:
+        df = pd.DataFrame(columns=["ticker", "amount_usd"])
+    df = df.dropna(how="all")
+    if "ticker" not in df.columns: df["ticker"] = []
+    if "amount_usd" not in df.columns: df["amount_usd"] = []
     df["ticker"] = df["ticker"].astype(str).str.upper().str.strip()
     df["amount_usd"] = pd.to_numeric(df["amount_usd"], errors="coerce").fillna(0.0)
 
@@ -459,106 +406,103 @@ def compute(
     amounts = {r["ticker"]: float(r["amount_usd"]) for _, r in df.iterrows()}
     rf_ann = RF_ANN
 
-    # --- moments & CAPM stats
+    # Moments
     moms = estimate_all_moments_aligned(symbols, years_lookback, rf_ann)
     betas, covA, erp_ann, sigma_mkt = moms["betas"], moms["cov_ann"], moms["erp_ann"], moms["sigma_m_ann"]
 
+    # Weights
     gross = sum(abs(v) for v in amounts.values())
     if gross <= 1e-12:
         return None, "All amounts are zero.", "Universe ok.", empty_positions_df(), empty_suggestion_df(), None
     weights = {k: v / gross for k, v in amounts.items()}
 
+    # Portfolio CAPM stats
     beta_p, mu_capm, sigma_hist = portfolio_stats(weights, covA, betas, rf_ann, erp_ann)
     sigma_capm = abs(beta_p) * sigma_mkt
 
-    # --- dataset & suggestions
-    synth = build_synthetic_dataset(UNIVERSE, covA, betas, rf_ann, erp_ann, n_rows=SYNTH_ROWS)
-    # save CSV for the grader / assignment
+    # Efficient alternatives (using historical σ and CAPM μ for reference)
+    a_sigma, b_sigma, mu_eff_sigma = efficient_same_sigma(sigma_hist, rf_ann, erp_ann, sigma_mkt)
+    a_mu, b_mu, sigma_eff_mu = efficient_same_return(mu_capm, rf_ann, erp_ann, sigma_mkt)
+
+    # Synthetic dataset & suggestions
+    synth = build_synthetic_dataset(UNIVERSE, covA, betas, rf_ann, erp_ann, sigma_mkt, n_rows=SYNTH_ROWS)
     csv_path = os.path.join(DATA_DIR, f"investor_profiles_{int(time.time())}.csv")
-    ensure_dir(os.path.dirname(csv_path))
     synth.to_csv(csv_path, index=False)
 
-    top3 = top3_by_return_in_band(synth, risk_band)
+    top3 = top3_by_return_in_band(synth, risk_band, sigma_mkt)
     if use_embeddings:
         top3 = rerank_with_embeddings(top3, risk_band)
-
-    # guard
     if top3.empty:
         top3 = synth.sort_values("mu_capm", ascending=False).head(3).reset_index(drop=True)
+        top3.insert(0, "pick", [1, 2, 3][: len(top3)])
 
-    # pick from carousel (1..3)
     idx = max(1, min(3, int(pick_idx))) - 1
     row = top3.iloc[idx]
 
-    # selected suggestion stats (CAPM)
     sugg_mu = float(row["mu_capm"])
-    sugg_sigma = float(row.get("sigma_capm", abs(row["beta"]) * sigma_mkt))
+    sugg_sigma = float(row["sigma_capm"])
 
-    # Build holdings table (% and $) for selected suggestion
+    # suggestion holdings (% and $)
     ts = [t.strip() for t in str(row["tickers"]).split(",")]
     ws = [float(x) for x in str(row["weights"]).split(",")]
-    wsum = sum(ws) if ws else 1.0
-    ws = [max(0.0, w) / wsum for w in ws]  # long-only normalized
+    s = sum(ws) if ws else 1.0
+    ws = [max(0.0, w) / s for w in ws]
     budget = gross if gross > 0 else 1.0
-    hold_rows = []
-    for t, w in zip(ts, ws):
-        hold_rows.append({
-            "ticker": t,
-            "weight_%": round(w * 100.0, 2),
-            "amount_$": round(w * budget, 0)
-        })
-    sugg_table = pd.DataFrame(hold_rows, columns=["ticker", "weight_%", "amount_$"])
+    sugg_table = pd.DataFrame(
+        [{"ticker": t, "weight_%": round(w*100.0, 2), "amount_$": round(w*budget, 0)} for t, w in zip(ts, ws)],
+        columns=["ticker", "weight_%", "amount_$"]
+    )
 
-    # positions table for current portfolio
-    pos_rows = []
-    for t in symbols:
-        pos_rows.append({
+    # positions table
+    pos_table = pd.DataFrame(
+        [{
             "ticker": t,
             "amount_usd": amounts.get(t, 0.0),
             "weight_exposure": weights.get(t, 0.0),
             "beta": 1.0 if t == MARKET_TICKER else betas.get(t, np.nan)
-        })
-    pos_table = pd.DataFrame(pos_rows, columns=["ticker", "amount_usd", "weight_exposure", "beta"])
-
-    # --- plot
-    img = plot_cml(
-        rf_ann, erp_ann, sigma_mkt,
-        beta_p, mu_capm, sigma_capm,
-        sugg_mu, sugg_sigma
+        } for t in symbols],
+        columns=["ticker", "amount_usd", "weight_exposure", "beta"]
     )
 
-    # --- info markdown
-    info_lines = []
-    info_lines.append("### Inputs")
-    info_lines.append(f"- Lookback years {years_lookback}")
-    info_lines.append(f"- Horizon years {int(round(HORIZON_YEARS))}")
-    info_lines.append(f"- Risk-free {to_pct_str(rf_ann)} from {RF_CODE}")
-    info_lines.append(f"- Market ERP {to_pct_str(erp_ann)}")
-    info_lines.append(f"- Market σ {to_pct_str(sigma_mkt)}")
-    info_lines.append("")
-    info_lines.append("### Your portfolio (CAPM)")
-    info_lines.append(f"- Beta {beta_p:.2f}")
-    info_lines.append(f"- Expected return (CAPM / SML) {to_pct_str(mu_capm)}")
-    info_lines.append(f"- on CML for your beta (|β|×σ_mkt) {to_pct_str(sigma_capm)}")
-    info_lines.append("")
-    info_lines.append("### Dataset-based suggestion (carousel)")
-    info_lines.append(f"- Risk band **{risk_band}**, showing **Pick #{idx+1} of 3**")
-    info_lines.append(f"- Suggested CAPM return {to_pct_str(sugg_mu)}")
-    info_lines.append(f"- Suggested CAPM σ {to_pct_str(sugg_sigma)}")
-    info_lines.append("")
-    info_lines.append("_Note: points are CAPM expectations on the CML (not historical means)._" )
-    info = "\n".join(info_lines)
+    # plot
+    img = plot_cml(rf_ann, erp_ann, sigma_mkt, beta_p, mu_capm, sigma_capm, sugg_mu, sugg_sigma)
+
+    info = "\n".join([
+        "### Inputs",
+        f"- Lookback years {years_lookback}",
+        f"- Horizon years {int(round(HORIZON_YEARS))}",
+        f"- Risk-free {rf_ann:.2%} from {RF_CODE}",
+        f"- Market ERP {erp_ann:.2%}",
+        f"- Market σ {sigma_mkt:.2%}",
+        "",
+        "### Your portfolio (CAPM)",
+        f"- Beta {beta_p:.2f}",
+        f"- Expected return (CAPM / SML) {mu_capm:.2%}",
+        f"- on CML for your beta (|β|×σ_mkt) {sigma_capm:.2%}",
+        "",
+        "### Efficient alternatives on CML",
+        f"- Same σ as your portfolio (historical): Market weight {a_sigma:.2f}, Bills weight {b_sigma:.2f}, return {mu_eff_sigma:.2%}",
+        f"- Same return (CAPM): Market weight {a_mu:.2f}, Bills weight {b_mu:.2f}, σ {sigma_eff_mu:.2%}",
+        "",
+        "### Dataset-based suggestions (risk: " + risk_band + ")",
+        f"- Use the carousel to flip between **Pick #1 / #2 / #3**.",
+        f"- Showing Pick **#{idx+1}** → CAPM return {sugg_mu:.2%}, CAPM σ {sugg_sigma:.2%}",
+        "",
+        "_Plot shows CAPM expectations on the CML (not historical means)._"
+    ])
 
     uni_msg = f"Universe set to: {', '.join(UNIVERSE)}"
-    return img, info, uni_msg, pos_table, sugg_table, csv_path
+    return img, info, uni_msg, pos_table, sugg_table, csv_path, gr.update(label=f"Pick #{idx+1} of 3")
 
+# -------------- UI --------------
+def inc_pick(i: int): return min(3, max(1, int(i or 1) + 1))
+def dec_pick(i: int): return max(1, min(3, int(i or 1) - 1))
 
-# ---------------- UI ----------------
 with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
     gr.Markdown(
         "## Efficient Portfolio Advisor\n"
         "Search symbols, enter **dollar amounts**, set horizon. Returns use Yahoo Finance monthly data; risk-free from FRED. "
-        "Plot shows **CAPM point on the CML** plus selected suggestion."
+        "Plot shows **CAPM point on the CML** plus efficient CML points."
     )
 
     with gr.Row():
@@ -577,21 +521,23 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
                 col_count=(2, "fixed")
             )
 
-            horizon = gr.Number(label="Horizon in years (1–100)", value=DEFAULT_HORIZON_YEARS, precision=0)
-            lookback = gr.Slider(1, 10, value=DEFAULT_LOOKBACK_YEARS, step=1, label="Lookback years for betas & covariances")
+            horizon = gr.Number(label="Horizon in years (1–100)", value=HORIZON_YEARS, precision=0)
+            lookback = gr.Slider(1, 15, value=DEFAULT_LOOKBACK_YEARS, step=1, label="Lookback years for betas & covariances")
 
             gr.Markdown("### Suggestions")
-            risk_band = gr.Radio(choices=["Low", "Medium", "High"], value="Medium", label="Risk tolerance")
-            use_emb = gr.Checkbox(label="Use finance embeddings to refine picks", value=False)
-            pick_idx = gr.Slider(1, 3, value=1, step=1, label="Suggestion (carousel)")
+            risk_band = gr.Radio(["Low", "Medium", "High"], value="Medium", label="Risk tolerance")
+            use_emb = gr.Checkbox(value=True, label="Use finance embeddings to refine picks")
 
-            run_btn = gr.Button("Compute (build dataset & suggest)")
+            with gr.Row():
+                prev_btn = gr.Button("◀ Prev")
+                pick_idx = gr.Number(value=1, precision=0, label="Carousel")
+                next_btn = gr.Button("Next ▶")
 
+            run_btn = gr.Button("Compute (build dataset & suggest)")
         with gr.Column(scale=1):
             plot = gr.Image(label="Capital Market Line (CAPM)", type="pil")
             summary = gr.Markdown(label="Inputs & Results")
             universe_msg = gr.Textbox(label="Universe status", interactive=False)
-
             positions = gr.Dataframe(
                 label="Computed positions",
                 headers=["ticker", "amount_usd", "weight_exposure", "beta"],
@@ -600,7 +546,6 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
                 value=empty_positions_df(),
                 interactive=False
             )
-
             sugg_table = gr.Dataframe(
                 label="Selected suggestion (carousel) — holdings shown in % and $",
                 headers=["ticker", "weight_%", "amount_$"],
@@ -609,21 +554,36 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
                 value=empty_suggestion_df(),
                 interactive=False
             )
-
             dl = gr.File(label="Generated dataset CSV", value=None, visible=True)
 
-    # wiring
-    search_btn.click(fn=search_cb, inputs=q, outputs=[search_note, matches])
+    # wire search / add / locking / horizon
+    search_btn.click(fn=search_tickers_cb, inputs=q, outputs=[search_note, matches])
     add_btn.click(fn=add_symbol, inputs=[matches, table], outputs=[table, search_note])
-    table.change(fn=lock_table, inputs=table, outputs=table)
+    table.change(fn=lock_ticker_column, inputs=table, outputs=table)
     horizon.change(fn=set_horizon, inputs=horizon, outputs=universe_msg)
 
+    # carousel buttons update pick index and then recompute
+    prev_btn.click(fn=dec_pick, inputs=pick_idx, outputs=pick_idx).then(
+        fn=compute,
+        inputs=[lookback, table, risk_band, use_emb, pick_idx],
+        outputs=[plot, summary, universe_msg, positions, sugg_table, dl, pick_idx]
+    )
+    next_btn.click(fn=inc_pick, inputs=pick_idx, outputs=pick_idx).then(
+        fn=compute,
+        inputs=[lookback, table, risk_band, use_emb, pick_idx],
+        outputs=[plot, summary, universe_msg, positions, sugg_table, dl, pick_idx]
+    )
+
     # main compute
     run_btn.click(
         fn=compute,
         inputs=[lookback, table, risk_band, use_emb, pick_idx],
-        outputs=[plot, summary, universe_msg, positions, sugg_table, dl]
+        outputs=[plot, summary, universe_msg, positions, sugg_table, dl, pick_idx]
     )
 
+# initialize risk-free at launch
+RF_CODE = fred_series_for_horizon(HORIZON_YEARS)
+RF_ANN = fetch_fred_yield_annual(RF_CODE)
+
 if __name__ == "__main__":
     demo.launch()