Update app.py

app.py

@@ -1,5 +1,14 @@
+Here’s a full, drop-in **app.py** that:
+
+* keeps the ticker search + portfolio table UX you liked
+* shows the CML with **percent axes**
+* builds a **1,000-row synthetic dataset** for your current universe
+* gives a **single, clean suggestion** (based on Low/Medium/High risk) as **weights (%) and dollars (\$)**
+* can optionally **re-rank** the suggestion with **finance embeddings** (FinLang)
+
+```python
 # app.py
-import os, io, math, json, hashlib, warnings
+import os, io, math, json, warnings, hashlib, random
 warnings.filterwarnings("ignore")
 
 from typing import List, Tuple, Dict, Optional
@@ -7,62 +16,39 @@ from typing import List, Tuple, Dict, Optional
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
-from matplotlib.ticker import PercentFormatter
-from PIL import Image
-
 import gradio as gr
+from PIL import Image
 import requests
 import yfinance as yf
 
-# Optional embeddings (lazy-loaded)
-_EMBED_MODEL = None
-def get_embed_model():
-    global _EMBED_MODEL
-    if _EMBED_MODEL is None:
-        try:
-            from sentence_transformers import SentenceTransformer
-            _EMBED_MODEL = SentenceTransformer("FinLang/finance-embeddings-investopedia")
-        except Exception as e:
-            _EMBED_MODEL = False
-    return _EMBED_MODEL
+from sklearn.neighbors import KNeighborsRegressor
+from sklearn.preprocessing import StandardScaler
 
 # ---------------- config ----------------
 DATA_DIR = "data"
 os.makedirs(DATA_DIR, exist_ok=True)
 
-MARKET_TICKER = "VOO"      # “market” proxy
-DEFAULT_LOOKBACK_YEARS = 5
 MAX_TICKERS = 30
-SYNTH_ROWS = 1000
+DEFAULT_LOOKBACK_YEARS = 10
+MARKET_TICKER = "VOO"  # fall back to SPY if needed
 
 # UI tables
 POS_COLS = ["ticker", "amount_usd", "weight_exposure", "beta"]
-SUG_COLS = ["pick", "ticker", "weight_exposure", "er_%", "sigma_%", "beta"]
 
-# FRED tenor map
+# FRED curve mapping: horizon -> series code
 FRED_MAP = [
-    (1,  "DGS1"),  (2,  "DGS2"),  (3,  "DGS3"),
-    (5,  "DGS5"),  (7,  "DGS7"),  (10, "DGS10"),
-    (20, "DGS20"), (30, "DGS30"), (100, "DGS30"),
+    (1,  "DGS1"),
+    (2,  "DGS2"),
+    (3,  "DGS3"),
+    (5,  "DGS5"),
+    (7,  "DGS7"),
+    (10, "DGS10"),
+    (20, "DGS20"),
+    (30, "DGS30"),
+    (100, "DGS30"),
 ]
 
-# Session globals
-HORIZON_YEARS = 5.0
-RF_CODE = "DGS5"
-RF_ANN = 0.02
-
-def ensure_data_dir():
-    os.makedirs(DATA_DIR, exist_ok=True)
-
-def dataset_path_for_universe(universe: List[str]) -> str:
-    # unique file per universe (order-independent)
-    key = hashlib.sha256((",".join(sorted(universe))).encode()).hexdigest()[:10]
-    return os.path.join(DATA_DIR, f"investor_profiles_{key}.csv")
-
-# ---------------- tiny utils ----------------
-def fmt_pct(x: float) -> str:
-    return f"{x*100:.2f}%"
-
+# ---------------- helpers ----------------
 def fred_series_for_horizon(years: float) -> str:
     y = max(1.0, min(100.0, float(years)))
     for cutoff, code in FRED_MAP:
@@ -81,7 +67,29 @@ def fetch_fred_yield_annual(code: str) -> float:
     except Exception:
         return 0.03
 
-# ---------------- Yahoo search ----------------
+def fetch_prices_monthly(tickers: List[str], years: int) -> pd.DataFrame:
+    start = pd.Timestamp.today(tz="UTC") - pd.DateOffset(years=years, days=7)
+    end = pd.Timestamp.today(tz="UTC")
+    df = yf.download(
+        list(dict.fromkeys(tickers)),
+        start=start.date(),
+        end=end.date(),
+        interval="1mo",
+        auto_adjust=True,
+        progress=False,
+        group_by="ticker",
+    )["Close"]
+    if isinstance(df, pd.Series):
+        df = df.to_frame()
+    df = df.dropna(how="all").fillna(method="ffill")
+    # If yfinance returns MultiIndex columns for multiple tickers, flatten
+    if isinstance(df.columns, pd.MultiIndex):
+        df.columns = [c[0] for c in df.columns]
+    return df
+
+def monthly_returns(prices: pd.DataFrame) -> pd.DataFrame:
+    return prices.pct_change().dropna()
+
 def yahoo_search(query: str):
     if not query or len(query.strip()) == 0:
         return []
@@ -105,78 +113,70 @@ def yahoo_search(query: str):
     except Exception:
         return [{"symbol": query.strip().upper(), "name": "typed symbol", "exchange": "n/a"}]
 
-def fetch_prices_monthly(tickers: List[str], years: int) -> pd.DataFrame:
-    start = pd.Timestamp.today(tz="UTC") - pd.DateOffset(years=years, days=7)
-    end = pd.Timestamp.today(tz="UTC")
-    df = yf.download(
-        list(dict.fromkeys(tickers)),
-        start=start.date(), end=end.date(),
-        interval="1mo", auto_adjust=True, progress=False
-    )["Close"]
-    if isinstance(df, pd.Series):
-        df = df.to_frame()
-    df = df.dropna(how="all").fillna(method="ffill")
-    return df
-
-def monthly_returns(prices: pd.DataFrame) -> pd.DataFrame:
-    return prices.pct_change().dropna()
-
 def validate_tickers(symbols: List[str], years: int) -> List[str]:
-    ok, df = [], fetch_prices_monthly(list(set(symbols)), years)
-    for s in symbols:
-        if s in df.columns:
-            ok.append(s)
+    if not symbols:
+        return []
+    # Always include market proxy so alignment works
+    base = [s for s in dict.fromkeys(symbols)]
+    px = fetch_prices_monthly(base + [MARKET_TICKER], years)
+    ok = [s for s in base if s in px.columns]
+    # If market ticker missing, try SPY as fallback
+    if MARKET_TICKER not in px.columns and "SPY" not in px.columns:
+        # Try once more with SPY added
+        px2 = fetch_prices_monthly(base + ["SPY"], years)
+        ok = [s for s in base if s in px2.columns]
     return ok
 
-# ---------------- moments (aligned) ----------------
+# -------------- aligned moments --------------
 def get_aligned_monthly_returns(symbols: List[str], years: int) -> pd.DataFrame:
-    uniq = [c for c in dict.fromkeys(symbols) if c != MARKET_TICKER]
+    uniq = [c for c in dict.fromkeys(symbols) if c]
     tickers = uniq + [MARKET_TICKER]
     px = fetch_prices_monthly(tickers, years)
+    # if VOO missing, try SPY as market
+    mkt = MARKET_TICKER if MARKET_TICKER in px.columns else ("SPY" if "SPY" in px.columns else None)
+    if mkt is None:
+        return pd.DataFrame()
     rets = monthly_returns(px)
-    cols = [c for c in uniq if c in rets.columns] + ([MARKET_TICKER] if MARKET_TICKER in rets.columns else [])
+    cols = [c for c in uniq if c in rets.columns] + [mkt]
     R = rets[cols].dropna(how="any")
-    return R.loc[:, ~R.columns.duplicated()]
+    return R, mkt
 
-def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
-    R = get_aligned_monthly_returns(symbols + [MARKET_TICKER], years)
-    if MARKET_TICKER not in R.columns or R.shape[0] < 3:
-        raise ValueError("Not enough aligned market data")
+def annualize_mean(m):
+    return np.asarray(m, dtype=float) * 12.0
 
+def annualize_sigma(s):
+    return np.asarray(s, dtype=float) * math.sqrt(12.0)
+
+def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
+    R, mkt = get_aligned_monthly_returns(symbols, years)
+    if R is None or R.empty or mkt is None or R.shape[0] < 3:
+        raise ValueError("Not enough aligned data for selected tickers / lookback.")
     rf_m = rf_ann / 12.0
 
-    # market series
-    m = R[MARKET_TICKER]
+    m = R[mkt]
     if isinstance(m, pd.DataFrame):
         m = m.iloc[:, 0].squeeze()
 
-    mu_m_ann = float(m.mean() * 12.0)
-    sigma_m_ann = float(m.std(ddof=1) * math.sqrt(12.0))
+    mu_m_ann = float(annualize_mean(m.mean()))
+    sigma_m_ann = float(annualize_sigma(m.std(ddof=1)))
     erp_ann = float(mu_m_ann - rf_ann)
 
     ex_m = m - rf_m
     var_m = float(np.var(ex_m.values, ddof=1))
-    var_m = max(var_m, 1e-10)
+    var_m = max(var_m, 1e-6)
 
-    # betas for each asset (including market==1)
     betas: Dict[str, float] = {}
-    for s in R.columns:
-        if s == MARKET_TICKER:
-            betas[s] = 1.0
-            continue
+    for s in [c for c in R.columns if c != mkt]:
         ex_s = R[s] - rf_m
-        cov_sm = float(np.cov(ex_s.values, ex_m.values, ddof=1)[0, 1])
-        betas[s] = float(cov_sm / var_m)
-
-    # IMPORTANT FIX: include MARKET in covariance so σ is never understated
-    asset_cols = list(R.columns)
-    if asset_cols:
-        cov_m = np.cov(R[asset_cols].values.T, ddof=1)
-        covA = pd.DataFrame(cov_m * 12.0, index=asset_cols, columns=asset_cols)
-    else:
-        covA = pd.DataFrame(np.zeros((0, 0)))
+        betas[s] = float(np.cov(ex_s.values, ex_m.values, ddof=1)[0, 1] / var_m)
+
+    betas[mkt] = 1.0
+    # asset covariance (annualized) excluding market column
+    asset_cols = [c for c in R.columns if c != mkt]
+    cov_m = np.cov(R[asset_cols].values.T, ddof=1) if asset_cols else np.zeros((0, 0))
+    covA = pd.DataFrame(cov_m * 12.0, index=asset_cols, columns=asset_cols)
 
-    return {"betas": betas, "cov_ann": covA, "erp_ann": erp_ann, "sigma_m_ann": sigma_m_ann}
+    return {"betas": betas, "cov_ann": covA, "erp_ann": erp_ann, "sigma_m_ann": sigma_m_ann, "mkt": mkt}
 
 def capm_er(beta: float, rf_ann: float, erp_ann: float) -> float:
     return float(rf_ann + beta * erp_ann)
@@ -195,71 +195,73 @@ def portfolio_stats(weights: Dict[str, float],
     beta_p = float(np.dot([betas.get(t, 0.0) for t in tickers], w_expo))
     er_p = capm_er(beta_p, rf_ann, erp_ann)
     cov = cov_ann.reindex(index=tickers, columns=tickers).fillna(0.0).to_numpy()
-    sigma_p = math.sqrt(float(max(w_expo.T @ cov @ w_expo, 0.0)))
+    v = float(w_expo.T @ cov @ w_expo)
+    sigma_p = math.sqrt(max(v, 0.0))
     return beta_p, er_p, sigma_p
 
-# ---------------- CML helpers & plot ----------------
+# -------------- CML helpers --------------
 def efficient_same_sigma(sigma_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
     if sigma_mkt <= 1e-12:
         return 0.0, 1.0, rf_ann
     a = sigma_target / sigma_mkt
-    return a, 1 - a, rf_ann + a * erp_ann
+    return a, 1.0 - a, rf_ann + a * erp_ann
 
 def efficient_same_return(mu_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
     if abs(erp_ann) <= 1e-12:
         return 0.0, 1.0, rf_ann
     a = (mu_target - rf_ann) / erp_ann
-    return a, 1 - a, abs(a) * sigma_mkt
+    return a, 1.0 - a, abs(a) * sigma_mkt
 
-def plot_cml_percent(rf_ann, erp_ann, sigma_mkt,
-                     pt_sigma, pt_mu,
-                     same_sigma_sigma, same_sigma_mu,
-                     same_mu_sigma, same_mu_mu,
-                     suggestion: Optional[Tuple[float, float]] = None) -> Image.Image:
-    fig = plt.figure(figsize=(6, 4), dpi=120)
+def plot_cml(
+    rf_ann, erp_ann, sigma_mkt,
+    pt_sigma, pt_mu,
+    same_sigma_sigma, same_sigma_mu,
+    same_mu_sigma, same_mu_mu,
+    sugg_sigma=None, sugg_mu=None
+) -> Image.Image:
+    fig = plt.figure(figsize=(6.2, 4.2), dpi=120)
 
     xmax = max(
-        0.3,
+        0.30,
         sigma_mkt * 2.0,
         pt_sigma * 1.4,
-        same_sigma_sigma * 1.4,
         same_mu_sigma * 1.4,
-        (suggestion[0] if suggestion else 0.0) * 1.5,
+        same_sigma_sigma * 1.4,
+        (sugg_sigma or 0.0) * 1.4,
     )
     xs = np.linspace(0, xmax, 160)
     slope = erp_ann / max(sigma_mkt, 1e-12)
     cml = rf_ann + slope * xs
-    plt.plot(xs, cml, label="CML via Market")
-
-    # Points
-    plt.scatter([0.0], [rf_ann], label="Risk-free (FRED)")
-    plt.scatter([sigma_mkt], [rf_ann + erp_ann], label=f"Market {MARKET_TICKER}")
-    plt.scatter([pt_sigma], [pt_mu], label="Your portfolio")
-    plt.scatter([same_sigma_sigma], [same_sigma_mu], label="Efficient same sigma")
-    plt.scatter([same_mu_sigma], [same_mu_mu], label="Efficient same return")
-    if suggestion is not None:
-        plt.scatter([suggestion[0]], [suggestion[1]], marker="X", s=70, label="Suggestion")
-
-    # Guides (percent annotated)
-    plt.plot([pt_sigma, same_sigma_sigma], [pt_mu, same_sigma_mu], ls="--", lw=1.0, alpha=0.7, c="gray")
-    d_ret = (same_sigma_mu - pt_mu) * 100.0
-    plt.annotate(f"Return gain at same σ {d_ret:+.2f}%",
-                 xy=(same_sigma_sigma, same_sigma_mu),
-                 xytext=(same_sigma_sigma, same_sigma_mu + 0.03),
-                 arrowprops=dict(arrowstyle="->", lw=1.0), fontsize=9, ha="center")
-
-    plt.plot([pt_sigma, same_mu_sigma], [pt_mu, same_mu_mu], ls="--", lw=1.0, alpha=0.7, c="gray")
-    d_sig = (same_mu_sigma - pt_sigma) * 100.0
-    plt.annotate(f"Risk change at same μ {d_sig:+.2f}%",
-                 xy=(same_mu_sigma, same_mu_mu),
-                 xytext=(same_mu_sigma + 0.01, same_mu_mu),
-                 arrowprops=dict(arrowstyle="->", lw=1.0), fontsize=9, va="center")
+    plt.plot(xs * 100.0, cml * 100.0, label="CML via Market")
+
+    # key points
+    plt.scatter([0.0], [rf_ann * 100.0], label="Risk-free (FRED)")
+    plt.scatter([sigma_mkt * 100.0], [(rf_ann + erp_ann) * 100.0], label="Market (VOO)")
+    plt.scatter([pt_sigma * 100.0], [pt_mu * 100.0], label="Your portfolio")
+
+    plt.scatter([same_sigma_sigma * 100.0], [same_sigma_mu * 100.0], label="Efficient same sigma")
+    plt.scatter([same_mu_sigma * 100.0], [same_mu_mu * 100.0], label="Efficient same return")
+
+    if sugg_sigma is not None and sugg_mu is not None:
+        plt.scatter([sugg_sigma * 100.0], [sugg_mu * 100.0], label="Suggestion")
+
+    # simple guides
+    plt.plot(
+        [pt_sigma * 100.0, same_sigma_sigma * 100.0],
+        [pt_mu * 100.0, same_sigma_mu * 100.0],
+        linestyle="--", linewidth=1.1, alpha=0.7, color="gray",
+    )
+    plt.plot(
+        [pt_sigma * 100.0, same_mu_sigma * 100.0],
+        [pt_mu * 100.0, same_mu_mu * 100.0],
+        linestyle="--", linewidth=1.1, alpha=0.7, color="gray",
+    )
 
     plt.xlabel("σ (annualized)")
     plt.ylabel("Expected return (annual)")
-    plt.gca().xaxis.set_major_formatter(PercentFormatter(1.0))
-    plt.gca().yaxis.set_major_formatter(PercentFormatter(1.0))
-    plt.legend(loc="best")
+    plt.gca().xaxis.set_major_formatter(lambda v, pos: f"{v:.0f}%")
+    plt.gca().yaxis.set_major_formatter(lambda v, pos: f"{v:.0f}%")
+    plt.legend(loc="best", fontsize=8)
     plt.tight_layout()
 
     buf = io.BytesIO()
@@ -268,161 +270,149 @@ def plot_cml_percent(rf_ann, erp_ann, sigma_mkt,
     buf.seek(0)
     return Image.open(buf)
 
-# ---------------- synthetic dataset ----------------
-def synth_profile(seed: int) -> str:
-    rng = np.random.default_rng(seed)
-    risk = rng.choice(["cautious", "balanced", "moderate", "growth", "aggressive"])
-    horizon = rng.choice(["3y", "5y", "7y", "10y", "15y"])
-    goal = rng.choice(["retirement", "first home", "education", "wealth building", "travel", "emergency"])
-    return f"{risk} investor, {horizon} horizon, goal {goal}"
-
-def build_synthetic_dataset(universe: List[str],
-                            covA: pd.DataFrame,
-                            betas: Dict[str, float],
-                            rf_ann: float,
-                            erp_ann: float,
-                            rows: int = SYNTH_ROWS) -> pd.DataFrame:
-    # Ensure MARKET in universe (we may sample it too)
-    symbols = list(sorted(set(universe + [MARKET_TICKER])))[:MAX_TICKERS]
-    rng = np.random.default_rng(123)
-    data = []
-    for i in range(rows):
-        k = rng.integers(low=min(2, len(symbols)), high=min(8, len(symbols)) + 1)
+# -------------- synthetic dataset --------------
+def _row_to_exposures(row: pd.Series, universe: List[str]) -> Optional[np.ndarray]:
+    try:
+        ts = [t.strip().upper() for t in str(row["tickers"]).split(",") if t.strip()]
+        ws = [float(x) for x in str(row["weights"]).split(",")]
+        wmap = {t: ws[i] for i, t in enumerate(ts) if i < len(ws)}
+        w = np.array([wmap.get(t, 0.0) for t in universe], dtype=float)
+        gross = float(np.sum(np.abs(w)))
+        if gross <= 1e-12:
+            return None
+        return w / gross
+    except Exception:
+        return None
+
+def build_synthetic_dataset(universe: List[str], years: int, rf_ann: float, erp_ann: float) -> pd.DataFrame:
+    symbols = list(sorted(set([s for s in universe if s])))
+    moms = estimate_all_moments_aligned(symbols, years, rf_ann)
+    covA, betas = moms["cov_ann"], moms["betas"]
+
+    rows, rng = [], np.random.default_rng(12345)
+    for i in range(1000):
+        k = int(rng.integers(low=min(2, len(symbols)), high=min(8, len(symbols)) + 1))
         picks = list(rng.choice(symbols, size=k, replace=False))
         signs = rng.choice([-1.0, 1.0], size=k, p=[0.25, 0.75])
         raw = rng.dirichlet(np.ones(k))
-        gross = 1.0 + float(rng.gamma(2.0, 0.5))
+        gross = 1.0 + float(rng.gamma(2.0, 0.7))
         w = gross * signs * raw
-        wmap = {picks[j]: w[j] for j in range(k)}
-
-        beta_p, er_p, sigma_p = portfolio_stats(wmap, covA, betas, rf_ann, erp_ann)
-        data.append({
+        beta_p, er_p, sigma_p = portfolio_stats({picks[j]: w[j] for j in range(k)}, covA, betas, rf_ann, erp_ann)
+        rows.append({
             "id": i,
-            "profile_text": synth_profile(10_000 + i),
             "tickers": ",".join(picks),
-            "weights": ",".join(f"{x:.5f}" for x in w),
+            "weights": ",".join(f"{x:.6f}" for x in w),
             "beta_p": beta_p,
             "er_p": er_p,
             "sigma_p": sigma_p
         })
-    return pd.DataFrame(data)
+    return pd.DataFrame(rows)
 
-def save_synth_csv(df: pd.DataFrame, path: str):
-    os.makedirs(os.path.dirname(path), exist_ok=True)
+def save_synth_csv(df: pd.DataFrame, universe: List[str]) -> str:
+    sig = hashlib.md5((",".join(sorted(universe)) + f":{len(df)}").encode()).hexdigest()[:8]
+    path = os.path.join(DATA_DIR, f"investor_profiles_{sig}.csv")
     df.to_csv(path, index=False)
-
-def _row_to_exposures(row: pd.Series, universe: List[str]) -> Optional[np.ndarray]:
+    return path
+
+# -------------- suggestion logic (dataset only, optional embeddings) --------------
+def describe_candidate_text(row: pd.Series, universe: List[str]) -> str:
+    xs = _row_to_exposures(row, universe)
+    if xs is None:
+        return ""
+    parts = []
+    for t, w in sorted(zip(universe, xs), key=lambda z: -abs(z[1]))[:8]:
+        if abs(w) > 1e-4:
+            parts.append(f"{t} {w:+.2f}")
+    desc = " ".join(parts)
+    return f"weights {desc}; beta {row['beta_p']:.2f}; sigma {row['sigma_p']:.2f}; return {row['er_p']:.2f}"
+
+def pick_by_risk_from_dataset(csv_path: str,
+                              universe: List[str],
+                              risk_label: str,
+                              use_embeddings: bool) -> Optional[Dict]:
     try:
-        ts = [t.strip() for t in str(row["tickers"]).split(",")]
-        ws = [float(x) for x in str(row["weights"]).split(",")]
-        wmap = {t: ws[i] for i, t in enumerate(ts) if i < len(ws)}
-        x = np.array([wmap.get(t, 0.0) for t in universe], dtype=float)
-        gross = float(np.sum(np.abs(x)))
-        if gross <= 1e-12:
-            return None
-        return x / gross
+        df = pd.read_csv(csv_path)
     except Exception:
         return None
+    if df.empty:
+        return None
 
-def candidate_text(weights_map: Dict[str, float], er: float, sigma: float, beta: float) -> str:
-    top = sorted(weights_map.items(), key=lambda kv: -abs(kv[1]))[:6]
-    parts = [f"{k} {v:+.2f}" for k, v in top]
-    return (
-        f"portfolio with expected return {er:.4f}, volatility {sigma:.4f}, beta {beta:.2f}. "
-        f"top exposures: {'; '.join(parts)}"
-    )
-
-def dataset_suggestions(csv_path: str,
-                        universe: List[str],
-                        risk_level: str,
-                        use_embeddings: bool,
-                        top_k: int = 3):
-    try:
-        df = pd.read_csv(csv_path)
-    except Exception:
-        return []
+    # candidates by sigma
+    sigmas = df["sigma_p"].astype(float).values
+    order_low = np.argsort(sigmas)
+    order_high = order_low[::-1]
+    med_value = float(np.median(sigmas))
+    order_mid = np.argsort(np.abs(sigmas - med_value))
+
+    if risk_label.lower() == "low":
+        idxs = order_low[:30]
+    elif risk_label.lower() == "high":
+        idxs = order_high[:30]
+    else:
+        idxs = order_mid[:30]
 
-    # Build rows usable for this universe
-    rows = []
-    for _, r in df.iterrows():
-        x = _row_to_exposures(r, universe)
-        if x is None:
-            continue
-        # recover a printable mapping for display
-        ts = [t.strip() for t in str(r["tickers"]).split(",")]
-        ws = [float(x) for x in str(r["weights"]).split(",")]
-        wmap = {}
-        for i in range(min(len(ts), len(ws))):
-            wmap[ts[i]] = ws[i]
-        gross = sum(abs(v) for v in wmap.values()) or 1.0
-        wmap = {k: v / gross for k, v in wmap.items()}
-        rows.append((wmap, float(r["er_p"]), float(r["sigma_p"]), float(r["beta_p"])))
-
-    if not rows:
-        return []
+    sub = df.iloc[idxs].copy()
+    if sub.empty:
+        return None
 
-    # Risk buckets by sigma
-    sigmas = np.array([r[2] for r in rows])
-    q10, q50, q90 = np.quantile(sigmas, [0.10, 0.50, 0.90])
-
-    if risk_level == "Low":
-        pool = [r for r in rows if r[2] <= q10]
-        target_sigma = q10
-        query = "low risk conservative stable portfolio minimize volatility"
-    elif risk_level == "High":
-        pool = [r for r in rows if r[2] >= q90]
-        target_sigma = q90
-        query = "high risk aggressive growth portfolio accept high volatility maximize returns"
-    else:
-        # Medium around median band
-        band = 0.03  # ±3% absolute sigma band around median
-        pool = [r for r in rows if abs(r[2] - q50) <= band]
-        if not pool:
-            # fallback: closest N to median
-            pool = sorted(rows, key=lambda r: abs(r[2] - q50))[: max(10, top_k)]
-        target_sigma = q50
-        query = "balanced moderate risk diversified portfolio"
-
-    if not pool:
-        # fallback: take closest overall
-        pool = sorted(rows, key=lambda r: abs(r[2] - target_sigma))[: max(10, top_k)]
-
-    # Rank inside pool
-    if use_embeddings and get_embed_model():
+    # optional: rerank with finance embeddings against a risk prompt
+    if use_embeddings:
+        prompt_map = {
+            "low": "low risk, stable, diversified, defensive, downside protection",
+            "medium": "balanced risk, moderate volatility, diversified growth and income",
+            "high": "high risk, aggressive growth, momentum, high volatility"
+        }
+        prompt = prompt_map.get(risk_label.lower(), prompt_map["medium"])
         try:
-            model = get_embed_model()
-            texts = [candidate_text(*r) for r in pool]
-            embs = model.encode([query] + texts, normalize_embeddings=True)
-            qv = embs[0:1]
-            tv = embs[1:]
-            sims = (tv @ qv.T).ravel()
-            ranked = [pool[i] for i in np.argsort(-sims)]
+            from sentence_transformers import SentenceTransformer, util
+            model = SentenceTransformer("FinLang/finance-embeddings-investopedia")
+            cand_texts = [describe_candidate_text(r, universe) for _, r in sub.iterrows()]
+            emb_prompt = model.encode([prompt], normalize_embeddings=True)
+            emb_cands = model.encode(cand_texts, normalize_embeddings=True)
+            sims = util.cos_sim(emb_prompt, emb_cands).cpu().numpy()[0]
+            best_i = int(np.argsort(-sims)[0])
+            chosen = sub.iloc[best_i]
         except Exception:
-            ranked = sorted(pool, key=lambda r: abs(r[2] - target_sigma))
+            chosen = sub.iloc[0]
     else:
-        ranked = sorted(pool, key=lambda r: abs(r[2] - target_sigma))
-
-    picks = ranked[:top_k]
-    out = []
-    for i, (wmap, er, sigma, beta) in enumerate(picks, start=1):
-        # normalize for display
-        gross = sum(abs(v) for v in wmap.values()) or 1.0
-        wmap = {k: v / gross for k, v in wmap.items()}
-        out.append({"pick": i, "weights": wmap, "er": er, "sigma": sigma, "beta": beta})
-    return out
-
-# ---------------- summary ----------------
+        chosen = sub.iloc[0]
+
+    # convert chosen row to exposure map on universe
+    xs = _row_to_exposures(chosen, universe)
+    if xs is None:
+        return None
+    wmap = {t: float(xs[i]) for i, t in enumerate(universe) if abs(xs[i]) > 1e-4}
+    return {"weights": wmap,
+            "er": float(chosen["er_p"]),
+            "sigma": float(chosen["sigma_p"]),
+            "beta": float(chosen["beta_p"])}
+
+def build_simple_suggestion_table(weights_exposure: Dict[str, float],
+                                  gross_capital: float,
+                                  top_n: int = 12) -> pd.DataFrame:
+    rows = []
+    for t, w in sorted(weights_exposure.items(), key=lambda kv: -abs(kv[1]))[:top_n]:
+        rows.append({
+            "ticker": t,
+            "weight_%": round(float(w) * 100.0, 2),
+            "dollars_$": round(float(w) * float(gross_capital), 0)
+        })
+    return pd.DataFrame(rows, columns=["ticker", "weight_%", "dollars_$"])
+
+# -------------- summary builder --------------
+def fmt_pct(x: float) -> str:
+    return f"{x*100:.2f}%"
+
 def build_summary_md(lookback, horizon, rf, rf_code, erp, sigma_mkt,
                      beta_p, er_p, sigma_p,
                      a_sigma, b_sigma, mu_eff_sigma,
                      a_mu, b_mu, sigma_eff_mu,
-                     risk_level: str,
-                     suggestion: Optional[Dict] = None) -> str:
+                     sugg=None, risk_label=None) -> str:
     lines = []
     lines.append("### Inputs")
-    lines.append(f"- Lookback years: **{int(lookback)}**")
+    lines.append(f"- Lookback years: **{lookback}**")
     lines.append(f"- Horizon years: **{int(round(horizon))}**")
-    lines.append(f"- Risk-free: **{fmt_pct(rf)}** from **{rf_code}**")
+    lines.append(f"- Risk-free: **{fmt_pct(rf)}** (FRED {rf_code})")
     lines.append(f"- Market ERP: **{fmt_pct(erp)}**")
     lines.append(f"- Market σ: **{fmt_pct(sigma_mkt)}**")
     lines.append("")
@@ -432,17 +422,25 @@ def build_summary_md(lookback, horizon, rf, rf_code, erp, sigma_mkt,
     lines.append(f"- Expected return: **{fmt_pct(er_p)}**")
     lines.append("")
     lines.append("### Efficient alternatives on CML")
-    lines.append(f"- Same σ: market **{a_sigma:.2f}**, bills **{b_sigma:.2f}**, μ **{fmt_pct(mu_eff_sigma)}**")
-    lines.append(f"- Same μ: market **{a_mu:.2f}**, bills **{b_mu:.2f}**, σ **{fmt_pct(sigma_eff_mu)}**")
-    lines.append("")
-    lines.append(f"### Dataset-based suggestions (risk = **{risk_level}**)")
-    if suggestion:
-        lines.append(f"- Top suggestion μ **{fmt_pct(suggestion['er'])}**, σ **{fmt_pct(suggestion['sigma'])}**, β **{suggestion['beta']:.2f}**")
-    else:
-        lines.append("- No suggestion available.")
+    lines.append("Same σ as your portfolio")
+    lines.append(f"- Market weight **{a_sigma:.2f}**, Bills weight **{b_sigma:.2f}**")
+    lines.append(f"- Expected return **{fmt_pct(mu_eff_sigma)}**")
+    lines.append("Same μ as your portfolio")
+    lines.append(f"- Market weight **{a_mu:.2f}**, Bills weight **{b_mu:.2f}**")
+    lines.append(f"- σ **{fmt_pct(sigma_eff_mu)}**")
+    if sugg is not None:
+        lines.append("")
+        lines.append(f"### Dataset-based suggestion (risk: **{risk_label}**)")
+        lines.append(f"- Suggested β **{sugg['beta']:.2f}**, σ **{fmt_pct(sugg['sigma'])}**, μ **{fmt_pct(sugg['er'])}**")
     return "\n".join(lines)
 
-# ---------------- gradio callbacks ----------------
+# -------------- global state --------------
+UNIVERSE = [MARKET_TICKER, "QQQ", "XLK", "XLP", "XLE", "VNQ", "IEF", "HYG", "GLD", "EEM"]
+HORIZON_YEARS = 10
+RF_CODE = fred_series_for_horizon(HORIZON_YEARS)
+RF_ANN = fetch_fred_yield_annual(RF_CODE)
+
+# -------------- gradio callbacks --------------
 def search_tickers_cb(q: str):
     hits = yahoo_search(q)
     if not hits:
@@ -489,124 +487,107 @@ def set_horizon(years: float):
     HORIZON_YEARS = y
     RF_CODE = code
     RF_ANN = rf
-    return f"Risk free series {code}. Latest annual rate {rf:.2%}. Will be used on compute."
-
-def compute_and_suggest(years_lookback: int,
-                        table: pd.DataFrame,
-                        risk_level: str,
-                        use_embeddings: bool):
-    # sanitize table
-    df = table.dropna()
+    return f"Risk-free series {code}. Latest annual rate {rf:.2%}. Will be used on compute."
+
+def compute(lookback: int,
+            table: pd.DataFrame,
+            risk_label: str,
+            use_embeddings: bool):
+
+    if table is None or len(table) == 0:
+        return None, "Add at least one ticker", "Universe empty", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=["ticker","weight_%","dollars_$"]), None
+
+    df = table.dropna().copy()
     df["ticker"] = df["ticker"].astype(str).str.upper().str.strip()
     df["amount_usd"] = pd.to_numeric(df["amount_usd"], errors="coerce").fillna(0.0)
 
     symbols = [t for t in df["ticker"].tolist() if t]
+    symbols = validate_tickers(symbols, lookback)
     if len(symbols) == 0:
-        return None, "Add at least one ticker", "Universe empty", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=SUG_COLS), None
-
-    symbols = validate_tickers(symbols, years_lookback)
-    if len(symbols) == 0:
-        return None, "Could not validate any tickers", "Universe invalid", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=SUG_COLS), None
+        return None, "Could not validate any tickers", "Universe invalid", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=["ticker","weight_%","dollars_$"]), None
 
-    # Universe includes market
-    universe = list(sorted(set([s for s in symbols] + [MARKET_TICKER])))[:MAX_TICKERS]
+    global UNIVERSE
+    UNIVERSE = list(sorted(set([s for s in symbols])))[:MAX_TICKERS]
 
-    # amounts -> weights
-    dfp = df[df["ticker"].isin(symbols)].copy()
-    amounts = {r["ticker"]: float(r["amount_usd"]) for _, r in dfp.iterrows()}
+    # amounts & gross (gross = sum of absolute exposures)
+    amounts = {r["ticker"]: float(r["amount_usd"]) for _, r in df.iterrows() if r["ticker"] in UNIVERSE}
+    gross = float(sum(abs(v) for v in amounts.values()))
     rf_ann = RF_ANN
 
-    # historical moments
-    moms = estimate_all_moments_aligned(universe, years_lookback, rf_ann)
+    # aligned moments
+    moms = estimate_all_moments_aligned(UNIVERSE, lookback, rf_ann)
     betas, covA, erp_ann, sigma_mkt = moms["betas"], moms["cov_ann"], moms["erp_ann"], moms["sigma_m_ann"]
 
-    gross = sum(abs(v) for v in amounts.values())
-    if gross == 0:
-        return None, "All amounts are zero", "Universe ok", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=SUG_COLS), None
-    weights = {k: v / gross for k, v in amounts.items()}
+    if gross <= 1e-12:
+        return None, "All amounts are zero", f"Universe set to: {', '.join(UNIVERSE)}", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=["ticker","weight_%","dollars_$"]), None
 
+    weights = {k: v / gross for k, v in amounts.items()}
     beta_p, er_p, sigma_p = portfolio_stats(weights, covA, betas, rf_ann, erp_ann)
 
     a_sigma, b_sigma, mu_eff_sigma = efficient_same_sigma(sigma_p, rf_ann, erp_ann, sigma_mkt)
     a_mu, b_mu, sigma_eff_mu = efficient_same_return(er_p, rf_ann, erp_ann, sigma_mkt)
 
-    # Build synthetic dataset for THIS universe each run
-    ds_path = dataset_path_for_universe(universe)
-    synth_df = build_synthetic_dataset(universe, covA, betas, rf_ann, erp_ann, rows=SYNTH_ROWS)
-    save_synth_csv(synth_df, ds_path)
-
-    # Suggestions from dataset (top 3)
-    picks = dataset_suggestions(ds_path, universe, risk_level, use_embeddings, top_k=3)
-
-    # For plot, show first suggestion if any
-    first_sugg = None
-    if picks:
-        first_sugg = (float(picks[0]["sigma"]), float(picks[0]["er"]))
+    # build (or reuse) synthetic dataset for this universe
+    csv_path = None
+    # make a stable filename per-universe
+    sig = hashlib.md5((",".join(sorted(UNIVERSE)) + f":{lookback}:{RF_CODE}").encode()).hexdigest()[:8]
+    candidate_path = os.path.join(DATA_DIR, f"investor_profiles_{sig}.csv")
+    if os.path.exists(candidate_path):
+        csv_path = candidate_path
+    else:
+        synth_df = build_synthetic_dataset(UNIVERSE, years=lookback, rf_ann=rf_ann, erp_ann=erp_ann)
+        csv_path = save_synth_csv(synth_df, UNIVERSE)
+
+    # dataset-based suggestion by risk
+    sug = pick_by_risk_from_dataset(csv_path, UNIVERSE, risk_label=risk_label, use_embeddings=use_embeddings)
+    suggestion_df = pd.DataFrame(columns=["ticker","weight_%","dollars_$"])
+    sugg_sigma_plot = None
+    sugg_mu_plot = None
+    if sug is not None:
+        suggestion_df = build_simple_suggestion_table(sug["weights"], gross_capital=gross)
+        sugg_sigma_plot = sug["sigma"]
+        sugg_mu_plot = sug["er"]
+
+    # positions table (computed from user's inputs)
+    rows = []
+    for t in UNIVERSE:
+        if t in amounts:
+            beta_val = 1.0 if t == moms["mkt"] else betas.get(t, np.nan)
+            rows.append({
+                "ticker": t,
+                "amount_usd": float(amounts.get(t, 0.0)),
+                "weight_exposure": float(weights.get(t, 0.0)),
+                "beta": float(beta_val),
+            })
+    pos_table = pd.DataFrame(rows, columns=POS_COLS)
 
-    img = plot_cml_percent(
+    # plot & summary
+    img = plot_cml(
         rf_ann, erp_ann, sigma_mkt,
         sigma_p, er_p,
         sigma_p, mu_eff_sigma,
         sigma_eff_mu, er_p,
-        suggestion=first_sugg
+        sugg_sigma=sugg_sigma_plot, sugg_mu=sugg_mu_plot
     )
 
-    # Build summary
     info = build_summary_md(
-        years_lookback, HORIZON_YEARS, rf_ann, RF_CODE, erp_ann, sigma_mkt,
+        lookback, HORIZON_YEARS, rf_ann, RF_CODE, erp_ann, sigma_mkt,
         beta_p, er_p, sigma_p,
         a_sigma, b_sigma, mu_eff_sigma,
         a_mu, b_mu, sigma_eff_mu,
-        risk_level=risk_level,
-        suggestion=picks[0] if picks else None
+        sugg=sug, risk_label=risk_label
     )
 
-    # Positions table
-    rows = []
-    for t in symbols:
-        rows.append({
-            "ticker": t,
-            "amount_usd": amounts.get(t, 0.0),
-            "weight_exposure": weights.get(t, 0.0),
-            "beta": 1.0 if t == MARKET_TICKER else betas.get(t, np.nan),
-        })
-    pos_table = pd.DataFrame(rows, columns=POS_COLS)
-
-    # Suggestions table (long format)
-    if picks:
-        sugg_rows = []
-        for p in picks:
-            for k, v in sorted(p["weights"].items(), key=lambda kv: -abs(kv[1]))[:12]:
-                sugg_rows.append({
-                    "pick": p["pick"],
-                    "ticker": k,
-                    "weight_exposure": v,
-                    "er_%": p["er"] * 100.0,
-                    "sigma_%": p["sigma"] * 100.0,
-                    "beta": p["beta"],
-                })
-        sugg_table = pd.DataFrame(sugg_rows, columns=SUG_COLS)
-    else:
-        sugg_table = pd.DataFrame(columns=SUG_COLS)
-
-    uni_msg = f"Universe set to: {', '.join(universe)}"
-    return img, info, uni_msg, pos_table, sugg_table, ds_path
-
-# ---------------- launch UI ----------------
-ensure_data_dir()
-
-# Initialize risk-free from default horizon
-HORIZON_YEARS = 5.0
-RF_CODE = fred_series_for_horizon(HORIZON_YEARS)
-RF_ANN = fetch_fred_yield_annual(RF_CODE)
+    uni_msg = f"Universe set to: {', '.join(UNIVERSE)}"
+    return img, info, uni_msg, pos_table, suggestion_df, csv_path
 
+# -------------- UI --------------
 with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
     gr.Markdown(
         "## Efficient Portfolio Advisor\n"
-        "Search symbols, enter dollar amounts, set your horizon. "
-        "Prices from **Yahoo Finance**. Risk-free from **FRED**. "
-        "Low/Medium/High suggestions are chosen **only** from a 1,000-row dataset generated from your current universe, "
-        "optionally refined with **finance embeddings**."
+        "Search symbols, enter dollar amounts, set your horizon. Prices from Yahoo Finance. Risk-free from FRED. "
+        "Low/Medium/High suggestions are chosen only from a 1,000-row dataset generated from your current universe, "
+        "optionally refined with finance embeddings."
     )
 
     with gr.Row():
@@ -623,22 +604,24 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
                 headers=["ticker", "amount_usd"],
                 datatype=["str", "number"],
                 row_count=0,
-                col_count=(2, "fixed")
+                col_count=(2, "fixed"),
+                wrap=True,
             )
 
-            horizon = gr.Number(label="Horizon in years (1–100)", value=int(HORIZON_YEARS), precision=0)
+            horizon = gr.Number(label="Horizon in years (1–100)", value=HORIZON_YEARS, precision=0)
             lookback = gr.Slider(1, 10, value=DEFAULT_LOOKBACK_YEARS, step=1, label="Lookback years for beta & sigma")
 
             gr.Markdown("### Suggestions")
-            risk_level = gr.Radio(["Low", "Medium", "High"], value="Medium", label="Risk tolerance")
-            use_embeddings = gr.Checkbox(label="Use finance embeddings to refine picks", value=True)
+            risk = gr.Radio(choices=["Low", "Medium", "High"], value="Medium", label="Risk tolerance")
+            use_emb = gr.Checkbox(label="Use finance embeddings to refine picks", value=True)
 
             run_btn = gr.Button("Compute (build dataset & suggest)")
 
         with gr.Column(scale=1):
             plot = gr.Image(label="Capital Market Line (CML)", type="pil")
-            summary = gr.Markdown(label="Summary")
+            summary = gr.Markdown(label="Inputs & Results")
             universe_msg = gr.Textbox(label="Universe status", interactive=False)
+
             positions = gr.Dataframe(
                 label="Computed positions",
                 headers=POS_COLS,
@@ -647,17 +630,18 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
                 value=pd.DataFrame(columns=POS_COLS),
                 interactive=False
             )
+
             suggestions = gr.Dataframe(
-                label="Dataset-based suggestions (top 3 — weights shown as exposures)",
-                headers=SUG_COLS,
-                datatype=["number", "str", "number", "number", "number", "number"],
-                col_count=(len(SUG_COLS), "fixed"),
-                value=pd.DataFrame(columns=SUG_COLS),
+                label="Suggested holdings (weights are % of gross capital; negatives = shorts)",
+                headers=["ticker", "weight_%", "dollars_$"],
+                datatype=["str", "number", "number"],
+                col_count=(3, "fixed"),
+                value=pd.DataFrame(columns=["ticker","weight_%","dollars_$"]),
                 interactive=False
             )
+
             dl = gr.File(label="Generated dataset CSV", value=None, visible=True)
 
-    # Wire up events
     def do_search(query):
         note, options = search_tickers_cb(query)
         return note, gr.update(choices=options)
@@ -668,10 +652,11 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
     horizon.change(fn=set_horizon, inputs=horizon, outputs=universe_msg)
 
     run_btn.click(
-        fn=compute_and_suggest,
-        inputs=[lookback, table, risk_level, use_embeddings],
+        fn=compute,
+        inputs=[lookback, table, risk, use_emb],
         outputs=[plot, summary, universe_msg, positions, suggestions, dl]
     )
 
 if __name__ == "__main__":
     demo.launch()
+```