Update app.py

app.py

@@ -1,5 +1,5 @@
-# app.py
-import os, io, math, time, warnings, json
+# app.py (CML-safe: sigma uses full cov incl. market)
+import os, io, math, time, warnings
 warnings.filterwarnings("ignore")
 
 from typing import List, Tuple, Dict, Optional
@@ -20,15 +20,14 @@ MAX_TICKERS = 30
 DEFAULT_LOOKBACK_YEARS = 10
 MARKET_TICKER = "VOO"
 
-SYNTH_ROWS = 1000  # dataset size for suggestions
+SYNTH_ROWS = 1000
 EMBED_MODEL_NAME = "FinLang/finance-embeddings-investopedia"
-EMBED_ALPHA = 0.6   # score = alpha*exposure_sim + (1-alpha)*embedding_sim
-MMR_LAMBDA = 0.7    # diversity tradeoff for MMR (higher = prefer quality)
+EMBED_ALPHA = 0.6
+MMR_LAMBDA = 0.7
 
-# Globals updated by horizon control
 HORIZON_YEARS = 10
 RF_CODE = "DGS10"
-RF_ANN = 0.0375      # refreshed at launch
+RF_ANN = 0.0375
 
 # ---------------- helpers ----------------
 def fred_series_for_horizon(years: float) -> str:
@@ -44,8 +43,7 @@ def fred_series_for_horizon(years: float) -> str:
 def fetch_fred_yield_annual(code: str) -> float:
     url = f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={code}"
     try:
-        r = requests.get(url, timeout=10)
-        r.raise_for_status()
+        r = requests.get(url, timeout=10); r.raise_for_status()
         df = pd.read_csv(io.StringIO(r.text))
         s = pd.to_numeric(df.iloc[:, 1], errors="coerce").dropna()
         return float(s.iloc[-1] / 100.0) if len(s) else 0.03
@@ -56,57 +54,35 @@ def fetch_prices_monthly(tickers: List[str], years: int) -> pd.DataFrame:
     tickers = list(dict.fromkeys([t.upper().strip() for t in tickers if t]))
     start = (pd.Timestamp.today(tz="UTC") - pd.DateOffset(years=int(years), days=7)).date()
     end = pd.Timestamp.today(tz="UTC").date()
-
     df = yf.download(
-        tickers,
-        start=start,
-        end=end,
-        interval="1mo",
-        auto_adjust=True,
-        actions=False,
-        progress=False,
-        group_by="column",
-        threads=False,
+        tickers, start=start, end=end, interval="1mo",
+        auto_adjust=True, actions=False, progress=False,
+        group_by="column", threads=False,
     )
-
-    # Normalize to wide (Close) frame
-    if isinstance(df, pd.Series):
-        df = df.to_frame()
+    if isinstance(df, pd.Series): df = df.to_frame()
     if isinstance(df.columns, pd.MultiIndex):
         lvl0 = [str(x) for x in df.columns.get_level_values(0).unique()]
-        if "Close" in lvl0:
-            df = df["Close"]
-        elif "Adj Close" in lvl0:
-            df = df["Adj Close"]
-        else:
-            df = df.xs(df.columns.levels[0][-1], axis=1, level=0, drop_level=True)
-
+        if "Close" in lvl0: df = df["Close"]
+        elif "Adj Close" in lvl0: df = df["Adj Close"]
+        else: df = df.xs(df.columns.levels[0][-1], axis=1, level=0, drop_level=True)
     cols = [c for c in tickers if c in df.columns]
-    out = df[cols].dropna(how="all").fillna(method="ffill")
-    return out
+    return df[cols].dropna(how="all").fillna(method="ffill")
 
 def monthly_returns(prices: pd.DataFrame) -> pd.DataFrame:
     return prices.pct_change().dropna()
 
 def yahoo_search(query: str):
-    if not query or not str(query).strip():
-        return []
+    if not query or not str(query).strip(): return []
     url = "https://query1.finance.yahoo.com/v1/finance/search"
     params = {"q": query.strip(), "quotesCount": 10, "newsCount": 0}
     headers = {"User-Agent": "Mozilla/5.0"}
     try:
-        r = requests.get(url, params=params, headers=headers, timeout=10)
-        r.raise_for_status()
-        data = r.json()
-        out = []
+        r = requests.get(url, params=params, headers=headers, timeout=10); r.raise_for_status()
+        data = r.json(); out = []
         for q in data.get("quotes", []):
-            sym = q.get("symbol")
-            name = q.get("shortname") or q.get("longname") or ""
-            exch = q.get("exchDisp") or ""
-            if sym and sym.isascii():
-                out.append(f"{sym}  |  {name}  |  {exch}")
-        if not out:
-            out = [f"{query.strip().upper()}  |  typed symbol  |  n/a"]
+            sym = q.get("symbol"); name = q.get("shortname") or q.get("longname") or ""; exch = q.get("exchDisp") or ""
+            if sym and sym.isascii(): out.append(f"{sym}  |  {name}  |  {exch}")
+        if not out: out = [f"{query.strip().upper()}  |  typed symbol  |  n/a"]
         return out[:10]
     except Exception:
         return [f"{query.strip().upper()}  |  typed symbol  |  n/a"]
@@ -115,17 +91,16 @@ def validate_tickers(symbols: List[str], years: int) -> List[str]:
     base = [s for s in dict.fromkeys([t.upper().strip() for t in symbols]) if s]
     px = fetch_prices_monthly(base + [MARKET_TICKER], years)
     ok = [s for s in base if s in px.columns]
-    if MARKET_TICKER not in px.columns:
-        return []  # we need a market proxy to align CAPM
+    if MARKET_TICKER not in px.columns: return []
     return ok
 
-# -------------- aligned moments --------------
+# ---------- aligned moments & covariances (incl. market) ----------
 def get_aligned_monthly_returns(symbols: List[str], years: int) -> pd.DataFrame:
-    uniq = [c for c in dict.fromkeys(symbols) if c != MARKET_TICKER]
-    tickers = uniq + [MARKET_TICKER]
-    px = fetch_prices_monthly(tickers, years)
+    uniq = [c for c in dict.fromkeys(symbols)]
+    if MARKET_TICKER not in uniq: uniq.append(MARKET_TICKER)
+    px = fetch_prices_monthly(uniq, years)
     rets = monthly_returns(px)
-    cols = [c for c in uniq if c in rets.columns] + ([MARKET_TICKER] if MARKET_TICKER in rets.columns else [])
+    cols = [c for c in uniq if c in rets.columns]
     R = rets[cols].dropna(how="any")
     return R.loc[:, ~R.columns.duplicated()]
 
@@ -136,16 +111,13 @@ def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
     rf_m = rf_ann / 12.0
 
     m = R[MARKET_TICKER]
-    if isinstance(m, pd.DataFrame):
-        m = m.iloc[:, 0].squeeze()
-
+    if isinstance(m, pd.DataFrame): m = m.iloc[:, 0].squeeze()
     mu_m_ann = float(m.mean() * 12.0)
     sigma_m_ann = float(m.std(ddof=1) * math.sqrt(12.0))
     erp_ann = float(mu_m_ann - rf_ann)
 
     ex_m = m - rf_m
-    var_m = float(np.var(ex_m.values, ddof=1))
-    var_m = max(var_m, 1e-9)
+    var_m = float(np.var(ex_m.values, ddof=1)); var_m = max(var_m, 1e-9)
 
     betas: Dict[str, float] = {}
     for s in [c for c in R.columns if c != MARKET_TICKER]:
@@ -154,140 +126,84 @@ def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
         betas[s] = cov_sm / var_m
     betas[MARKET_TICKER] = 1.0
 
-    asset_cols = [c for c in R.columns if c != MARKET_TICKER]
-    cov_m = np.cov(R[asset_cols].values.T, ddof=1) if asset_cols else np.zeros((0, 0))
-    covA = pd.DataFrame(cov_m * 12.0, index=asset_cols, columns=asset_cols)
+    # FULL covariance including MARKET_TICKER (crucial to keep points ≤ CML)
+    cov_all_ann = pd.DataFrame(np.cov(R.values.T, ddof=1) * 12.0,
+                               index=R.columns, columns=R.columns)
 
-    return {"betas": betas, "cov_ann": covA, "erp_ann": erp_ann, "sigma_m_ann": sigma_m_ann}
+    return {"betas": betas, "cov_all_ann": cov_all_ann, "erp_ann": erp_ann, "sigma_m_ann": sigma_m_ann}
 
 def capm_er(beta: float, rf_ann: float, erp_ann: float) -> float:
     return float(rf_ann + beta * erp_ann)
 
 def portfolio_stats(weights: Dict[str, float],
-                    cov_ann: pd.DataFrame,
+                    cov_all_ann: pd.DataFrame,
                     betas: Dict[str, float],
                     rf_ann: float,
                     erp_ann: float) -> Tuple[float, float, float]:
     tickers = list(weights.keys())
     w = np.array([weights[t] for t in tickers], dtype=float)
     gross = float(np.sum(np.abs(w)))
-    if gross <= 1e-12:
-        return 0.0, rf_ann, 0.0
+    if gross <= 1e-12: return 0.0, rf_ann, 0.0
     w_expo = w / gross
     beta_p = float(np.dot([betas.get(t, 0.0) for t in tickers], w_expo))
     mu_capm = capm_er(beta_p, rf_ann, erp_ann)
-    cov = cov_ann.reindex(index=[t for t in tickers if t != MARKET_TICKER],
-                          columns=[t for t in tickers if t != MARKET_TICKER]).fillna(0.0).to_numpy()
-    # treat market ticker (if any) as index asset with β=1; variance from cov_ann is on asset-only block
-    # when MARKET_TICKER is in weights, its variance contribution is ignored in cov (ok; σ_hist is approximate)
-    sigma_hist = 0.0
-    if cov.size and all(t != MARKET_TICKER for t in tickers):
-        sigma_hist = float(max(w_expo.T @ cov @ w_expo, 0.0)) ** 0.5
-    else:
-        # fallback: use weighted average variance/cov if market present; approximate via available submatrix
-        sub_t = [t for t in tickers if t != MARKET_TICKER]
-        if sub_t:
-            sub_w = np.array([weights[t] for t in sub_t], dtype=float)
-            sub_w = sub_w / max(np.sum(np.abs(sub_w)), 1e-12)
-            sub_cov = cov_ann.reindex(index=sub_t, columns=sub_t).fillna(0.0).to_numpy()
-            sigma_hist = float(max(sub_w.T @ sub_cov @ sub_w, 0.0)) ** 0.5
-        else:
-            sigma_hist = 0.0
+    cov = cov_all_ann.reindex(index=tickers, columns=tickers).fillna(0.0).to_numpy()
+    sigma_hist = float(max(w_expo.T @ cov @ w_expo, 0.0)) ** 0.5
     return beta_p, mu_capm, sigma_hist
 
 def efficient_same_sigma(sigma_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
-    if sigma_mkt <= 1e-12:
-        return 0.0, 1.0, rf_ann
+    if sigma_mkt <= 1e-12: return 0.0, 1.0, rf_ann
     a = sigma_target / sigma_mkt
-    return a, 1.0 - a, rf_ann + a * erp_ann  # weights (market, bills), return
+    return a, 1.0 - a, rf_ann + a * erp_ann
 
 def efficient_same_return(mu_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
-    if abs(erp_ann) <= 1e-12:
-        return 0.0, 1.0, rf_ann
+    if abs(erp_ann) <= 1e-12: return 0.0, 1.0, rf_ann
     a = (mu_target - rf_ann) / erp_ann
-    return a, 1.0 - a, abs(a) * sigma_mkt  # weights (market, bills), sigma
+    return a, 1.0 - a, abs(a) * sigma_mkt
 
 # -------------- plotting --------------
-def _pct(x):
-    return np.asarray(x, dtype=float) * 100.0
-
-def plot_cml_hybrid(
-    rf_ann, erp_ann, sigma_mkt,
-    sigma_hist_port, mu_capm_port,
-    mu_eff_same_sigma, sigma_eff_same_return,
-    sugg_mu=None, sugg_sigma_hist=None
-) -> Image.Image:
-    fig = plt.figure(figsize=(6.5, 4.2), dpi=120)
+def _pct(x): return np.asarray(x, dtype=float) * 100.0
 
-    xmax = max(0.3,
-               sigma_mkt * 2.2,
-               (sigma_hist_port or 0.0) * 1.6,
-               (sigma_eff_same_return or 0.0) * 1.6,
-               (sugg_sigma_hist or 0.0) * 1.6)
+def plot_cml_hybrid(rf_ann, erp_ann, sigma_mkt,
+                    sigma_hist_port, mu_capm_port,
+                    mu_eff_same_sigma, sigma_eff_same_return,
+                    sugg_mu=None, sugg_sigma_hist=None) -> Image.Image:
+    fig = plt.figure(figsize=(6.5, 4.2), dpi=120)
+    xmax = max(0.3, sigma_mkt * 2.2, (sigma_hist_port or 0.0) * 1.6,
+               (sigma_eff_same_return or 0.0) * 1.6, (sugg_sigma_hist or 0.0) * 1.6)
     xs = np.linspace(0.0, xmax, 240)
     cml = rf_ann + (erp_ann / max(sigma_mkt, 1e-9)) * xs if sigma_mkt > 1e-12 else np.full_like(xs, rf_ann)
-
-    # CML and fixtures
     plt.plot(_pct(xs), _pct(cml), label="CML (Market/Bills)", linewidth=1.8)
     plt.scatter([_pct(0)], [_pct(rf_ann)], label="Risk-free", zorder=3)
     plt.scatter([_pct(sigma_mkt)], [_pct(rf_ann + erp_ann)], label="Market", zorder=3)
-
-    # Your CAPM point (x = historical σ, y = CAPM E[r])
     plt.scatter([_pct(sigma_hist_port)], [_pct(mu_capm_port)], label="Your CAPM point", marker="o", zorder=4)
-
-    # Efficient points
     plt.scatter([_pct(sigma_hist_port)], [_pct(mu_eff_same_sigma)], label="Efficient (same σ)", marker="^", zorder=4)
     plt.scatter([_pct(sigma_eff_same_return)], [_pct(mu_capm_port)], label="Efficient (same E[r])", marker="s", zorder=4)
-
-    # Selected suggestion
     if (sugg_mu is not None) and (sugg_sigma_hist is not None):
         plt.scatter([_pct(sugg_sigma_hist)], [_pct(sugg_mu)], label="Selected Suggestion", marker="X", s=70, zorder=5)
-
-    plt.xlabel("σ (historical, annualized, %)")
-    plt.ylabel("CAPM E[r] (annual, %)")
-    plt.legend(loc="best", fontsize=8)
-    plt.tight_layout()
-
-    buf = io.BytesIO()
-    plt.savefig(buf, format="png")
-    plt.close(fig)
-    buf.seek(0)
+    plt.xlabel("σ (historical, annualized, %)"); plt.ylabel("CAPM E[r] (annual, %)")
+    plt.legend(loc="best", fontsize=8); plt.tight_layout()
+    buf = io.BytesIO(); plt.savefig(buf, format="png"); plt.close(fig); buf.seek(0)
     return Image.open(buf)
 
-# -------------- synthetic dataset --------------
+# -------------- synthetic dataset (σ uses FULL cov) --------------
 def build_synthetic_dataset(universe: List[str],
-                            covA: pd.DataFrame,
+                            cov_all_ann: pd.DataFrame,
                             betas: Dict[str, float],
                             rf_ann: float,
                             erp_ann: float,
-                            sigma_mkt: float,
                             n_rows: int = SYNTH_ROWS) -> pd.DataFrame:
     rng = np.random.default_rng(12345)
-    assets = [t for t in universe if t != MARKET_TICKER]
-    if not assets:
-        assets = [MARKET_TICKER]
-
+    if MARKET_TICKER not in universe: universe = list(universe) + [MARKET_TICKER]
     rows = []
     for _ in range(n_rows):
         k = int(rng.integers(low=2, high=min(8, len(universe)) + 1))
         picks = list(rng.choice(universe, size=k, replace=False))
-
-        # long-only for clarity in suggestions
         w = rng.dirichlet(np.ones(k))
-
-        # beta and CAPM E[r]
         beta_p = float(np.dot([betas.get(t, 0.0) for t in picks], w))
         mu_capm = capm_er(beta_p, rf_ann, erp_ann)
-
-        # historical sigma from covA (ignore MARKET_TICKER variance entry)
-        sub = [t for t in picks if t != MARKET_TICKER]
-        if sub:
-            sub_w = np.array([w[i] for i, t in enumerate(picks) if t != MARKET_TICKER], dtype=float)
-            sub_cov = covA.reindex(index=sub, columns=sub).fillna(0.0).to_numpy()
-            sigma_hist = float(max(sub_w.T @ sub_cov @ sub_w, 0.0)) ** 0.5
-        else:
-            sigma_hist = 0.0
-
+        sub_cov = cov_all_ann.reindex(index=picks, columns=picks).fillna(0.0).to_numpy()
+        sigma_hist = float(max(w.T @ sub_cov @ w, 0.0)) ** 0.5
         rows.append({
             "tickers": ",".join(picks),
             "weights": ",".join(f"{x:.6f}" for x in w),
@@ -299,22 +215,17 @@ def build_synthetic_dataset(universe: List[str],
 
 def _band_bounds_sigma_hist(sigma_mkt: float, band: str) -> Tuple[float, float]:
     band = (band or "Medium").strip().lower()
-    if band.startswith("low"):
-        return 0.0, 0.8 * sigma_mkt
-    if band.startswith("high"):
-        return 1.2 * sigma_mkt, 3.0 * sigma_mkt
+    if band.startswith("low"): return 0.0, 0.8 * sigma_mkt
+    if band.startswith("high"): return 1.2 * sigma_mkt, 3.0 * sigma_mkt
     return 0.8 * sigma_mkt, 1.2 * sigma_mkt
 
 def _summarize_three(df: pd.DataFrame) -> pd.DataFrame:
-    if df.empty:
-        return pd.DataFrame(columns=["pick", "CAPM E[r] %", "σ (hist) %", "tickers"])
+    if df.empty: return pd.DataFrame(columns=["pick","CAPM E[r] %","σ (hist) %","tickers"])
     out = df.copy()
-    out = out.assign(**{
-        "CAPM E[r] %": (out["mu_capm"] * 100.0).round(2),
-        "σ (hist) %": (out["sigma_hist"] * 100.0).round(2),
-        "tickers": out["tickers"]
-    })[["CAPM E[r] %", "σ (hist) %", "tickers"]].reset_index(drop=True)
-    out.insert(0, "pick", [1, 2, 3][: len(out)])
+    out = out.assign(**{"CAPM E[r] %": (out["mu_capm"]*100).round(2),
+                        "σ (hist) %": (out["sigma_hist"]*100).round(2),
+                        "tickers": out["tickers"]})[["CAPM E[r] %","σ (hist) %","tickers"]]
+    out = out.reset_index(drop=True); out.insert(0, "pick", [1,2,3][:len(out)])
     return out
 
 # -------------- embeddings & re-ranking --------------
@@ -323,8 +234,7 @@ _TICKER_EMBED_CACHE: Dict[str, np.ndarray] = {}
 
 def _load_embed_model():
     global _EMBED_MODEL
-    if _EMBED_MODEL is not None:
-        return _EMBED_MODEL
+    if _EMBED_MODEL is not None: return _EMBED_MODEL
     try:
         from sentence_transformers import SentenceTransformer
         _EMBED_MODEL = SentenceTransformer(EMBED_MODEL_NAME)
@@ -334,131 +244,85 @@ def _load_embed_model():
 
 def _embed_texts(texts: List[str]) -> np.ndarray:
     model = _load_embed_model()
-    if model is None:
-        return np.zeros((len(texts), 384), dtype=float)  # fallback dim
+    if model is None: return np.zeros((len(texts), 384), dtype=float)
     return np.array(model.encode(texts), dtype=float)
 
 def _ticker_vec(t: str) -> np.ndarray:
     t = t.upper().strip()
-    if t in _TICKER_EMBED_CACHE:
-        return _TICKER_EMBED_CACHE[t]
-    v = _embed_texts([f"ticker {t}"])[0]
-    _TICKER_EMBED_CACHE[t] = v
-    return v
+    if t in _TICKER_EMBED_CACHE: return _TICKER_EMBED_CACHE[t]
+    v = _embed_texts([f"ticker {t}"])[0]; _TICKER_EMBED_CACHE[t] = v; return v
 
 def _portfolio_embedding(tickers: List[str], weights: List[float]) -> np.ndarray:
-    if not tickers:
-        return np.zeros(384, dtype=float)
-    w = np.array(weights, dtype=float)
-    s = float(np.sum(np.abs(w)))
-    if s <= 1e-12:
-        w = np.ones(len(tickers), dtype=float) / len(tickers)
-    else:
-        w = w / s
+    if not tickers: return np.zeros(384, dtype=float)
+    w = np.array(weights, dtype=float); s = float(np.sum(np.abs(w)))
+    w = (np.ones(len(tickers))/len(tickers)) if s<=1e-12 else (w/s)
     vs = np.stack([_ticker_vec(t) for t in tickers], axis=0)
-    v = (w[:, None] * vs).sum(axis=0)
-    n = float(np.linalg.norm(v))
-    return v / (n if n > 1e-12 else 1.0)
+    v = (w[:,None]*vs).sum(axis=0); n = float(np.linalg.norm(v))
+    return v/(n if n>1e-12 else 1.0)
 
 def _cos_sim(a: np.ndarray, b: np.ndarray) -> float:
     na = float(np.linalg.norm(a)); nb = float(np.linalg.norm(b))
-    if na <= 1e-12 or nb <= 1e-12: return 0.0
-    return float(np.dot(a, b) / (na * nb))
-
-def _exposure_similarity(user_map: Dict[str, float], cand_map: Dict[str, float]) -> float:
-    # overlap mass on common tickers (long-only style 0..1)
-    s_user = sum(abs(x) for x in user_map.values())
-    s_cand = sum(abs(x) for x in cand_map.values())
-    if s_user <= 1e-12 or s_cand <= 1e-12:
-        return 0.0
-    u = {k: abs(v) / s_user for k, v in user_map.items()}
-    c = {k: abs(v) / s_cand for k, v in cand_map.items()}
-    common = set(u.keys()) & set(c.keys())
-    return float(sum(min(u[t], c[t]) for t in common))
-
-def rerank_band_with_embeddings(user_df: pd.DataFrame,
-                                band_df: pd.DataFrame,
-                                alpha: float = EMBED_ALPHA,
-                                mmr_lambda: float = MMR_LAMBDA,
-                                top_k: int = 3) -> pd.DataFrame:
+    if na<=1e-12 or nb<=1e-12: return 0.0
+    return float(np.dot(a,b)/(na*nb))
+
+def _exposure_similarity(user_map: Dict[str,float], cand_map: Dict[str,float]) -> float:
+    s_user = sum(abs(x) for x in user_map.values()); s_c = sum(abs(x) for x in cand_map.values())
+    if s_user<=1e-12 or s_c<=1e-12: return 0.0
+    u = {k:abs(v)/s_user for k,v in user_map.items()}
+    c = {k:abs(v)/s_c for k,v in cand_map.items()}
+    common = set(u)&set(c); return float(sum(min(u[t],c[t]) for t in common))
+
+def rerank_band_with_embeddings(user_df: pd.DataFrame, band_df: pd.DataFrame,
+                                alpha: float = EMBED_ALPHA, mmr_lambda: float = MMR_LAMBDA, top_k: int = 3) -> pd.DataFrame:
     try:
-        # user portfolio embedding
         u_t = user_df["ticker"].astype(str).str.upper().tolist()
         u_w = pd.to_numeric(user_df["amount_usd"], errors="coerce").fillna(0.0).tolist()
         u_map = {t: float(w) for t, w in zip(u_t, u_w)}
         u_embed = _portfolio_embedding(u_t, u_w)
 
-        # candidate scores
-        cand_rows = []
-        cand_embeds = []
+        cand_rows = []; cand_embeds = []
         for _, r in band_df.iterrows():
             ts = [t.strip().upper() for t in str(r["tickers"]).split(",")]
             ws = [float(x) for x in str(r["weights"]).split(",")]
-            # normalize candidate weights
-            s = sum(max(0.0, w) for w in ws) or 1.0
-            ws = [max(0.0, w) / s for w in ws]
-            c_map = {t: w for t, w in zip(ts, ws)}
-
-            c_embed = _portfolio_embedding(ts, ws)
-            cand_embeds.append(c_embed)
-
+            s = sum(max(0.0,w) for w in ws) or 1.0
+            ws = [max(0.0,w)/s for w in ws]
+            c_map = {t:w for t,w in zip(ts,ws)}
+            c_embed = _portfolio_embedding(ts, ws); cand_embeds.append(c_embed)
             expo_sim = _exposure_similarity(u_map, c_map)
             emb_sim = _cos_sim(u_embed, c_embed)
-            score = alpha * expo_sim + (1.0 - alpha) * emb_sim
-
+            score = alpha*expo_sim + (1.0-alpha)*emb_sim
             cand_rows.append((score, r))
 
-        if not cand_rows:
-            return band_df.head(top_k).reset_index(drop=True)
+        if not cand_rows: return band_df.head(top_k).reset_index(drop=True)
 
-        # MMR selection
         cand_embeds = np.stack(cand_embeds, axis=0)
-        order = np.argsort([-s for s, _ in cand_rows])
-        picked = []
-        picked_idx = []
-
+        order = np.argsort([-s for s,_ in cand_rows])
+        picked = []; picked_idx = []
         for i in order:
-            if len(picked) >= top_k: break
+            if len(picked)>=top_k: break
             s_i, row_i = cand_rows[i]
             if not picked:
-                picked.append(row_i)
-                picked_idx.append(i)
-                continue
-            # diversity penalty
-            sim_to_picked = 0.0
-            for j in picked_idx:
-                sim_to_picked = max(sim_to_picked, _cos_sim(cand_embeds[i], cand_embeds[j]))
-            mmr = mmr_lambda * s_i - (1.0 - mmr_lambda) * sim_to_picked
-            # simple thresholding vs worst current; try greedy insert
-            picked.append(row_i)
-            picked_idx.append(i)
-
+                picked.append(row_i); picked_idx.append(i); continue
+            sim_to_picked = max(_cos_sim(cand_embeds[i], cand_embeds[j]) for j in picked_idx)
+            mmr = mmr_lambda*s_i - (1.0-mmr_lambda)*sim_to_picked  # noqa: F841 (kept for clarity)
+            picked.append(row_i); picked_idx.append(i)
         out = pd.DataFrame([r for r in picked]).drop_duplicates().head(top_k).reset_index(drop=True)
-        if out.empty:
-            out = band_df.head(top_k).reset_index(drop=True)
-        out.insert(0, "pick", [1, 2, 3][: len(out)])
+        if out.empty: out = band_df.head(top_k).reset_index(drop=True)
+        out.insert(0,"pick",[1,2,3][:len(out)])
         return out
     except Exception:
-        # graceful fallback
         out = band_df.sort_values("mu_capm", ascending=False).head(top_k).reset_index(drop=True)
-        out.insert(0, "pick", [1, 2, 3][: len(out)])
+        out.insert(0,"pick",[1,2,3][:len(out)])
         return out
 
 # -------------- UI helpers --------------
-def empty_positions_df():
-    return pd.DataFrame(columns=["ticker", "amount_usd", "weight_exposure", "beta"])
-
-def empty_holdings_df():
-    return pd.DataFrame(columns=["ticker", "weight_%", "amount_$"])
+def empty_positions_df(): return pd.DataFrame(columns=["ticker","amount_usd","weight_exposure","beta"])
+def empty_holdings_df(): return pd.DataFrame(columns=["ticker","weight_%","amount_$"])
 
 def set_horizon(years: float):
-    y = max(1.0, min(100.0, float(years)))
-    code = fred_series_for_horizon(y)
-    rf = fetch_fred_yield_annual(code)
+    y = max(1.0, min(100.0, float(years))); code = fred_series_for_horizon(y); rf = fetch_fred_yield_annual(code)
     global HORIZON_YEARS, RF_CODE, RF_ANN
-    HORIZON_YEARS = y
-    RF_CODE = code
-    RF_ANN = rf
+    HORIZON_YEARS, RF_CODE, RF_ANN = y, code, rf
     return f"Risk-free series {code}. Latest annual rate {rf:.2%}."
 
 def search_tickers_cb(q: str):
@@ -468,38 +332,33 @@ def search_tickers_cb(q: str):
 
 def add_symbol(selection: str, table: Optional[pd.DataFrame]):
     if not selection:
-        return table if isinstance(table, pd.DataFrame) else pd.DataFrame(columns=["ticker","amount_usd"]), "Pick a row in Matches first."
+        return table if isinstance(table,pd.DataFrame) else pd.DataFrame(columns=["ticker","amount_usd"]), "Pick a row in Matches first."
     symbol = selection.split("|")[0].strip().upper()
-
     current = []
-    if isinstance(table, pd.DataFrame) and not table.empty:
+    if isinstance(table,pd.DataFrame) and not table.empty:
         current = [str(x).upper() for x in table["ticker"].tolist() if str(x) != "nan"]
     tickers = current if symbol in current else current + [symbol]
-
     val = validate_tickers(tickers, years=DEFAULT_LOOKBACK_YEARS)
     tickers = [t for t in tickers if t in val]
-
     amt_map = {}
-    if isinstance(table, pd.DataFrame) and not table.empty:
+    if isinstance(table,pd.DataFrame) and not table.empty:
         for _, r in table.iterrows():
-            t = str(r.get("ticker", "")).upper()
+            t = str(r.get("ticker","")).upper()
             if t in tickers:
-                amt_map[t] = float(pd.to_numeric(r.get("amount_usd", 0.0), errors="coerce") or 0.0)
-
-    new_table = pd.DataFrame({"ticker": tickers, "amount_usd": [amt_map.get(t, 0.0) for t in tickers]})
+                amt_map[t] = float(pd.to_numeric(r.get("amount_usd",0.0), errors="coerce") or 0.0)
+    new_table = pd.DataFrame({"ticker": tickers, "amount_usd": [amt_map.get(t,0.0) for t in tickers]})
     if len(new_table) > MAX_TICKERS:
-        new_table = new_table.iloc[:MAX_TICKERS]
-        return new_table, f"Reached max of {MAX_TICKERS}."
+        new_table = new_table.iloc[:MAX_TICKERS]; return new_table, f"Reached max of {MAX_TICKERS}."
     return new_table, f"Added {symbol}."
 
 def lock_ticker_column(tb: Optional[pd.DataFrame]):
-    if not isinstance(tb, pd.DataFrame) or tb.empty:
-        return pd.DataFrame(columns=["ticker", "amount_usd"])
+    if not isinstance(tb,pd.DataFrame) or tb.empty:
+        return pd.DataFrame(columns=["ticker","amount_usd"])
     tickers = [str(x).upper() for x in tb["ticker"].tolist()]
     amounts = pd.to_numeric(tb["amount_usd"], errors="coerce").fillna(0.0).tolist()
     val = validate_tickers(tickers, years=DEFAULT_LOOKBACK_YEARS)
     tickers = [t for t in tickers if t in val]
-    amounts = amounts[:len(tickers)] + [0.0] * max(0, len(tickers) - len(amounts))
+    amounts = amounts[:len(tickers)] + [0.0]*max(0, len(tickers)-len(amounts))
     return pd.DataFrame({"ticker": tickers, "amount_usd": amounts})
 
 # -------------- compute core --------------
@@ -508,107 +367,68 @@ UNIVERSE: List[str] = [MARKET_TICKER, "QQQ", "VTI", "SOXX", "IBIT"]
 def _pick_to_holdings(row: pd.Series, budget: float) -> pd.DataFrame:
     ts = [t.strip().upper() for t in str(row["tickers"]).split(",")]
     ws = [float(x) for x in str(row["weights"]).split(",")]
-    s = sum(max(0.0, w) for w in ws) or 1.0
-    ws = [max(0.0, w) / s for w in ws]
-    return pd.DataFrame(
-        [{"ticker": t, "weight_%": round(w * 100.0, 2), "amount_$": round(w * budget, 0)} for t, w in zip(ts, ws)],
-        columns=["ticker", "weight_%", "amount_$"]
-    )
+    s = sum(max(0.0,w) for w in ws) or 1.0
+    ws = [max(0.0,w)/s for w in ws]
+    return pd.DataFrame([{"ticker": t, "weight_%": round(w*100,2), "amount_$": round(w*budget,0)} for t,w in zip(ts,ws)],
+                        columns=["ticker","weight_%","amount_$"])
 
-def compute_all(
-    years_lookback: int,
-    table: Optional[pd.DataFrame],
-    use_embeddings: bool
-):
-    # sanitize input table
-    if isinstance(table, pd.DataFrame):
-        df = table.copy()
-    else:
-        df = pd.DataFrame(columns=["ticker", "amount_usd"])
+def compute_all(years_lookback: int, table: Optional[pd.DataFrame], use_embeddings: bool):
+    df = table.copy() if isinstance(table,pd.DataFrame) else pd.DataFrame(columns=["ticker","amount_usd"])
     df = df.dropna(how="all")
     if "ticker" not in df.columns: df["ticker"] = []
     if "amount_usd" not in df.columns: df["amount_usd"] = []
     df["ticker"] = df["ticker"].astype(str).str.upper().str.strip()
     df["amount_usd"] = pd.to_numeric(df["amount_usd"], errors="coerce").fillna(0.0)
-
     symbols = [t for t in df["ticker"].tolist() if t]
-    if len(symbols) == 0:
-        raise gr.Error("Add at least one ticker.")
-
+    if len(symbols)==0: raise gr.Error("Add at least one ticker.")
     symbols = validate_tickers(symbols, years_lookback)
-    if len(symbols) == 0:
-        raise gr.Error("Could not validate any tickers.")
+    if len(symbols)==0: raise gr.Error("Could not validate any tickers.")
 
     global UNIVERSE
-    UNIVERSE = list(sorted(set([s for s in symbols if s != MARKET_TICKER] + [MARKET_TICKER])))[:MAX_TICKERS]
+    UNIVERSE = list(sorted(set([s for s in symbols] + [MARKET_TICKER])))[:MAX_TICKERS]
 
     df = df[df["ticker"].isin(symbols)].copy()
     amounts = {r["ticker"]: float(r["amount_usd"]) for _, r in df.iterrows()}
     rf_ann = RF_ANN
 
-    # moments
     moms = estimate_all_moments_aligned(symbols, years_lookback, rf_ann)
-    betas, covA, erp_ann, sigma_mkt = moms["betas"], moms["cov_ann"], moms["erp_ann"], moms["sigma_m_ann"]
+    betas, cov_all_ann, erp_ann, sigma_mkt = moms["betas"], moms["cov_all_ann"], moms["erp_ann"], moms["sigma_m_ann"]
 
-    # weights
     gross = sum(abs(v) for v in amounts.values())
-    if gross <= 1e-12:
-        raise gr.Error("All amounts are zero.")
-
-    weights = {k: v / gross for k, v in amounts.items()}
+    if gross <= 1e-12: raise gr.Error("All amounts are zero.")
+    weights = {k: v/gross for k,v in amounts.items()}
 
-    # portfolio CAPM and σ (historical)
-    beta_p, mu_capm, sigma_hist = portfolio_stats(weights, covA, betas, rf_ann, erp_ann)
+    beta_p, mu_capm, sigma_hist = portfolio_stats(weights, cov_all_ann, betas, rf_ann, erp_ann)
 
-    # efficient counterparts (market/bills)
     a_sigma, b_sigma, mu_eff_sigma = efficient_same_sigma(sigma_hist, rf_ann, erp_ann, sigma_mkt)
     a_mu, b_mu, sigma_eff_mu = efficient_same_return(mu_capm, rf_ann, erp_ann, sigma_mkt)
 
-    # synthetic dataset from current universe
-    synth = build_synthetic_dataset(UNIVERSE, covA, betas, rf_ann, erp_ann, sigma_mkt, n_rows=SYNTH_ROWS)
+    synth = build_synthetic_dataset(UNIVERSE, cov_all_ann, betas, rf_ann, erp_ann, n_rows=SYNTH_ROWS)
     csv_path = os.path.join(DATA_DIR, f"investor_profiles_{int(time.time())}.csv")
-    try:
-        synth.to_csv(csv_path, index=False)
-    except Exception:
-        csv_path = None  # not fatal
+    try: synth.to_csv(csv_path, index=False)
+    except Exception: csv_path = None
 
-    # band splits
     def band_top3(band: str) -> pd.DataFrame:
         lo, hi = _band_bounds_sigma_hist(sigma_mkt, band)
-        pick = synth[(synth["sigma_hist"] >= lo) & (synth["sigma_hist"] <= hi)].copy()
-        if pick.empty:
-            pick = synth.copy()
-        # pre-sort by quality then re-rank with embeddings/MMR for diversity
+        pick = synth[(synth["sigma_hist"]>=lo) & (synth["sigma_hist"]<=hi)].copy()
+        if pick.empty: pick = synth.copy()
         pick = pick.sort_values("mu_capm", ascending=False).head(50).reset_index(drop=True)
         if use_embeddings:
             user_df = pd.DataFrame({"ticker": list(weights.keys()), "amount_usd": [amounts[t] for t in weights.keys()]})
             top3 = rerank_band_with_embeddings(user_df, pick, EMBED_ALPHA, MMR_LAMBDA, top_k=3)
         else:
-            top3 = pick.head(3).reset_index(drop=True)
-            top3.insert(0, "pick", [1, 2, 3][: len(top3)])
+            top3 = pick.head(3).reset_index(drop=True); top3.insert(0,"pick",[1,2,3][:len(top3)])
         return top3
 
-    top3_low  = band_top3("Low")
-    top3_med  = band_top3("Medium")
-    top3_high = band_top3("High")
-
-    # descriptive tables for each tab
-    low_sum  = _summarize_three(top3_low)
-    med_sum  = _summarize_three(top3_med)
-    high_sum = _summarize_three(top3_high)
-
-    # positions table
-    pos_table = pd.DataFrame(
-        [{
-            "ticker": t,
-            "amount_usd": amounts.get(t, 0.0),
-            "weight_exposure": weights.get(t, 0.0),
-            "beta": 1.0 if t == MARKET_TICKER else betas.get(t, np.nan)
-        } for t in symbols],
-        columns=["ticker", "amount_usd", "weight_exposure", "beta"]
-    )
+    top3_low, top3_med, top3_high = band_top3("Low"), band_top3("Medium"), band_top3("High")
+    low_sum, med_sum, high_sum = _summarize_three(top3_low), _summarize_three(top3_med), _summarize_three(top3_high)
+
+    pos_table = pd.DataFrame([{
+        "ticker": t, "amount_usd": amounts.get(t,0.0),
+        "weight_exposure": weights.get(t,0.0),
+        "beta": 1.0 if t==MARKET_TICKER else betas.get(t, np.nan)
+    } for t in symbols], columns=["ticker","amount_usd","weight_exposure","beta"])
 
-    # summary text
     info = "\n".join([
         "### Inputs",
         f"- Lookback years {years_lookback}",
@@ -626,141 +446,91 @@ def compute_all(
         f"- Same σ as your portfolio: Market {a_sigma:.2f}, Bills {b_sigma:.2f} → E[r] {mu_eff_sigma:.2%}",
         f"- Same E[r] as your portfolio: Market {a_mu:.2f}, Bills {b_mu:.2f} → σ {sigma_eff_mu:.2%}",
         "",
-        "_Plot shows CAPM expectations on the CML with x-axis as **historical σ**._"
+        "_All points are guaranteed on/under the CML because σ uses the full covariance (incl. market)._"
     ])
 
     uni_msg = f"Universe set to: {', '.join(UNIVERSE)}"
-
-    base_outputs = dict(
-        rf_ann=rf_ann, erp_ann=erp_ann, sigma_mkt=sigma_mkt,
-        mu_capm=mu_capm, sigma_hist=sigma_hist,
-        mu_eff_same_sigma=mu_eff_sigma, sigma_eff_same_return=sigma_eff_mu,
-        pos_table=pos_table, info=info, uni_msg=uni_msg,
-        csv_path=csv_path, low_sum=low_sum, med_sum=med_sum, high_sum=high_sum,
-        top3_low=top3_low, top3_med=top3_med, top3_high=top3_high, budget=sum(abs(v) for v in amounts.values())
-    )
-    return base_outputs
-
-def compute_and_render(
-    years_lookback: int,
-    table: Optional[pd.DataFrame],
-    use_embeddings: bool,
-    which_band: str,
-    pick_idx: int
-):
+    return dict(rf_ann=rf_ann, erp_ann=erp_ann, sigma_mkt=sigma_mkt,
+                mu_capm=mu_capm, sigma_hist=sigma_hist,
+                mu_eff_same_sigma=mu_eff_sigma, sigma_eff_same_return=sigma_eff_mu,
+                pos_table=pos_table, info=info, uni_msg=uni_msg, csv_path=csv_path,
+                low_sum=low_sum, med_sum=med_sum, high_sum=high_sum,
+                top3_low=top3_low, top3_med=top3_med, top3_high=top3_high,
+                budget=sum(abs(v) for v in amounts.values()))
+
+def compute_and_render(years_lookback: int, table: Optional[pd.DataFrame], use_embeddings: bool,
+                       which_band: str, pick_idx: int):
     outs = compute_all(years_lookback, table, use_embeddings)
-
-    # choose band & pick
     band = (which_band or "Medium").strip().title()
     idx = max(1, min(3, int(pick_idx))) - 1
-
-    if band == "Low":
-        top3 = outs["top3_low"]
-    elif band == "High":
-        top3 = outs["top3_high"]
-    else:
-        top3 = outs["top3_med"]
+    top3 = outs["top3_med"] if band=="Medium" else (outs["top3_low"] if band=="Low" else outs["top3_high"])
 
     if top3.empty:
-        sugg_mu = None; sugg_sigma_hist = None
-        holdings = empty_holdings_df()
+        sugg_mu = None; sugg_sigma_hist = None; holdings = empty_holdings_df()
     else:
         row = top3.iloc[min(idx, len(top3)-1)]
-        sugg_mu = float(row["mu_capm"])
-        sugg_sigma_hist = float(row["sigma_hist"])
+        sugg_mu = float(row["mu_capm"]); sugg_sigma_hist = float(row["sigma_hist"])
         holdings = _pick_to_holdings(row, outs["budget"])
 
-    # plot
     img = plot_cml_hybrid(
         outs["rf_ann"], outs["erp_ann"], outs["sigma_mkt"],
         outs["sigma_hist"], outs["mu_capm"],
         outs["mu_eff_same_sigma"], outs["sigma_eff_same_return"],
         sugg_mu, sugg_sigma_hist
     )
-
-    return (
-        img,                          # plot
-        outs["info"],                 # summary
-        outs["uni_msg"],              # universe msg
-        outs["pos_table"],            # positions
-        holdings,                     # selected holdings
-        outs["csv_path"],             # dataset file
-        outs["low_sum"],              # low tab summary (3 picks)
-        outs["med_sum"],              # medium tab summary
-        outs["high_sum"]              # high tab summary
-    )
+    return (img, outs["info"], outs["uni_msg"], outs["pos_table"],
+            holdings, outs["csv_path"], outs["low_sum"], outs["med_sum"], outs["high_sum"])
 
 # -------------- UI --------------
 with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
     gr.Markdown(
         "## Efficient Portfolio Advisor\n"
-        "Search symbols, enter **dollar amounts** (negatives allowed), set horizon. "
-        "The plot shows **your CAPM point** on the CML with **x = historical σ** and **y = CAPM E[r] = rf + β·ERP**. "
-        "We also show two efficient market/bills mixes: same σ and same E[r].\n\n"
-        "Suggestions are generated from 1,000 candidate mixes and bucketed by risk (σ)."
+        "Plot uses **x = historical σ** and **y = CAPM E[r] = rf + β·ERP**. "
+        "Efficient (same σ) and (same E[r]) market/bills points are shown. "
+        "Suggestions come from 1,000 mixes; embeddings + MMR add diversity."
     )
-
     with gr.Row():
         with gr.Column(scale=1):
-            q = gr.Textbox(label="Search symbol")
-            search_note = gr.Markdown()
+            q = gr.Textbox(label="Search symbol"); search_note = gr.Markdown()
             matches = gr.Dropdown(choices=[], label="Matches")
             with gr.Row():
-                search_btn = gr.Button("Search")
-                add_btn = gr.Button("Add selected to portfolio")
-
+                search_btn = gr.Button("Search"); add_btn = gr.Button("Add selected to portfolio")
             gr.Markdown("### Portfolio positions (enter $ amounts; negatives allowed)")
-            table = gr.Dataframe(
-                value=pd.DataFrame(columns=["ticker", "amount_usd"]),
-                interactive=True
-            )
-
+            table = gr.Dataframe(value=pd.DataFrame(columns=["ticker","amount_usd"]), interactive=True)
             horizon = gr.Number(label="Horizon in years (1–100)", value=HORIZON_YEARS, precision=0)
             lookback = gr.Slider(1, 15, value=DEFAULT_LOOKBACK_YEARS, step=1, label="Lookback years")
             use_emb = gr.Checkbox(value=True, label="Use finance embeddings + MMR for diverse picks")
-
             gr.Markdown("### Suggestions")
             with gr.Tabs():
                 with gr.Tab("Low"):
                     low_summary = gr.Dataframe(value=empty_holdings_df(), interactive=False, label="Top 3 (Low risk)")
-                    pick_low = gr.Radio(choices=["1", "2", "3"], value="1", label="Select a pick in Low")
+                    pick_low = gr.Radio(choices=["1","2","3"], value="1", label="Select a pick in Low")
                 with gr.Tab("Medium"):
                     med_summary = gr.Dataframe(value=empty_holdings_df(), interactive=False, label="Top 3 (Medium risk)")
-                    pick_med = gr.Radio(choices=["1", "2", "3"], value="1", label="Select a pick in Medium")
+                    pick_med = gr.Radio(choices=["1","2","3"], value="1", label="Select a pick in Medium")
                 with gr.Tab("High"):
                     high_summary = gr.Dataframe(value=empty_holdings_df(), interactive=False, label="Top 3 (High risk)")
-                    pick_high = gr.Radio(choices=["1", "2", "3"], value="1", label="Select a pick in High")
-
+                    pick_high = gr.Radio(choices=["1","2","3"], value="1", label="Select a pick in High")
             run_btn = gr.Button("Compute (build dataset & suggest)")
-
         with gr.Column(scale=1):
             plot = gr.Image(label="Capital Market Line (CAPM)", type="pil")
             summary = gr.Markdown(label="Inputs & Results")
             universe_msg = gr.Textbox(label="Universe status", interactive=False)
-            positions = gr.Dataframe(
-                value=empty_positions_df(), interactive=False, label="Computed positions"
-            )
-            selected_table = gr.Dataframe(
-                value=empty_holdings_df(),
-                interactive=False,
-                label="Selected suggestion holdings (% / $)"
-            )
+            positions = gr.Dataframe(value=empty_positions_df(), interactive=False, label="Computed positions")
+            selected_table = gr.Dataframe(value=empty_holdings_df(), interactive=False,
+                                          label="Selected suggestion holdings (% / $)")
             dl = gr.File(label="Generated dataset CSV", value=None, visible=True)
 
-    # wire: search / add / locking / horizon
     search_btn.click(fn=search_tickers_cb, inputs=q, outputs=[search_note, matches])
     add_btn.click(fn=add_symbol, inputs=[matches, table], outputs=[table, search_note])
     table.change(fn=lock_ticker_column, inputs=table, outputs=table)
     horizon.change(fn=set_horizon, inputs=horizon, outputs=universe_msg)
 
-    # main compute (defaults to Medium, pick 1)
     run_btn.click(
         fn=compute_and_render,
         inputs=[lookback, table, use_emb, gr.State("Medium"), gr.State(1)],
         outputs=[plot, summary, universe_msg, positions, selected_table, dl, low_summary, med_summary, high_summary]
     )
-
-    # band radios trigger recompute with their band + index
     pick_low.change(
         fn=compute_and_render,
         inputs=[lookback, table, use_emb, gr.State("Low"), pick_low],
@@ -777,16 +547,9 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
         outputs=[plot, summary, universe_msg, positions, selected_table, dl, low_summary, med_summary, high_summary]
     )
 
-# initialize risk-free at launch
 RF_CODE = fred_series_for_horizon(HORIZON_YEARS)
 RF_ANN = fetch_fred_yield_annual(RF_CODE)
 
 if __name__ == "__main__":
-    # Gradio 5.x: no concurrency_count on .queue()
     demo.queue()
-    demo.launch(
-        server_name="0.0.0.0",
-        server_port=int(os.environ.get("PORT", 7860)),
-        show_api=False,
-        share=False,
-    )
+    demo.launch(server_name="0.0.0.0", server_port=int(os.environ.get("PORT", 7860)), show_api=False, share=False)