Update app.py

app.py

@@ -1,134 +1,707 @@
-# app.py - Part 1
+import os, io, math, warnings
+warnings.filterwarnings("ignore")
+
+from typing import List, Tuple, Dict, Optional
 
-import pandas as pd
 import numpy as np
-import yfinance as yf
+import pandas as pd
+import matplotlib.pyplot as plt
 import gradio as gr
-from itertools import combinations_with_replacement
-
-# -------------------
-# Helper functions
-# -------------------
-
-def fetch_live_data(tickers, period="1y"):
-    """Fetch historical adjusted close prices for given tickers."""
-    data = yf.download(tickers, period=period)["Adj Close"]
-    return data.dropna()
-
-def calculate_portfolio_metrics(weights, mean_returns, cov_matrix, risk_free_rate=0.045):
-    """Return expected portfolio return, volatility, and beta."""
-    weights = np.array(weights)
-    portfolio_return = np.sum(mean_returns * weights)
-    portfolio_volatility = np.sqrt(np.dot(weights.T, np.dot(cov_matrix, weights)))
-    beta = np.sum(weights)  # Placeholder if no real beta calc
-    return portfolio_return, portfolio_volatility, beta
-
-def generate_synthetic_portfolios(tickers, num_portfolios=1000):
-    """Generate synthetic portfolios from live data for given tickers."""
-    df_prices = fetch_live_data(tickers)
-    returns = df_prices.pct_change().dropna()
-    mean_returns = returns.mean()
-    cov_matrix = returns.cov()
-
-    synthetic_data = []
-    for _ in range(num_portfolios):
-        weights = np.random.random(len(tickers))
-        weights /= np.sum(weights)
-        er, sigma, beta = calculate_portfolio_metrics(weights, mean_returns, cov_matrix)
-        synthetic_data.append({
-            "weights": weights,
-            "er_p": er,
-            "sigma_p": sigma,
-            "beta_p": beta
+from PIL import Image
+import requests
+import yfinance as yf
+
+from sklearn.neighbors import KNeighborsRegressor
+from sklearn.preprocessing import StandardScaler
+
+# ---------------- config ----------------
+DATA_DIR = "data"
+DATASET_PATH = os.path.join(DATA_DIR, "investor_profiles.csv")
+
+MAX_TICKERS = 30
+DEFAULT_LOOKBACK_YEARS = 5
+MARKET_TICKER = "VOO"
+
+POS_COLS = ["ticker", "amount_usd", "weight_exposure", "beta"]
+SUG_COLS = ["ticker", "suggested_weight_exposure"]
+
+FRED_MAP = [
+    (1,  "DGS1"),
+    (2,  "DGS2"),
+    (3,  "DGS3"),
+    (5,  "DGS5"),
+    (7,  "DGS7"),
+    (10, "DGS10"),
+    (20, "DGS20"),
+    (30, "DGS30"),
+    (100, "DGS30"),
+]
+
+# ---------------- helpers ----------------
+def ensure_data_dir():
+    os.makedirs(DATA_DIR, exist_ok=True)
+
+def empty_positions_df():
+    return pd.DataFrame(columns=POS_COLS)
+
+def empty_suggest_df():
+    return pd.DataFrame(columns=SUG_COLS)
+
+def fred_series_for_horizon(years: float) -> str:
+    y = max(1.0, min(100.0, float(years)))
+    for cutoff, code in FRED_MAP:
+        if y <= cutoff:
+            return code
+    return "DGS30"
+
+def fetch_fred_yield_annual(code: str) -> float:
+    # FRED CSV endpoint
+    url = f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={code}"
+    try:
+        r = requests.get(url, timeout=10)
+        r.raise_for_status()
+        df = pd.read_csv(io.StringIO(r.text))
+        s = pd.to_numeric(df.iloc[:, 1], errors="coerce").dropna()
+        return float(s.iloc[-1] / 100.0) if len(s) else 0.03
+    except Exception:
+        return 0.03
+
+def fetch_prices_monthly(tickers: List[str], years: int) -> pd.DataFrame:
+    """
+    Fetch monthly adjusted Close for given tickers. Handles both single-ticker (Series)
+    and multi-ticker (DataFrame) returns from yfinance and ensures columns are ticker symbols.
+    """
+    if not tickers:
+        return pd.DataFrame()
+
+    start = pd.Timestamp.today(tz="UTC") - pd.DateOffset(years=years, days=7)
+    end = pd.Timestamp.today(tz="UTC")
+
+    df_close = yf.download(
+        list(dict.fromkeys(tickers)),
+        start=start.date(),
+        end=end.date(),
+        interval="1mo",
+        auto_adjust=True,
+        progress=False
+    )["Close"]
+
+    # If a single ticker is requested, yfinance gives a Series named "Close".
+    # Make it a DataFrame and name the column with the ticker symbol.
+    if isinstance(df_close, pd.Series):
+        df_close = df_close.to_frame()
+        # name column if we know the ticker
+        if len(tickers) == 1:
+            df_close.columns = [tickers[0].upper()]
+
+    # Standardize column names to uppercase tickers when possible.
+    if isinstance(df_close.columns, pd.Index):
+        df_close.columns = [str(c).upper() for c in df_close.columns]
+
+    df_close = df_close.dropna(how="all").fillna(method="ffill")
+    return df_close
+
+def monthly_returns(prices: pd.DataFrame) -> pd.DataFrame:
+    return prices.pct_change().dropna()
+
+def annualize_mean(m):
+    return np.asarray(m, dtype=float) * 12.0
+
+def annualize_sigma(s):
+    return np.asarray(s, dtype=float) * math.sqrt(12.0)
+
+def yahoo_search(query: str):
+    # Yahoo symbol search
+    if not query or len(query.strip()) == 0:
+        return []
+    url = "https://query1.finance.yahoo.com/v1/finance/search"
+    params = {"q": query.strip(), "quotesCount": 10, "newsCount": 0}
+    headers = {"User-Agent": "Mozilla/5.0"}
+    try:
+        r = requests.get(url, params=params, headers=headers, timeout=10)
+        r.raise_for_status()
+        data = r.json()
+        out = []
+        for q in data.get("quotes", []):
+            sym = q.get("symbol")
+            name = q.get("shortname") or q.get("longname") or ""
+            exch = q.get("exchDisp") or ""
+            if sym and sym.isascii():
+                out.append({"symbol": sym, "name": name, "exchange": exch})
+        if not out:
+            out = [{"symbol": query.strip().upper(), "name": "typed symbol", "exchange": "n/a"}]
+        return out[:10]
+    except Exception:
+        return [{"symbol": query.strip().upper(), "name": "typed symbol", "exchange": "n/a"}]
+
+def validate_tickers(symbols: List[str], years: int) -> List[str]:
+    ok, df = [], fetch_prices_monthly(list(set(symbols)), years)
+    for s in symbols:
+        if s.upper() in df.columns:
+            ok.append(s.upper())
+    return ok
+
+# -------------- aligned moments --------------
+def get_aligned_monthly_returns(symbols: List[str], years: int) -> pd.DataFrame:
+    uniq = [c.upper() for c in dict.fromkeys(symbols) if c.upper() != MARKET_TICKER]
+    tickers = uniq + [MARKET_TICKER]
+    px = fetch_prices_monthly(tickers, years)
+    rets = monthly_returns(px)
+    cols = [c for c in uniq if c in rets.columns] + ([MARKET_TICKER] if MARKET_TICKER in rets.columns else [])
+    R = rets[cols].dropna(how="any")
+    return R.loc[:, ~R.columns.duplicated()]
+
+def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
+    R = get_aligned_monthly_returns(symbols, years)
+    if MARKET_TICKER not in R.columns or R.shape[0] < 3:
+        raise ValueError("Not enough aligned data for market/tickers")
+    rf_m = rf_ann / 12.0
+
+    m = R[MARKET_TICKER]
+    if isinstance(m, pd.DataFrame):
+        m = m.iloc[:, 0].squeeze()
+
+    mu_m_ann = float(annualize_mean(m.mean()))
+    sigma_m_ann = float(annualize_sigma(m.std(ddof=1)))
+    erp_ann = float(mu_m_ann - rf_ann)
+
+    ex_m = m - rf_m
+    var_m = float(np.var(ex_m.values, ddof=1))
+    var_m = max(var_m, 1e-6)
+
+    betas: Dict[str, float] = {}
+    for s in [c for c in R.columns if c != MARKET_TICKER]:
+        ex_s = R[s] - rf_m
+        betas[s] = float(np.cov(ex_s.values, ex_m.values, ddof=1)[0, 1] / var_m)
+
+    betas[MARKET_TICKER] = 1.0  # by definition
+
+    asset_cols = [c for c in R.columns if c != MARKET_TICKER]
+    cov_m = np.cov(R[asset_cols].values.T, ddof=1) if asset_cols else np.zeros((0, 0))
+    covA = pd.DataFrame(cov_m * 12.0, index=asset_cols, columns=asset_cols)
+
+    return {"betas": betas, "cov_ann": covA, "erp_ann": erp_ann, "sigma_m_ann": sigma_m_ann}
+
+def capm_er(beta: float, rf_ann: float, erp_ann: float) -> float:
+    return float(rf_ann + beta * erp_ann)
+
+def portfolio_stats(weights: Dict[str, float],
+                    cov_ann: pd.DataFrame,
+                    betas: Dict[str, float],
+                    rf_ann: float,
+                    erp_ann: float) -> Tuple[float, float, float]:
+    tickers = list(weights.keys())
+    w = np.array([weights[t] for t in tickers], dtype=float)
+    gross = float(np.sum(np.abs(w)))
+    if gross == 0:
+        return 0.0, 0.0, 0.0
+    w_expo = w / gross
+    beta_p = float(np.dot([betas.get(t, 0.0) for t in tickers], w_expo))
+    er_p = capm_er(beta_p, rf_ann, erp_ann)
+    cov = cov_ann.reindex(index=tickers, columns=tickers).fillna(0.0).to_numpy()
+    sigma_p = math.sqrt(float(max(w_expo.T @ cov @ w_expo, 0.0)))
+    return beta_p, er_p, sigma_p
+
+# -------------- CML helpers --------------
+def efficient_same_sigma(sigma_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
+    if sigma_mkt <= 1e-12:
+        return 0.0, 1.0, rf_ann
+    a = sigma_target / sigma_mkt
+    return a, 1.0 - a, rf_ann + a * erp_ann
+
+def efficient_same_return(mu_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
+    if abs(erp_ann) <= 1e-12:
+        return 0.0, 1.0, rf_ann
+    a = (mu_target - rf_ann) / erp_ann
+    return a, 1.0 - a, abs(a) * sigma_mkt
+
+def plot_cml(
+    rf_ann, erp_ann, sigma_mkt,
+    pt_sigma, pt_mu,
+    same_sigma_sigma, same_sigma_mu,
+    same_mu_sigma, same_mu_mu,
+    targ_sigma=None, targ_mu=None
+) -> Image.Image:
+    fig = plt.figure(figsize=(6, 4), dpi=120)
+
+    xmax = max(
+        0.3,
+        sigma_mkt * 2.0,
+        pt_sigma * 1.4,
+        same_mu_sigma * 1.4,
+        same_sigma_sigma * 1.4,
+        (targ_sigma or 0.0) * 1.4,
+    )
+    xs = np.linspace(0, xmax, 160)
+    slope = erp_ann / max(sigma_mkt, 1e-12)
+    cml = rf_ann + slope * xs
+    plt.plot(xs, cml, label="CML through VOO")
+
+    plt.scatter([0.0], [rf_ann], label="Risk free")
+    plt.scatter([sigma_mkt], [rf_ann + erp_ann], label="Market VOO")
+    plt.scatter([pt_sigma], [pt_mu], label="Your portfolio")
+    plt.scatter([same_sigma_sigma], [same_sigma_mu], label="Efficient same sigma")
+    plt.scatter([same_mu_sigma], [same_mu_mu], label="Efficient same return")
+    if targ_sigma is not None and targ_mu is not None:
+        plt.scatter([targ_sigma], [targ_mu], label="Target suggestion")
+
+    # Gap guides
+    plt.plot([pt_sigma, same_sigma_sigma], [pt_mu, same_sigma_mu], linestyle="--", linewidth=1.2, alpha=0.7, color="gray")
+    d_ret = (same_sigma_mu - pt_mu) * 100.0
+    plt.annotate(
+        f"Return gain at same sigma {d_ret:+.2f}%",
+        xy=(same_sigma_sigma, same_sigma_mu),
+        xytext=(same_sigma_sigma + 0.02 * xmax, same_sigma_mu),
+        arrowprops=dict(arrowstyle="->", lw=1.0),
+        fontsize=9,
+        va="center",
+    )
+
+    plt.plot([pt_sigma, same_mu_sigma], [pt_mu, same_mu_mu], linestyle="--", linewidth=1.2, alpha=0.7, color="gray")
+    d_sig = (same_mu_sigma - pt_sigma) * 100.0
+    plt.annotate(
+        f"Risk change at same return {d_sig:+.2f}%",
+        xy=(same_mu_sigma, same_mu_mu),
+        xytext=(same_mu_sigma, same_mu_mu + 0.03),
+        arrowprops=dict(arrowstyle="->", lw=1.0),
+        fontsize=9,
+        ha="center",
+    )
+
+    plt.xlabel("Standard deviation")
+    plt.ylabel("Expected return")
+    plt.legend(loc="best")
+    plt.tight_layout()
+
+    buf = io.BytesIO()
+    plt.savefig(buf, format="png")
+    plt.close(fig)
+    buf.seek(0)
+    return Image.open(buf)
+
+# -------------- synthetic dataset --------------
+def synth_profile(seed: int) -> str:
+    rng = np.random.default_rng(seed)
+    risk = rng.choice(["cautious", "balanced", "moderate", "growth", "aggressive"])
+    horizon = rng.choice(["three years", "five years", "seven years", "ten years", "fifteen years"])
+    goal = rng.choice(["retirement savings", "first home", "education fund", "wealth building", "travel fund", "emergency buffer"])
+    return f"{risk} investor, {horizon} horizon, goal is {goal}."
+
+def build_synthetic_dataset(universe: List[str], years: int, rf_ann: float, erp_ann: float) -> pd.DataFrame:
+    symbols = list(sorted(set([s for s in universe if s != MARKET_TICKER] + [MARKET_TICKER])))[:MAX_TICKERS]
+    moms = estimate_all_moments_aligned(symbols, years, rf_ann)
+    covA, betas = moms["cov_ann"], moms["betas"]
+    rows, rng = [], np.random.default_rng(123)
+    for i in range(1000):
+        k = rng.integers(low=min(2, len(symbols)), high=min(8, len(symbols)) + 1)
+        picks = list(rng.choice(symbols, size=k, replace=False))
+        signs = rng.choice([-1.0, 1.0], size=k, p=[0.25, 0.75])
+        raw = rng.dirichlet(np.ones(k))
+        gross = 1.0 + float(rng.gamma(2.0, 0.5))
+        w = gross * signs * raw
+        beta_p, er_p, sigma_p = portfolio_stats({picks[j]: w[j] for j in range(k)}, covA, betas, rf_ann, erp_ann)
+        rows.append({
+            "id": i,
+            "profile_text": synth_profile(10_000 + i),
+            "tickers": ",".join(picks),
+            "weights": ",".join(f"{x:.4f}" for x in w),
+            "beta_p": beta_p,
+            "er_p": er_p,
+            "sigma_p": sigma_p
         })
+    return pd.DataFrame(rows)
+
+def save_synth_csv(df: pd.DataFrame, path: str = DATASET_PATH):
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    df.to_csv(path, index=False)
+
+# ----------- surrogate from saved CSV only -----------
+def _row_to_exposures(row: pd.Series, universe: List[str]) -> Optional[np.ndarray]:
+    try:
+        ts = [t.strip() for t in str(row["tickers"]).split(",")]
+        ws = [float(x) for x in str(row["weights"]).split(",")]
+        wmap = {t: ws[i] for i, t in enumerate(ts) if i < len(ws)}
+        w = np.array([wmap.get(t, 0.0) for t in universe], dtype=float)
+        gross = float(np.sum(np.abs(w)))
+        if gross <= 1e-12:
+            return None
+        return w / gross
+    except Exception:
+        return None
+
+def fit_surrogate_from_csv(csv_path: str, universe: List[str]):
+    try:
+        df = pd.read_csv(csv_path)
+    except Exception:
+        return None, None, 0
+    X_list, Y_list = [], []
+    for _, r in df.iterrows():
+        x = _row_to_exposures(r, universe)
+        if x is None:
+            continue
+        y = np.array([float(r["er_p"]), float(r["sigma_p"]), float(r["beta_p"])], dtype=float)
+        X_list.append(x); Y_list.append(y)
+    if not X_list:
+        return None, None, 0
+    X = np.vstack(X_list); Y = np.vstack(Y_list)
+    scaler = StandardScaler().fit(X)
+    Xn = scaler.transform(X)
+    k = min(25, len(Xn))
+    knn = KNeighborsRegressor(n_neighbors=k, weights="distance")
+    knn.fit(Xn, Y)
+    return scaler, knn, len(Xn)
+
+def predict_from_surrogate(amounts_map: Dict[str, float], universe: List[str],
+                           scaler: StandardScaler, knn: KNeighborsRegressor):
+    gross = sum(abs(v) for v in amounts_map.values())
+    if gross <= 1e-12:
+        return None
+    w = np.array([amounts_map.get(t, 0.0) for t in universe], dtype=float) / gross
+    yhat = knn.predict(scaler.transform([w]))[0]
+    er_hat, sigma_hat, beta_hat = float(yhat[0]), float(yhat[1]), float(yhat[2])
+    return er_hat, sigma_hat, beta_hat
+
+# ----------- target search over synthetic dataset -----------
+def target_best_from_synth(csv_path: str,
+                           universe: List[str],
+                           target_mu: Optional[float],
+                           target_sigma: Optional[float]):
+    try:
+        df = pd.read_csv(csv_path)
+    except Exception:
+        return None
+
+    if target_mu is None and target_sigma is None:
+        return None
+
+    rows = []
+    for _, r in df.iterrows():
+        x = _row_to_exposures(r, universe)
+        if x is None:
+            continue
+        rows.append((x, float(r["er_p"]), float(r["sigma_p"]), float(r["beta_p"]), r))
+
+    if not rows:
+        return None
+
+    mu_w = 1.0
+    sig_w = 1.0
+    best = None
+    best_d = float("inf")
+    for x, er_p, sig_p, beta_p, r in rows:
+        d = 0.0
+        if target_mu is not None:
+            d += mu_w * (er_p - target_mu) ** 2
+        if target_sigma is not None:
+            d += sig_w * (sig_p - target_sigma) ** 2
+        if d < best_d:
+            best_d = d
+            best = (x, er_p, sig_p, beta_p, r)
+
+    if best is None:
+        return None
+
+    x, er_p, sig_p, beta_p, r = best
+    wmap = {t: float(x[i]) for i, t in enumerate(universe) if abs(float(x[i])) > 1e-4}
+    top = sorted(wmap.items(), key=lambda kv: -abs(kv[1]))[:12]
+    wmap_top = dict(top)
+    return {"weights": wmap_top, "er": er_p, "sigma": sig_p, "beta": beta_p}
+
+# -------------- summary builder --------------
+def fmt_pct(x: float) -> str:
+    return f"{x*100:.2f}%"
+
+def humanize_synth(er_hat, sigma_hat, beta_hat, dmu, dsig, dbeta):
+    close_mu = abs(dmu) <= 0.005
+    close_sig = abs(dsig) <= 0.005
+    close_beta = abs(dbeta) <= 0.05
+    parts = []
+    parts.append(f"- Predicted annual return {fmt_pct(er_hat)} , difference {fmt_pct(dmu)}")
+    parts.append(f"- Predicted annual volatility {fmt_pct(sigma_hat)} , difference {fmt_pct(dsig)}")
+    parts.append(f"- Predicted beta {beta_hat:.2f} , difference {dbeta:+.02f}")
+    if close_mu and close_sig and close_beta:
+        verdict = "The synthetic model matches the historical calculation closely. You can trust these quick predictions for similar mixes."
+    else:
+        verdict = "The synthetic model is not very close here. Rely more on the historical calculation for this mix."
+    return "\n".join(parts + ["", f"**Verdict** {verdict}"])
+
+def build_summary_md(lookback, horizon, rf, rf_code, erp, sigma_mkt,
+                     beta_p, er_p, sigma_p,
+                     a_sigma, b_sigma, mu_eff_sigma,
+                     a_mu, b_mu, sigma_eff_mu,
+                     synth=None, synth_nrows: int = 0,
+                     targ=None) -> str:
+    lines = []
+    lines.append("### Inputs")
+    lines.append(f"- Lookback years {lookback}")
+    lines.append(f"- Horizon years {int(round(horizon))}")
+    lines.append(f"- Risk free {fmt_pct(rf)} from {rf_code}")
+    lines.append(f"- Market ERP {fmt_pct(erp)}")
+    lines.append(f"- Market sigma {fmt_pct(sigma_mkt)}")
+    lines.append("")
+    lines.append("### Your portfolio")
+    lines.append(f"- Beta {beta_p:.2f}")
+    lines.append(f"- Sigma {fmt_pct(sigma_p)}")
+    lines.append(f"- Expected return {fmt_pct(er_p)}")
+    if synth is not None:
+        er_hat, sigma_hat, beta_hat, dmu, dsig, dbeta = synth
+        lines.append("")
+        lines.append("### Synthetic prediction from data/investor_profiles.csv")
+        lines.append(f"- Samples used {synth_nrows}")
+        lines.append(humanize_synth(er_hat, sigma_hat, beta_hat, dmu, dsig, dbeta))
+    if targ is not None:
+        lines.append("")
+        lines.append("### Target driven suggestion from synthetic dataset")
+        lines.append(f"- Suggested expected return {fmt_pct(targ['er'])}")
+        lines.append(f"- Suggested sigma {fmt_pct(targ['sigma'])}")
+        lines.append(f"- Suggested beta {targ['beta']:.2f}")
+        pretty = ", ".join([f"{k} {v:+.2f}" for k, v in targ["weights"].items()])
+        lines.append(f"- Weights, exposure terms {pretty}")
+    lines.append("")
+    lines.append("### Efficient alternatives on CML")
+    lines.append("Efficient same sigma")
+    lines.append(f"- Market weight {a_sigma:.2f} , Bills weight {b_sigma:.2f}")
+    lines.append(f"- Expected return {fmt_pct(mu_eff_sigma)}")
+    lines.append("Efficient same return")
+    lines.append(f"- Market weight {a_mu:.2f} , Bills weight {b_mu:.2f}")
+    lines.append(f"- Sigma {fmt_pct(sigma_eff_mu)}")
+    return "\n".join(lines)
+
+# -------------- app state on launch --------------
+ensure_data_dir()
+UNIVERSE = [MARKET_TICKER, "QQQ", "XLK", "XLP", "XLE", "VNQ", "IEF", "HYG", "GLD", "EEM"]
+HORIZON_YEARS = 5
+RF_CODE = fred_series_for_horizon(HORIZON_YEARS)
+RF_ANN = fetch_fred_yield_annual(RF_CODE)
+
+# -------------- gradio callbacks --------------
+def search_tickers_cb(q: str):
+    hits = yahoo_search(q)
+    if not hits:
+        return "No matches", []
+    opts = [f"{h['symbol']}  |  {h['name']}  |  {h['exchange']}" for h in hits]
+    return "Select a symbol and click Add", opts
+
+def add_symbol(selection: str, table: pd.DataFrame):
+    if not selection:
+        return table, "Pick a row from Matches first"
+    symbol = selection.split("|")[0].strip().upper()
+    current = [] if table is None or len(table) == 0 else [str(x).upper() for x in table["ticker"].tolist() if str(x) != "nan"]
+    tickers = current if symbol in current else current + [symbol]
+    val = validate_tickers(tickers, years=DEFAULT_LOOKBACK_YEARS)
+    tickers = [t for t in tickers if t in val]
+    amt_map = {}
+    if table is not None and len(table) > 0:
+        for _, r in table.iterrows():
+            t = str(r.get("ticker", "")).upper()
+            if t in tickers:
+                amt_map[t] = float(pd.to_numeric(r.get("amount_usd", 0.0), errors="coerce") or 0.0)
+    new_table = pd.DataFrame({"ticker": tickers, "amount_usd": [amt_map.get(t, 0.0) for t in tickers]})
+    msg = f"Added {symbol}" if symbol in tickers else f"{symbol} not valid"
+    if len(new_table) > MAX_TICKERS:
+        new_table = new_table.iloc[:MAX_TICKERS]
+        msg = f"Reached max of {MAX_TICKERS}"
+    return new_table, msg
+
+def lock_ticker_column(tb: pd.DataFrame):
+    if tb is None or len(tb) == 0:
+        return pd.DataFrame(columns=["ticker", "amount_usd"])
+    tickers = [str(x).upper() for x in tb["ticker"].tolist()]
+    amounts = pd.to_numeric(tb["amount_usd"], errors="coerce").fillna(0.0).tolist()
+    val = validate_tickers(tickers, years=DEFAULT_LOOKBACK_YEARS)
+    tickers = [t for t in tickers if t in val]
+    amounts = amounts[:len(tickers)] + [0.0] * max(0, len(tickers) - len(amounts))
+    return pd.DataFrame({"ticker": tickers, "amount_usd": amounts})
+
+def set_horizon(years: float):
+    y = max(1.0, min(100.0, float(years)))
+    code = fred_series_for_horizon(y)
+    rf = fetch_fred_yield_annual(code)
+    global HORIZON_YEARS, RF_CODE, RF_ANN
+    HORIZON_YEARS = y
+    RF_CODE = code
+    RF_ANN = rf
+    return f"Risk free series {code}. Latest annual rate {rf:.2%}. Dataset will use this rate on compute."
+
+def compute(years_lookback: int, table: pd.DataFrame,
+            target_mu: Optional[float], target_sigma: Optional[float],
+            use_synth: bool):
+    if table is None or len(table) == 0:
+        return None, "Add at least one ticker", "Universe empty", empty_positions_df(), empty_suggest_df(), None
+
+    df = table.dropna()
+    df["ticker"] = df["ticker"].astype(str).str.upper().str.strip()
+    df["amount_usd"] = pd.to_numeric(df["amount_usd"], errors="coerce").fillna(0.0)
+
+    symbols = [t for t in df["ticker"].tolist() if t]
+    if len(symbols) == 0:
+        return None, "Add at least one ticker", "Universe empty", empty_positions_df(), empty_suggest_df(), None
+
+    symbols = validate_tickers(symbols, years_lookback)
+    if len(symbols) == 0:
+        return None, "Could not validate any tickers", "Universe invalid", empty_positions_df(), empty_suggest_df(), None
+
+    global UNIVERSE
+    UNIVERSE = list(sorted(set([s for s in symbols if s != MARKET_TICKER] + [MARKET_TICKER])))[:MAX_TICKERS]
+
+    df = df[df["ticker"].isin(symbols)].copy()
+    amounts = {r["ticker"]: float(r["amount_usd"]) for _, r in df.iterrows()}
+    rf_ann = RF_ANN
+
+    moms = estimate_all_moments_aligned(symbols, years_lookback, rf_ann)
+    betas, covA, erp_ann, sigma_mkt = moms["betas"], moms["cov_ann"], moms["erp_ann"], moms["sigma_m_ann"]
+
+    gross = sum(abs(v) for v in amounts.values())
+    if gross == 0:
+        return None, "All amounts are zero", "Universe ok", empty_positions_df(), empty_suggest_df(), None
+    weights = {k: v / gross for k, v in amounts.items()}
+
+    beta_p, er_p, sigma_p = portfolio_stats(weights, covA, betas, rf_ann, erp_ann)
+
+    a_sigma, b_sigma, mu_eff_sigma = efficient_same_sigma(sigma_p, rf_ann, erp_ann, sigma_mkt)
+    a_mu, b_mu, sigma_eff_mu = efficient_same_return(er_p, rf_ann, erp_ann, sigma_mkt)
+
+    # ensure dataset exists once
+    if not os.path.exists(DATASET_PATH):
+        synth_df = build_synthetic_dataset(
+            universe=list(sorted(set(symbols + [MARKET_TICKER]))),
+            years=DEFAULT_LOOKBACK_YEARS,
+            rf_ann=rf_ann,
+            erp_ann=erp_ann
+        )
+        save_synth_csv(synth_df)
+    csv_path = DATASET_PATH if os.path.exists(DATASET_PATH) else None
+
+    scaler, knn, nrows = None, None, 0
+    synth_tuple = None
+    if use_synth and csv_path:
+        scaler, knn, nrows = fit_surrogate_from_csv(csv_path, UNIVERSE)
+        if scaler is not None and knn is not None:
+            pred = predict_from_surrogate(amounts, UNIVERSE, scaler, knn)
+            if pred is not None:
+                er_hat, sigma_hat, beta_hat = pred
+                synth_tuple = (
+                    er_hat, sigma_hat, beta_hat,
+                    er_hat - er_p, sigma_hat - sigma_p, beta_hat - beta_p
+                )
+
+    # target driven suggestion from synthetic dataset
+    targ = None
+    targ_table = empty_suggest_df()
+    targ_sigma_plot = None
+    targ_mu_plot = None
+    if csv_path and (target_mu is not None or target_sigma is not None):
+        cand = target_best_from_synth(csv_path, UNIVERSE, target_mu, target_sigma)
+        if cand is not None:
+            targ = cand
+            targ_sigma_plot = cand["sigma"]
+            targ_mu_plot = cand["er"]
+            rows = [{"ticker": k, "suggested_weight_exposure": v} for k, v in cand["weights"].items()]
+            targ_table = pd.DataFrame(rows, columns=SUG_COLS)
+
+    img = plot_cml(
+        rf_ann, erp_ann, sigma_mkt,
+        sigma_p, er_p,
+        sigma_p, mu_eff_sigma,
+        sigma_eff_mu, er_p,
+        targ_sigma=targ_sigma_plot, targ_mu=targ_mu_plot
+    )
+
+    info = build_summary_md(
+        years_lookback, HORIZON_YEARS, rf_ann, RF_CODE, erp_ann, sigma_mkt,
+        beta_p, er_p, sigma_p,
+        a_sigma, b_sigma, mu_eff_sigma,
+        a_mu, b_mu, sigma_eff_mu,
+        synth=synth_tuple, synth_nrows=nrows,
+        targ=targ
+    )
+
+    rows = []
+    for t in symbols:
+        beta_val = 1.0 if t == MARKET_TICKER else betas.get(t, np.nan)
+        rows.append({
+            "ticker": t,
+            "amount_usd": amounts.get(t, 0.0),
+            "weight_exposure": weights.get(t, 0.0),
+            "beta": beta_val,
+        })
+    pos_table = pd.DataFrame(rows, columns=POS_COLS)
+
+    uni_msg = f"Universe set to {', '.join(UNIVERSE)}"
+    return img, info, uni_msg, pos_table, targ_table, csv_path
+
+# -------------- UI --------------
+ensure_data_dir()
+
+with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
+    gr.Markdown(
+        "## Efficient Portfolio Advisor\n"
+        "Search symbols, enter dollar amounts, set your horizon. "
+        "Prices come from Yahoo Finance. Risk free comes from FRED."
+    )
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            q = gr.Textbox(label="Search symbol")
+            search_note = gr.Markdown()
+            matches = gr.Dropdown(choices=[], label="Matches")
+            search_btn = gr.Button("Search")
+            add_btn = gr.Button("Add selected to portfolio")
+
+            gr.Markdown("### Portfolio positions (type dollar amounts; negatives allowed for shorts)")
+            table = gr.Dataframe(
+                headers=["ticker", "amount_usd"],
+                datatype=["str", "number"],
+                row_count=0,
+                col_count=(2, "fixed")
+            )
+
+            horizon = gr.Number(label="Horizon in years (1–100)", value=5, precision=0)
+            lookback = gr.Slider(1, 10, value=DEFAULT_LOOKBACK_YEARS, step=1, label="Lookback years for beta & sigma")
+
+            gr.Markdown("### Optional targets on the CML")
+            target_mu = gr.Number(label="Target expected return (annual, e.g. 0.12 = 12%)", value=None, precision=6)
+            target_sigma = gr.Number(label="Target sigma (annual, e.g. 0.18 = 18%)", value=None, precision=6)
+            use_synth = gr.Checkbox(label="Use synthetic predictor", value=True)
+
+            run_btn = gr.Button("Compute and suggest")
+        with gr.Column(scale=1):
+            plot = gr.Image(label="Capital Market Line", type="pil")
+            summary = gr.Markdown(label="Summary")
+            universe_msg = gr.Textbox(label="Universe status", interactive=False)
+            positions = gr.Dataframe(
+                label="Computed positions",
+                headers=POS_COLS,
+                datatype=["str", "number", "number", "number"],
+                col_count=(len(POS_COLS), "fixed"),
+                value=empty_positions_df(),
+                interactive=False
+            )
+            suggestions = gr.Dataframe(
+                label="Suggested portfolio from targets",
+                headers=SUG_COLS,
+                datatype=["str", "number"],
+                col_count=(len(SUG_COLS), "fixed"),
+                value=empty_suggest_df(),
+                interactive=False
+            )
+            dl = gr.File(label="Session CSV path", value=None, visible=True)
+
+    def do_search(query):
+        note, options = search_tickers_cb(query)
+        return note, gr.update(choices=options)
+
+    search_btn.click(fn=do_search, inputs=q, outputs=[search_note, matches])
+    add_btn.click(fn=add_symbol, inputs=[matches, table], outputs=[table, search_note])
+    table.change(fn=lock_ticker_column, inputs=table, outputs=table)
+    horizon.change(fn=set_horizon, inputs=horizon, outputs=universe_msg)
 
-    return pd.DataFrame(synthetic_data)
-
-def select_risk_profiles(synth_df):
-    """Select high/high, medium/medium, low/low risk profiles from synthetic dataset."""
-    high = synth_df.sort_values("er_p", ascending=False).iloc[0]
-    low = synth_df.sort_values("sigma_p", ascending=True).iloc[0]
-
-    median_idx = ((synth_df["sigma_p"] - synth_df["sigma_p"].median()).abs() +
-                  (synth_df["er_p"] - synth_df["er_p"].median()).abs()).idxmin()
-    medium = synth_df.loc[median_idx]
-
-    return high, medium, low
-
-def find_efficient_same_sigma(user_er, user_sigma, synth_df):
-    """Find portfolio with same sigma but highest return."""
-    close_sigma = synth_df[np.isclose(synth_df["sigma_p"], user_sigma, atol=0.002)]
-    if close_sigma.empty:
-        return synth_df.iloc[0]
-    return close_sigma.sort_values("er_p", ascending=False).iloc[0]
-
-def find_efficient_same_return(user_er, user_sigma, synth_df):
-    """Find portfolio with same return but lowest sigma."""
-    close_return = synth_df[np.isclose(synth_df["er_p"], user_er, atol=0.002)]
-    if close_return.empty:
-        return synth_df.iloc[0]
-    return close_return.sort_values("sigma_p", ascending=True).iloc[0]
-# -------------------
-# Main compute function
-# -------------------
-
-def compute(user_tickers):
-    # Convert comma-separated string into ticker list
-    tickers = [t.strip().upper() for t in user_tickers.split(",") if t.strip()]
-    if len(tickers) < 2:
-        return "Please enter at least two tickers.", None
-
-    # Fetch live data & compute user portfolio metrics (equal weights for now)
-    df_prices = fetch_live_data(tickers)
-    if df_prices.empty:
-        return "Could not fetch data. Check tickers.", None
-
-    returns = df_prices.pct_change().dropna()
-    mean_returns = returns.mean()
-    cov_matrix = returns.cov()
-    user_weights = np.ones(len(tickers)) / len(tickers)
-    user_er, user_sigma, user_beta = calculate_portfolio_metrics(user_weights, mean_returns, cov_matrix)
-
-    # Generate synthetic dataset
-    synth_df = generate_synthetic_portfolios(tickers, num_portfolios=1000)
-
-    # Select profiles
-    eff_sigma = find_efficient_same_sigma(user_er, user_sigma, synth_df)
-    eff_return = find_efficient_same_return(user_er, user_sigma, synth_df)
-    high, medium, low = select_risk_profiles(synth_df)
-
-    # Prepare results DataFrame
-    portfolios = {
-        "User Portfolio": [user_er, user_sigma, user_beta, user_weights],
-        "Efficient (Same Sigma)": [eff_sigma.er_p, eff_sigma.sigma_p, eff_sigma.beta_p, eff_sigma.weights],
-        "Efficient (Same Return)": [eff_return.er_p, eff_return.sigma_p, eff_return.beta_p, eff_return.weights],
-        "High Risk / High Return": [high.er_p, high.sigma_p, high.beta_p, high.weights],
-        "Medium Risk / Medium Return": [medium.er_p, medium.sigma_p, medium.beta_p, medium.weights],
-        "Low Risk / Low Return": [low.er_p, low.sigma_p, low.beta_p, low.weights],
-    }
-
-    df_out = pd.DataFrame(portfolios, index=["Expected Return", "Sigma", "Beta", "Weights"])
-
-    return df_out.to_markdown(), df_out
-
-# -------------------
-# Gradio Interface
-# -------------------
-
-with gr.Blocks() as demo:
-    gr.Markdown("## Portfolio Optimizer and Risk Profiles")
-    tickers_input = gr.Textbox(label="Enter tickers (comma separated)", placeholder="AAPL, MSFT, GOOG")
-    output_md = gr.Markdown()
-    output_df = gr.Dataframe(headers=["Portfolio", "Value"], interactive=False)
-
-    def run_and_display(tickers):
-        md, df = compute(tickers)
-        if df is None:
-            return md, None
-        return md, df
-
-    run_btn = gr.Button("Run Analysis")
-    run_btn.click(fn=run_and_display, inputs=tickers_input, outputs=[output_md, output_df])
+    run_btn.click(
+        fn=compute,
+        inputs=[lookback, table, target_mu, target_sigma, use_synth],
+        outputs=[plot, summary, universe_msg, positions, suggestions, dl]
+    )
 
 if __name__ == "__main__":
-    demo.launch()
+    # Disable SSR to avoid experimental issues in some deployments
+    demo.launch(ssr_mode=False)