{ "projectInfo": { "name": "PaperTrade", "description": "NSE Indian-equity short-term price-direction prediction engine + paper-trading book. Predicts 1D/3D/INTRADAY direction using backtested technical strategies, a standalone supervised ML quantile model, macro gates, news sentiment, and an LLM bull/bear/fundamentals debate. Served via a Flask web app and an MCP server.", "platform": "Python backend (Flask web app + MCP server) with an offline-trained scikit-learn ML model", "language": "Python 3.13", "deployment": { "primary": "Hugging Face Spaces (persistent /data disk, container). Think HF Spaces first for storage/paths/env.", "local": "python app.py from project root", "notes": [ "Use os.path.dirname(__file__) or HF paths for data files; NOT pathlib.Path(__file__).parent.", "Pickle/file caches do NOT survive HF container restarts — use SQLite (paper_trading.db) instead.", "Static JS/CSS is inline in templates (HF CSP blocks external CDNs).", "Secrets: .env locally; HF Spaces Secrets tab in prod; sync via export_env_secrets.py." ] } }, "architecturalKnowledge": { "pattern": "Layered prediction pipeline + service modules (not MVC/MVVM)", "confidence": 95, "layers": [ "Entry points: app.py (Flask UI + paper-trading book + validation), stock_predictor_mcp.py (MCP tools)", "Prediction core: predictor_core.py (public API predict_stock_v2 / rank_stocks_v2)", "Signal/feature layer: trial_run.py (S1..S20 strategy signals), ml_combiner.py (ML feature funcs)", "Context providers: macro_context.py, fred_data.py, news_sentiment.py, fundamentals.py, sector_pulse.py, fii_flow.py, social_sentiment.py, intraday_live.py, price_targets.py", "LLM layer: ai_forecast.py (bull/bear/fundamentals debate) -> llm_client.py (provider chain) / ollama_client.py", "Data layer: data_sources.py (multi-source OHLCV + live price), universe.py (dynamic NSE universe), database.py (SQLite: trades, orders, snapshots, postmortems)", "Standalone ML: ml_predictor/ (features -> dataset -> train -> infer, 21 committed joblib estimators)", "Research/backtests: research/ (offline analysis, never imported by prod prediction path)" ], "dataFlowPredictionPipeline": [ "1. Market gates (VIX>25 hard block; VIX 20-25 size cut; Nifty 0-100 + logistic prob)", "4. News sentiment (Claude Haiku -> BULLISH/NEUTRAL/BEARISH)", "5. Sector pulse (NSE sector heatmap -> leading/lagging flag)", "6. Fundamentals (PE/D-E/ROE/FCF score, 24h cache)", "7. AI forecast (up to 4-call LLM debate -> synthesis JSON) with heuristic fallback", "8. Confidence scoring (additive -> HIGH/MEDIUM/LOW)", "9. Risk (ATR14 stop, R:R targets scaled to timeframe)" ] }, "technologyStack": { "web": "Flask (server-rendered templates/index.html, inline JS/CSS in static/)", "mcp": "stock_predictor_mcp.py exposes predict_stocks, rank_best_stocks", "ml": "scikit-learn HistGradientBoosting quantile regressors + isotonic-calibrated direction classifier; joblib persistence. No lightgbm/xgboost/torch (HF-safe).", "dataSources": "yfinance + NSE archives + Twelve Data + Alpha Vantage (fallback chain in data_sources.py)", "llmProviderChain": "OpenRouter free -> Groq -> Cerebras -> HuggingFace Router -> Gemini -> SambaNova -> NVIDIA NIM -> Ollama (local last-resort). GitHub Models removed.", "database": "SQLite (paper_trading.db) — trades, pending orders, prediction snapshots, postmortems, ohlcv_cache blob", "testing": "pytest (tests/test_api_contract.py — Flask endpoint schema/type checks)", "caches": [ ".universe_cache.json (24h fresh / 7d stale, /data on HF)", "fred_macro_cache.json (24h)", "fundamentals_cache.json (24h/ticker)", "sector pulse (5-min in-memory)", "ohlcv_cache table in paper_trading.db (SQLite blob, NOT file-based)" ] }, "moduleGraph": { "predictor_core.py": {"role": "Main prediction API (used by MCP + Flask). MUST stay stable.", "publicApi": ["predict_stock_v2(ticker, start_date, end_date, ...)", "rank_stocks_v2(...)", "timeframe_to_dates(tf)", "get_ml_feature_score()", "DEFAULT_UNIVERSE"]}, "trial_run.py": {"role": "All strategy signal generators S1..S20, S_CTRIO, S_CAPFLOW, S_SEASONAL etc. NSE-verified stats in predictor_core._STRATEGY_STATS_DEFAULT — do not change without re-running backtest.", "publicApi": ["gen_s1..gen_s20", "gen_s_confluence_trio", "gen_mfs", "gen_nira", "gen_ped", "gen_supertrend"]}, "ml_combiner.py": {"role": "ML feature functions used by predictor_core.get_ml_feature_score", "publicApi": ["bollinger_position", "ema_stack_score", "shadow_flag", "build_feature_matrix"]}, "ai_forecast.py": {"role": "LLM bull/bear/fundamentals debate -> synthesis. Trigger guardrails + ATR clamp (prod-only). Output: should_buy, entry_price, direction, ranges.", "publicApi": ["get_ai_forecast(...)"]}, "llm_client.py": {"role": "Provider chain + Ollama fallback + rate-limit handling", "publicApi": ["reset_ollama_state()"]}, "data_sources.py": {"role": "Multi-source OHLCV + live price with fallback chains. Do NOT change signatures.", "publicApi": ["fetch_ohlcv(ticker, period)", "fetch_live_price(ticker, allow_delayed=True)", "cached_tickers(period)", "fetch_market_data(period_days)"]}, "universe.py": {"role": "Dynamic full-NSE universe (~2062 EQ stocks). Replaces old nse_universe.py.", "publicApi": ["get_universe(force_refresh=False)", "refresh_universe()"]}, "database.py": {"role": "SQLite paper-trading book + prediction validation audit trail", "publicApi": ["get_open_trades_with_live_prices()", "save_prediction_snapshot(...)", "get_prediction_snapshots(...)", "get_validation_summary(...)", "save_postmortem(...)"]}, "macro_context.py": {"role": "Macro gates (S&P500, USD/INR, crude) + FRED regime gate", "publicApi": ["get_macro_gate()", "global_risk_on"]}, "fred_data.py": {"role": "US macro indicators", "publicApi": ["get_fred_macro()", "get_fred_gate()"]}, "fundamentals.py": {"role": "Stock fundamentals scorer (PE/D-E/rev/FCF/ROE). MUST stay stable.", "publicApi": ["get_fundamentals(ticker)"]}, "sector_pulse.py": {"role": "NSE 10-sector heatmap + rotation. MUST stay stable.", "publicApi": ["get_sector_pulse()", "get_sector_for_ticker(ticker)"]}, "risk_engine.py": {"role": "Portfolio risk metrics (Sharpe, drawdown, beta, Kelly)", "publicApi": ["compute_risk_metrics()"]}, "top5_picker.py": {"role": "Top picks (up to 20) INTRADAY/1D/3D, concurrent, ATR-ranked, progressive streaming", "publicApi": ["get_top5_picks(top_n=20, _universe_size=700, progress_cb=None)", "get_weekly_picks(...)"]}, "app.py": {"role": "Flask UI + paper-trading book + validation + all /api routes", "publicApi": ["Flask endpoints (see flaskEndpoints)"]}, "stock_predictor_mcp.py": {"role": "MCP server entry point. Imports predictor_core.", "publicApi": ["predict_stocks", "rank_best_stocks"]}, "ml_predictor/features.py": {"role": "Shared point-in-time feature builder (37 lookahead-safe features)", "publicApi": ["compute_features(...)", "FEATURE_COLUMNS"]}, "ml_predictor/infer.py": {"role": "MLPredictor — quantile forecast per TF. Batch via _raw_predict (fast); per-row _predict_tf is ~1000x slower.", "publicApi": ["get_ml_predictor()", "MLPredictor.predict_all_tf(ticker, live_price, today_high, news_score)", "MLPredictor._raw_predict(tf, X)", "MLPredictor._derive(...)"]}, "ml_predictor/train.py": {"role": "Fit 21 estimators + manifest.json (offline)", "publicApi": ["train"]}, "ml_predictor/dataset.py": {"role": "Build training_data.csv from ohlcv cache (offline). Labels up/dn = max/min excursion; dir = excess-of-Nifty.", "publicApi": ["_fwd_labels(...)"]} }, "flaskEndpoints": { "predictions": ["POST /api/predict", "POST /api/rank", "GET /api/top5", "GET /api/watchlist-picks", "GET /api/watchlist-pick/", "GET /api/universe", "POST /api/universe/refresh", "GET /api/search", "GET /api/chart/", "GET /api/live-price/", "GET /api/ml-predict/"], "context": ["GET /api/sector-pulse", "GET /api/fundamentals/", "GET /api/portfolio", "GET /api/portfolio-insight/", "GET /api/signal-accuracy"], "paperTrading": ["GET /api/open-trades", "GET /api/trades/open", "GET /api/trades/history", "POST /api/trades", "POST /api/trades//close", "GET /api/trades//price", "POST /api/trades/check-stops", "GET /api/orders/pending", "POST /api/orders/check", "POST /api/orders//cancel", "GET|POST /api/watchlist", "DELETE /api/watchlist/"], "validation": ["GET /api/prediction-snapshots", "GET /api/prediction-validation", "GET /api/prediction-misses", "GET /api/validation/pending", "POST /api/validation/execute", "GET /api/validation/summary", "GET /api/postmortems", "POST /api/postmortem"] }, "criticalConstraintsDoNotBreak": [ "predict_stock_v2 / rank_stocks_v2 / timeframe_to_dates signatures (used by MCP + Flask).", "data_sources.fetch_ohlcv / fetch_live_price signatures.", "universe.get_universe, fundamentals.get_fundamentals, sector_pulse.get_sector_pulse, fred_data.get_fred_macro/get_fred_gate signatures.", "ml_predictor: compute_features, MLPredictor.predict_all_tf, MLPredictor.predict signatures.", "trial_run.py strategy stats in predictor_core._STRATEGY_STATS_DEFAULT are NSE-verified — do not change without re-running the backtest.", "Do not add lightgbm/xgboost/torch (HF Spaces image safety).", "Do not put data files behind pathlib.Path(__file__).parent; do not rely on file/pickle caches surviving HF restarts." ], "codeGenerationGuidelines": { "paths": "os.path.dirname(__file__) or HF /data paths; never pathlib(__file__).parent for data.", "persistence": "SQLite (paper_trading.db) for anything that must survive HF restarts — not pickle/file caches.", "frontend": "Inline JS/CSS only (HF CSP blocks external URLs). Bump cache-buster ?v=YYYYMMDD in templates/index.html on JS/CSS change.", "stability": "Keep listed public signatures stable; prod prediction path must not import research/.", "ml": "Batch model inference with MLPredictor._raw_predict over a full matrix; never loop _predict_tf per row in bulk jobs.", "testing": "Add/extend tests/test_api_contract.py for new endpoints (schema + field types).", "backtestMetrics": "Distinguish MidHit (band-midpoint, ~90%, soft) from DirAcc/DirHit (directional, ~40-50%, the real metric). Use training_data_extra.csv (has 12 extra features); training_data.csv lacks them." }, "envVars": { "llmKeys": ["OPENROUTER_API_KEY", "GROQ_API_KEY", "CEREBRAS_API_KEY", "HF_TOKEN", "GEMINI_API_KEY", "SAMBANOVA_API_KEY", "NVIDIA_API_KEY", "OLLAMA_ENDPOINT", "OLLAMA_MODEL"], "dataKeys": ["FRED_API_KEY (optional)", "ALPHA_VANTAGE_API_KEY (optional)"], "unused": ["GITHUB_TOKEN (GitHub Models removed)", "ANTHROPIC_API_KEY (currently empty)"], "tuning": ["BACKTEST_LLM_PACE_SECS", "ML_EXCESS_LABELS", "ML_INTRADAY_FAR_MULT/MED_MULT/NEAR_MULT", "HF_ML_MODEL_REPO_ID"] }, "researchScripts": { "note": "Offline analysis only; never imported by the production prediction path.", "key": ["research/backtest.py (LLM prompt accuracy 1D/3D/5D)", "research/ml_backtest.py (ML accuracy + target-exit P&L; authoritative DirAcc/MidHit table)", "research/ml_selection_backtest.py (top-N selection edge)", "research/ml_intraday_backtest.py (true 15-min intraday)", "research/validate_on_trades.py (validate on real paper-trade dates)", "research/strategy_combo_swing.py (strategy-confluence swing study — batched inference)"] }, "knowledgeBaseAndMemory": { "livingTruth": ["CLAUDE.md (authoritative architecture + pipeline + calibration notes)", "project_context.json (this file — structured index)"], "memoryDir": "memory/ (thin pointers + legacy findings)", "repoScopedNotes": "/memories/repo/ (agent notes: metrics, calibration, provider chain, gotchas)", "refreshPolicy": "Update CLAUDE.md + project_context.json after each meaningful feature (see .github/skills/context-updater)." }, "contextMetadata": { "generatedDate": "2026-07-24", "commitHash": "4bb9928", "pythonFilesAtRoot": 30, "source": "Derived from CLAUDE.md (633 lines) + repository structure", "maturity": "Established/Enterprise (30+ root modules, ml_predictor package, research suite)", "pathPolicy": "Relative paths only; no machine-specific or personal directories." } }