fix: server-side validation scheduler at 4:30pm IST + history persistence

- Add background thread (_start_validation_scheduler) that auto-runs
pending validations at 4:30pm IST every NSE trading day — no longer
depends on the browser opening the validation tab
- self_learning.analyze_and_update now saves individual validated records
to learnings.json ("records" key, capped at 500) so history tab
survives DB pruning
- validation_summary always merges DB + JSON records (dedup by id) instead
of using JSON only as a fallback when DB is empty
- revalidate-all now covers JSON-only records (pruned from DB) and writes
updated hit/miss back to learnings.json
- Prune only fires when analyze_and_update returns valid data (not
insufficient_data), ensuring data is never deleted before JSON is written
- Execute response now includes hits/misses counts; toast shows
"X HIT · Y MISS" in green/red instead of just "validated X predictions"

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

app.py

@@ -2399,17 +2399,27 @@ def validation_execute():
                 logging.warning(f"Error validating snapshot {snap.get('id')}: {e}")
 
     # Auto-update self-learning after validations complete, then prune validated rows.
+    # Only prune if learnings.json was successfully written with valid data so the
+    # history tab can fall back to the records stored there.
     if validated_count > 0:
         try:
             from self_learning import analyze_and_update
-            analyze_and_update(days=30)
-            pruned = db.prune_validated_snapshots()
-            if pruned:
-                app.logger.info("Pruned %d validated snapshots after self-learning update", pruned)
+            learn_result = analyze_and_update(days=30)
+            if learn_result.get("status") != "insufficient_data":
+                pruned = db.prune_validated_snapshots()
+                if pruned:
+                    app.logger.info("Pruned %d validated snapshots after self-learning update", pruned)
         except Exception as _le:
             app.logger.warning("Self-learning update failed: %s", _le)
 
-    return _json_no_store({"validated": validated_count, "results": results})
+    hits   = sum(1 for r in results if r.get("target_hit"))
+    misses = sum(1 for r in results if not r.get("target_hit"))
+    return _json_no_store({
+        "validated": validated_count,
+        "hits": hits,
+        "misses": misses,
+        "results": results,
+    })
 
 
 @app.route("/api/ai-learn", methods=["POST"])
@@ -2419,7 +2429,9 @@ def ai_learn():
         from self_learning import analyze_and_update
         days = int((request.get_json(silent=True) or {}).get("days", 30))
         result = analyze_and_update(days=days)
-        pruned = db.prune_validated_snapshots()
+        pruned = 0
+        if result.get("status") != "insufficient_data":
+            pruned = db.prune_validated_snapshots()
         result["pruned"] = pruned
         return jsonify(result)
     except Exception as e:
@@ -2442,76 +2454,149 @@ def validation_summary():
     days = request.args.get("days", "30", type=int)
     summary = db.get_validation_summary(days=days)
     history = db.get_validation_history(limit=200)
-    
+
+    # Always merge DB records with JSON records (pruned records live in learnings.json).
+    # DB records are authoritative for rows that still exist; JSON fills the rest.
+    try:
+        import self_learning
+        ldata = self_learning._read() or {}
+        json_records = ldata.get("records", [])
+        if json_records:
+            db_ids = {r["id"] for r in history}
+            combined = list(history) + [r for r in json_records if r.get("id") not in db_ids]
+            combined.sort(key=lambda x: x.get("validated_at") or "", reverse=True)
+            history = combined[:200]
+    except Exception:
+        pass
+
     return _json_no_store({"summary": summary, "history": history})
 
 
 @app.route("/api/validation/revalidate-all", methods=["POST"])
 def validation_revalidate_all():
-    """Re-run validation for all previously validated snapshots using correct historical prices."""
-    history = db.get_validation_history(limit=9999)
+    """Re-run validation for all previously validated snapshots using correct historical prices.
+    Covers both DB rows and JSON-only records (rows pruned from DB but preserved in learnings.json).
+    """
+    db_history = db.get_validation_history(limit=9999)
+
+    # Pull JSON-only records (ids not in DB) so pruned rows can also be revalidated.
+    json_only_records = []
+    try:
+        import self_learning as _sl
+        ldata = _sl._read() or {}
+        db_ids = {r["id"] for r in db_history}
+        json_only_records = [r for r in ldata.get("records", []) if r.get("id") not in db_ids]
+    except Exception:
+        pass
+
     revalidated = 0
     results = []
-    for snap in history:
-        try:
-            snapshot_id = snap.get("id")
-            ticker = snap.get("ticker")
-            entry_price = snap.get("current_price", 0)
-            direction = snap.get("direction", "NEUTRAL")
-            target_date_str = snap.get("validation_target_date")
+    updated_json_records = {}  # id -> updated record (for JSON-only rows)
 
-            if (direction or "").upper() in ("NO TRADE", "N/A"):
-                db.mark_prediction_skipped(snapshot_id)
-                continue
-            if not entry_price or entry_price <= 0:
-                continue
+    def _revalidate_snap(snap, is_json_only=False):
+        nonlocal revalidated
+        snapshot_id = snap.get("id")
+        ticker = snap.get("ticker")
+        entry_price = snap.get("current_price", 0)
+        direction = snap.get("direction", "NEUTRAL")
+        target_date_str = snap.get("validation_target_date")
 
-            created_at_str = (snap.get("created_at") or "")[:10]
-            try:
-                window_start_str = (
-                    datetime.strptime(created_at_str, "%Y-%m-%d") + timedelta(days=1)
-                ).strftime("%Y-%m-%d")
-            except ValueError:
-                window_start_str = target_date_str
-
-            window_high, window_low, actual_price = _fetch_price_window(
-                ticker, window_start_str, target_date_str
-            )
-            if not actual_price:
-                continue
+        if (direction or "").upper() in ("NO TRADE", "N/A"):
+            if not is_json_only:
+                db.mark_prediction_skipped(snapshot_id)
+            return
+        if not entry_price or entry_price <= 0:
+            return
 
-            actual_return = round((actual_price - entry_price) / entry_price * 100, 2)
-            target_price_lo = snap.get("target_price_lo") or 0
-            target_price_hi = snap.get("target_price_hi") or 0
+        created_at_str = (snap.get("created_at") or "")[:10]
+        try:
+            window_start_str = (
+                datetime.strptime(created_at_str, "%Y-%m-%d") + timedelta(days=1)
+            ).strftime("%Y-%m-%d")
+        except ValueError:
+            window_start_str = target_date_str
+
+        window_high, window_low, actual_price = _fetch_price_window(
+            ticker, window_start_str, target_date_str
+        )
+        if not actual_price:
+            return
 
-            hit = _intraday_target_hit(
-                direction, window_high, window_low, actual_price,
-                target_price_lo, target_price_hi, entry_price,
-            )
-            if hit is None:
-                continue
-            target_hit = hit
+        actual_return = round((actual_price - entry_price) / entry_price * 100, 2)
+        target_price_lo = snap.get("target_price_lo") or 0
+        target_price_hi = snap.get("target_price_hi") or 0
 
+        hit = _intraday_target_hit(
+            direction, window_high, window_low, actual_price,
+            target_price_lo, target_price_hi, entry_price,
+        )
+        if hit is None:
+            return
+
+        validation_result = "HIT" if hit else "MISS"
+        from datetime import timezone
+        validated_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
+
+        if is_json_only:
+            # Can't update DB row (pruned); update the in-memory record for learnings.json.
+            updated = dict(snap)
+            updated.update({
+                "actual_price_at_validation": actual_price,
+                "actual_return_at_validation": actual_return,
+                "window_high": window_high,
+                "window_low": window_low,
+                "validation_result": validation_result,
+                "validated_at": validated_at,
+            })
+            updated_json_records[snapshot_id] = updated
+        else:
             db.validate_prediction(
                 snapshot_id=snapshot_id,
                 actual_price=actual_price,
                 actual_return=actual_return,
-                target_hit=target_hit,
+                target_hit=hit,
                 window_high=window_high,
                 window_low=window_low,
             )
-            results.append({
-                "ticker": ticker,
-                "timeframe": snap.get("timeframe"),
-                "direction": direction,
-                "window_high": window_high,
-                "window_low": window_low,
-                "actual_return": actual_return,
-                "target_hit": target_hit,
-            })
-            revalidated += 1
+
+        results.append({
+            "ticker": ticker,
+            "timeframe": snap.get("timeframe"),
+            "direction": direction,
+            "window_high": window_high,
+            "window_low": window_low,
+            "actual_return": actual_return,
+            "target_hit": hit,
+            "source": "json" if is_json_only else "db",
+        })
+        revalidated += 1
+
+    for snap in db_history:
+        try:
+            _revalidate_snap(snap, is_json_only=False)
         except Exception as e:
-            logging.warning(f"revalidate-all error for snapshot {snap.get('id')}: {e}")
+            logging.warning(f"revalidate-all DB error for snapshot {snap.get('id')}: {e}")
+
+    for snap in json_only_records:
+        try:
+            _revalidate_snap(snap, is_json_only=True)
+        except Exception as e:
+            logging.warning(f"revalidate-all JSON error for snapshot {snap.get('id')}: {e}")
+
+    # Write updated JSON records back to learnings.json.
+    if updated_json_records:
+        try:
+            import self_learning as _sl
+            ldata = _sl._read() or {}
+            existing = ldata.get("records", [])
+            merged = [updated_json_records.get(r["id"], r) if r.get("id") in updated_json_records else r
+                      for r in existing]
+            merged.sort(key=lambda x: x.get("validated_at") or "", reverse=True)
+            ldata["records"] = merged[:500]
+            _sl._write(ldata)
+        except Exception as e:
+            logging.warning(f"revalidate-all: failed to write JSON records: {e}")
+
     return _json_no_store({"revalidated": revalidated, "results": results})
 
 
@@ -2644,10 +2729,126 @@ def _start_trade_monitor():
     threading.Thread(target=_run, daemon=True, name="trade-monitor").start()
 
 
+def _start_validation_scheduler():
+    """Background thread: auto-run pending validations after NSE market close (3:45pm IST) daily.
+
+    Polls every 5 minutes.  Tracks which calendar date it last ran so it only
+    fires once per trading day regardless of how many times the poll fires.
+    """
+    import threading, time as _time
+
+    _IST = timezone(timedelta(hours=5, minutes=30))
+    POLL_SECS = 300  # check every 5 minutes
+
+    def _run():
+        last_ran_date = None
+        _time.sleep(15)  # let Flask finish binding before starting
+        while True:
+            try:
+                now_ist = datetime.now(timezone.utc).astimezone(_IST)
+                today = now_ist.date()
+
+                # Only fire on trading days, after 15:45 IST, and at most once per day.
+                if (
+                    last_ran_date != today
+                    and nse_is_trading_day(today)
+                    and (now_ist.hour, now_ist.minute) >= (16, 30)
+                ):
+                    due_count = db.get_validation_pending_count(due_only=True)
+                    if due_count > 0:
+                        app.logger.info(
+                            "Validation scheduler: %d items due — running auto-execute", due_count
+                        )
+                        # Import and reuse the same execute logic inline to avoid an HTTP round-trip.
+                        from self_learning import analyze_and_update
+                        pending = db.get_validation_pending(limit=500, due_only=True)
+                        actionable = [
+                            s for s in pending
+                            if (s.get("direction") or "").upper() not in ("NO TRADE", "N/A")
+                            and (s.get("current_price") or 0) > 0
+                        ]
+                        validated = 0
+                        hits = 0
+                        misses = 0
+                        today_str = today.isoformat()
+                        with ThreadPoolExecutor(max_workers=min(len(actionable), 10) or 1) as pool:
+                            def _fw(snap):
+                                tf = (snap.get("timeframe") or "").upper()
+                                tgt = snap.get("validation_target_date")
+                                try:
+                                    if tf == "INTRADAY":
+                                        return snap, *_fetch_intraday_window_capped(snap["ticker"], tgt)
+                                    created = (snap.get("created_at") or "")[:10]
+                                    try:
+                                        ws = (datetime.strptime(created, "%Y-%m-%d") + timedelta(days=1)).strftime("%Y-%m-%d")
+                                    except ValueError:
+                                        ws = tgt
+                                    wh, wl, ap = _fetch_price_window(snap["ticker"], ws, tgt)
+                                    if ap is None and tgt == today_str:
+                                        wh2, wl2, ap2 = _fetch_intraday_window_capped(snap["ticker"], tgt, cutoff_hour=15, cutoff_minute=30)
+                                        if ap2 is None:
+                                            from data_sources import fetch_live_price as _flp
+                                            live = _flp(snap["ticker"], allow_delayed=True)
+                                            if live and live > 0:
+                                                ap2, wh2, wl2 = live, (wh2 or live), (wl2 or live)
+                                        wh, wl, ap = (wh2 or wh), (wl2 or wl), ap2
+                                    return snap, wh, wl, ap
+                                except Exception as exc:
+                                    app.logger.warning("scheduler _fw(%s): %s", snap.get("ticker"), exc)
+                                    return snap, None, None, None
+
+                            for snap, wh, wl, ap in pool.map(_fw, actionable):
+                                try:
+                                    if not ap:
+                                        continue
+                                    ep = snap.get("current_price", 0)
+                                    ar = round((ap - ep) / ep * 100, 2)
+                                    tlo = snap.get("target_price_lo") or 0
+                                    thi = snap.get("target_price_hi") or 0
+                                    direction = (snap.get("direction") or "NEUTRAL").upper()
+                                    hit = _intraday_target_hit(direction, wh, wl, ap, tlo, thi, ep)
+                                    if hit is None:
+                                        db.mark_prediction_skipped(snap.get("id"))
+                                        continue
+                                    db.validate_prediction(snap.get("id"), ap, ar, hit, wh, wl)
+                                    validated += 1
+                                    if hit:
+                                        hits += 1
+                                    else:
+                                        misses += 1
+                                except Exception as exc:
+                                    app.logger.warning("scheduler validate(%s): %s", snap.get("id"), exc)
+
+                        app.logger.info(
+                            "Validation scheduler done: %d validated (%d HIT, %d MISS)",
+                            validated, hits, misses,
+                        )
+
+                        if validated > 0:
+                            try:
+                                learn_result = analyze_and_update(days=30)
+                                if learn_result.get("status") != "insufficient_data":
+                                    pruned = db.prune_validated_snapshots()
+                                    if pruned:
+                                        app.logger.info("Scheduler pruned %d validated snapshots", pruned)
+                            except Exception as le:
+                                app.logger.warning("Scheduler self-learning update failed: %s", le)
+
+                    last_ran_date = today
+
+            except Exception as exc:
+                app.logger.warning("Validation scheduler error: %s", exc)
+
+            _time.sleep(POLL_SECS)
+
+    threading.Thread(target=_run, daemon=True, name="validation-scheduler").start()
+
+
 # Start background services at import time so both `python app.py` and
 # WSGI servers (gunicorn) warm the top5 cache and run the trade monitor.
 _prewarm_top5()
 _start_trade_monitor()
+_start_validation_scheduler()
 
 if __name__ == "__main__":
     port = int(os.environ.get("PORT", 7860))

self_learning.py

@@ -43,6 +43,9 @@ def analyze_and_update(days: int = 30) -> dict:
     merged_conf: dict = {}
     historical_total = 0
     historical_hits = 0
+    # Accumulated individual records from prior prune cycles (dedup by id).
+    existing_records: list = existing.get("records", [])
+    existing_record_ids: set = {r["id"] for r in existing_records if r.get("id")}
 
     if is_existing_valid:
         for k, v in existing.get("buckets", {}).items():
@@ -61,6 +64,7 @@ def analyze_and_update(days: int = 30) -> dict:
 
     # Merge new DB rows on top of accumulated counts.
     new_hits = 0
+    new_records = []
     for r in new_validated:
         direction = r["direction"].upper()
         timeframe = (r.get("timeframe") or "1D").upper()
@@ -80,6 +84,34 @@ def analyze_and_update(days: int = 30) -> dict:
 
         new_hits += int(hit)
 
+        # Keep a slimmed individual record for history tab fallback after DB pruning.
+        if r.get("id") and r["id"] not in existing_record_ids:
+            new_records.append({
+                "id": r["id"],
+                "ticker": r.get("ticker"),
+                "timeframe": r.get("timeframe"),
+                "direction": r.get("direction"),
+                "confidence": r.get("confidence"),
+                "target_price_lo": r.get("target_price_lo"),
+                "target_price_hi": r.get("target_price_hi"),
+                "predicted_return_lo": r.get("predicted_return_lo"),
+                "predicted_return_hi": r.get("predicted_return_hi"),
+                "current_price": r.get("current_price"),
+                "actual_price_at_validation": r.get("actual_price_at_validation"),
+                "actual_return_at_validation": r.get("actual_return_at_validation"),
+                "window_high": r.get("window_high"),
+                "window_low": r.get("window_low"),
+                "validation_result": r.get("validation_result"),
+                "created_at": r.get("created_at"),
+                "validated_at": r.get("validated_at"),
+                "validation_target_date": r.get("validation_target_date"),
+            })
+
+    # Merge new records with existing, sort newest-validated first, cap at 500.
+    all_records = new_records + existing_records
+    all_records.sort(key=lambda x: x.get("validated_at") or "", reverse=True)
+    all_records = all_records[:500]
+
     total = historical_total + len(new_validated)
 
     if total < _MIN_SAMPLES:
@@ -89,6 +121,7 @@ def analyze_and_update(days: int = 30) -> dict:
             "new_in_this_run": len(new_validated),
             "min_required": _MIN_SAMPLES,
             "calibration_notes": [],
+            "records": all_records,
         }
         _write(result)
         return result
@@ -155,6 +188,7 @@ def analyze_and_update(days: int = 30) -> dict:
             k: {"hits": v["hits"], "total": v["total"], "hit_rate": round(v["hits"] / v["total"], 3)}
             for k, v in merged_conf.items() if v["total"] >= 5
         },
+        "records": all_records,
     }
     _write(result)
     logging.info(

static/app.js

@@ -2422,11 +2422,13 @@ async function loadValidation() {
           const execData = await execRes.json();
           autoValidated = execData.validated || 0;
           if (autoValidated > 0) {
+            const hits   = execData.hits   ?? 0;
+            const misses = execData.misses ?? 0;
             const toast = document.createElement('div');
             toast.className = 'val-toast';
-            toast.textContent = `✓ Auto-validated ${autoValidated} prediction${autoValidated !== 1 ? 's' : ''}`;
+            toast.innerHTML = `✓ Validated ${autoValidated}: <span class="val-toast-hit">${hits} HIT</span> · <span class="val-toast-miss">${misses} MISS</span>`;
             document.body.appendChild(toast);
-            setTimeout(() => toast.remove(), 4000);
+            setTimeout(() => toast.remove(), 6000);
             // Reload both pending and summary so stat cards reflect new validations
             const [summRes2, pendRes2] = await Promise.all([
               fetch('/api/validation/summary', { cache: 'no-store' }),

static/style.css

@@ -815,10 +815,12 @@ input::placeholder { color: var(--text-dim); }
 .val-toast {
   position: fixed; bottom: 24px; right: 24px; z-index: 9999;
   background: var(--bg2); border: 1px solid var(--green); border-radius: var(--radius);
-  color: var(--green); font-size: 13px; font-weight: 600;
+  color: var(--text); font-size: 13px; font-weight: 600;
   padding: 10px 18px; box-shadow: 0 4px 16px rgba(0,0,0,0.4);
   animation: fadeInUp 0.3s ease;
 }
+.val-toast-hit  { color: var(--green); }
+.val-toast-miss { color: var(--red, #e55); }
 @keyframes fadeInUp {
   from { opacity: 0; transform: translateY(12px); }
   to   { opacity: 1; transform: translateY(0); }