Upload 2 files

hub_dashboard_service.py

@@ -594,95 +594,204 @@ def api_trades_closed():
 
 @app.route("/api/ranker/logs/recent")
 def api_logs_recent():
-    """Get recent ranker logs — reads directly from /app/ranker_logs/*.log"""
-    limit    = int(request.args.get("limit", 50))
-    category = request.args.get("category")          # optional filter
-
-    logs = []
-
-    # ── Collect the 3 newest log files ───────────────────────────────────────
-    # FIXED: Now includes rotated files with *.log*
-    pattern = str(Path(_LOG_DIR) / "*.log*")
-    files   = sorted(glob.glob(pattern))
-    if not files:
-        return jsonify({"logs": [], "count": 0, "error": f"No *.log* files in {_LOG_DIR}"})
-
-    # Read enough lines from the tail of each file
-    raw_lines = []
-    for fpath in files[-3:]:
-        try:
-            with open(fpath, "r", encoding="utf-8", errors="replace") as f:
-                lines = f.readlines()
-                raw_lines.extend(lines[-200:])
-        except OSError:
-            pass
+    """Get recent ranker logs — reads directly from log files.
+    v2.3 FIX: wrapped in try/except so internal errors return JSON (not HTML 500).
+    Also scans multiple candidate directories so relative vs absolute paths both work.
+    """
+    try:
+        limit    = int(request.args.get("limit", 50))
+        category = request.args.get("category")          # optional filter
+
+        logs = []
+
+        # ── Find log files — try several candidate directories ────────────────
+        # The ranker may write to ./ranker_logs/ (relative to its cwd) which
+        # resolves differently from the service's _LOG_DIR.  We try all candidates
+        # and use the first one that contains *.log* files.
+        candidate_dirs = [
+            _LOG_DIR,                                               # /app/ranker_logs
+            str(Path(__file__).parent / "ranker_logs"),             # next to this file
+            "./ranker_logs",                                        # cwd-relative
+            str(Path.home() / "ranker_logs"),                       # home dir
+        ]
+        files = []
+        found_dir = None
+        for cdir in candidate_dirs:
+            pattern = str(Path(cdir) / "*.log*")
+            found = sorted(glob.glob(pattern))
+            if found:
+                files = found
+                found_dir = cdir
+                logger.debug(f"[api_logs_recent] found {len(files)} log file(s) in {cdir}")
+                break
 
-    # Reverse so newest is first
-    raw_lines.reverse()
+        if not files:
+            searched = ", ".join(candidate_dirs)
+            return jsonify({
+                "logs":  [],
+                "count": 0,
+                "error": f"No *.log* files found. Searched: {searched}",
+            })
+
+        # Read enough lines from the tail of the 3 newest files
+        raw_lines = []
+        for fpath in files[-3:]:
+            try:
+                with open(fpath, "r", encoding="utf-8", errors="replace") as f:
+                    lines = f.readlines()
+                    raw_lines.extend(lines[-200:])
+            except OSError:
+                pass
 
-    # Pre-compiled patterns for structured field extraction
-    _CAT_RE      = re.compile(r'\|\s*(INFO|DEBUG|WARNING|ERROR|CRITICAL)\s*\|\s*([A-Z_]+)\s*\|')
-    _ASSET_RE    = re.compile(r'\|\s*([A-Z][A-Z0-9_]+)\s*\|', re.IGNORECASE)
-    _TRAINING_RE = re.compile(
-        r'step=(\d+)\s*\|\s*loss=([\d.]+)\s*\|\s*lr=([\d.eE+\-]+)\s*\|\s*assets=(\d+)'
-    )
-    _JSON_RE = re.compile(r'(\{.*\})\s*$')
+        # Newest first
+        raw_lines.reverse()
 
-    for line in raw_lines[:limit]:
-        if not line.strip():
-            continue
-        ts_match = re.search(r'\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]', line)
-        if not ts_match:
-            continue
+        # Pre-compiled patterns
+        _CAT_RE      = re.compile(r'\|\s*(INFO|DEBUG|WARNING|ERROR|CRITICAL)\s*\|\s*([A-Z_]+)\s*\|')
+        _TRAINING_RE = re.compile(
+            r'step=(\d+)\s*\|\s*loss=([\d.]+)\s*\|\s*lr=([\d.eE+\-]+)\s*\|\s*assets=(\d+)'
+        )
+        _JSON_RE     = re.compile(r'(\{.*\})\s*$')
+        _ASSET_RE    = re.compile(r'\|\s*TRADE\s*\|\s*(\w+)\s*\|')
 
-        ts = ts_match.group(1)
+        for line in raw_lines[:limit]:
+            if not line.strip():
+                continue
+            ts_match = re.search(r'\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]', line)
+            if not ts_match:
+                continue
 
-        # Extract level AND category from the pipe-delimited format:
-        # "[ts] | LEVEL    | CATEGORY        | ..."
-        cat_m = _CAT_RE.search(line)
-        level    = cat_m.group(1) if cat_m else "INFO"
-        category_field = cat_m.group(2).strip() if cat_m else ""
+            ts = ts_match.group(1)
 
-        if category:
-            if category.upper() not in line.upper():
+            # Extract level + category from "[ts] | LEVEL    | CATEGORY        | ..."
+            cat_m          = _CAT_RE.search(line)
+            level          = cat_m.group(1) if cat_m else "INFO"
+            category_field = cat_m.group(2).strip() if cat_m else ""
+
+            if category and category.upper() not in line.upper():
                 continue
 
-        # Build base entry
-        entry = {
-            "timestamp":  ts,
-            "level":      level,
-            "category":   category_field,
-            "message":    line.strip(),
-            "data":       None,
-        }
+            # Try to extract asset name (present for SIGNAL / TRADE lines)
+            asset_m = _ASSET_RE.search(line)
+            asset   = asset_m.group(1) if asset_m else None
+
+            entry = {
+                "timestamp": ts,
+                "level":     level,
+                "category":  category_field,
+                "message":   line.strip(),
+                "asset":     asset,
+                "data":      None,
+            }
 
-        # Parse structured data for TRAINING entries
-        if category_field == "TRAINING":
-            tm = _TRAINING_RE.search(line)
-            if tm:
-                entry["data"] = {
-                    "step":        int(tm.group(1)),
-                    "loss":        float(tm.group(2)),
-                    "lr":          float(tm.group(3)),
-                    "asset_count": int(tm.group(4)),
-                }
-            else:
-                # Fallback: try the trailing JSON blob
-                jm = _JSON_RE.search(line)
-                if jm:
-                    try:
-                        entry["data"] = json.loads(jm.group(1))
-                    except ValueError:
-                        pass
+            if category_field == "TRAINING":
+                tm = _TRAINING_RE.search(line)
+                if tm:
+                    entry["data"] = {
+                        "step":        int(tm.group(1)),
+                        "loss":        float(tm.group(2)),
+                        "lr":          float(tm.group(3)),
+                        "asset_count": int(tm.group(4)),
+                    }
+                else:
+                    # Fallback: JSON blob at end of line
+                    jm = _JSON_RE.search(line)
+                    if jm:
+                        try:
+                            blob = json.loads(jm.group(1))
+                            # Normalise key names so JS always sees step/loss/lr/asset_count
+                            if "step" in blob:
+                                entry["data"] = {
+                                    "step":        blob.get("step", 0),
+                                    "loss":        blob.get("loss", 0.0),
+                                    "lr":          blob.get("lr", 0.0),
+                                    "asset_count": blob.get("asset_count", blob.get("assets", 0)),
+                                }
+                        except (ValueError, KeyError):
+                            pass
+
+            logs.append(entry)
+
+        return jsonify({
+            "logs":      logs,
+            "count":     len(logs),
+            "log_dir":   found_dir,
+        })
+
+    except Exception as exc:
+        logger.exception(f"[api_logs_recent] unhandled error: {exc}")
+        return jsonify({"logs": [], "count": 0, "error": str(exc)}), 200
+
+
+@app.route("/api/ranker/logs/stats")
+def api_logs_stats():
+    """Return aggregate stats parsed from log files.
+    Previously missing endpoint — dashboard Logs tab called this and got 404.
+    """
+    try:
+        by_category: dict = {}
+        by_level:    dict = {}
+        by_asset:    dict = {}
+        errors:      dict = {}
+        total        = 0
+
+        candidate_dirs = [
+            _LOG_DIR,
+            str(Path(__file__).parent / "ranker_logs"),
+            "./ranker_logs",
+            str(Path.home() / "ranker_logs"),
+        ]
+        files = []
+        for cdir in candidate_dirs:
+            found = sorted(glob.glob(str(Path(cdir) / "*.log*")))
+            if found:
+                files = found
+                break
+
+        _CAT_RE   = re.compile(r'\|\s*(INFO|DEBUG|WARNING|ERROR|CRITICAL)\s*\|\s*([A-Z_]+)\s*\|')
+        _ASSET_RE = re.compile(r'\|\s*TRADE\s*\|\s*(\w+)\s*\|')
 
-        logs.append(entry)
+        for fpath in files[-3:]:
+            try:
+                with open(fpath, "r", encoding="utf-8", errors="replace") as f:
+                    for line in f:
+                        if not line.strip():
+                            continue
+                        m = _CAT_RE.search(line)
+                        if not m:
+                            continue
+                        level = m.group(1)
+                        cat   = m.group(2).strip()
+                        by_level[level]    = by_level.get(level, 0) + 1
+                        by_category[cat]   = by_category.get(cat, 0) + 1
+                        total += 1
+                        if level in ("ERROR", "CRITICAL"):
+                            errors[cat] = errors.get(cat, 0) + 1
+                        am = _ASSET_RE.search(line)
+                        if am:
+                            a = am.group(1)
+                            by_asset[a] = by_asset.get(a, 0) + 1
+            except OSError:
+                pass
 
-    return jsonify({"logs": logs, "count": len(logs)})
+        return jsonify({
+            "total_events": total,
+            "by_level":     by_level,
+            "by_category":  by_category,
+            "by_asset":     by_asset,
+            "errors":       errors,
+            "buffer_size":  total,
+            "buffer_capacity": total,
+        })
+    except Exception as exc:
+        logger.exception(f"[api_logs_stats] error: {exc}")
+        return jsonify({"total_events": 0, "by_level": {}, "by_category": {},
+                        "by_asset": {}, "errors": {}, "error": str(exc)}), 200
 
 
 @app.route("/api/health")
 def health():
-    return jsonify({"status": "ok", "version": "v2.2-fixed"})
+    return jsonify({"status": "ok", "version": "v2.3-fixed"})
 
 
 if __name__ == "__main__":