feat: 面板日志页(实时tail+过滤+高亮) + 生图链路耗时埋点(image_gen 每轮耗时/fence/换号)
Browse files- app/adapters/openai_images.py +23 -5
- app/admin.py +20 -0
- app/static/admin.html +65 -0
app/adapters/openai_images.py
CHANGED
|
@@ -11,12 +11,15 @@ from __future__ import annotations
|
|
| 11 |
|
| 12 |
import base64
|
| 13 |
import json
|
|
|
|
| 14 |
import re
|
| 15 |
import time
|
| 16 |
import uuid
|
| 17 |
from pathlib import Path
|
| 18 |
from typing import Any
|
| 19 |
|
|
|
|
|
|
|
| 20 |
from fastapi import APIRouter, Depends, HTTPException, Request
|
| 21 |
from pydantic import BaseModel
|
| 22 |
|
|
@@ -138,6 +141,10 @@ async def _collect(client: Any, prompt: str, model_id: str | None,
|
|
| 138 |
"""
|
| 139 |
max_switches = getattr(client, "max_switches", 5) or 1
|
| 140 |
full = ""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
for attempt in range(max_switches):
|
| 142 |
parts: list[str] = []
|
| 143 |
kw: dict[str, Any] = {}
|
|
@@ -145,6 +152,7 @@ async def _collect(client: Any, prompt: str, model_id: str | None,
|
|
| 145 |
kw["image_model"] = image_model
|
| 146 |
if image_inputs:
|
| 147 |
kw["image_inputs"] = image_inputs
|
|
|
|
| 148 |
try:
|
| 149 |
agen = stream_with_retry(client, prompt, None, model_id=model_id, **kw)
|
| 150 |
try:
|
|
@@ -168,10 +176,10 @@ async def _collect(client: Any, prompt: str, model_id: str | None,
|
|
| 168 |
raise HTTPException(status_code=502, detail=ir.error)
|
| 169 |
if ir.kind == "text" and ir.text:
|
| 170 |
parts.append(ir.text)
|
| 171 |
-
# finish 不 break:图片 markdown 在它之后的 output_text.done 里
|
| 172 |
except TimeoutError:
|
| 173 |
# httpx 总超时(request_timeout=120s)掐断:上游超时/挂死,非账号级。
|
| 174 |
-
# 交给 _RetryingClient 换号兜底,这里原样 502 不烧号。
|
| 175 |
raise HTTPException(status_code=502, detail="image upstream timeout")
|
| 176 |
except HTTPException:
|
| 177 |
raise
|
|
@@ -179,16 +187,26 @@ async def _collect(client: Any, prompt: str, model_id: str | None,
|
|
| 179 |
raise HTTPException(status_code=502, detail=getattr(e, "detail", None) or str(e)) from e
|
| 180 |
full = "".join(parts)
|
| 181 |
urls = _extract_image_urls(parts)
|
|
|
|
| 182 |
if urls or "<tool_call>" not in full:
|
|
|
|
|
|
|
|
|
|
| 183 |
return full, urls
|
| 184 |
# fence:换号重试(当前账号标记冷却)。下次循环用新账号的 provider。
|
|
|
|
|
|
|
| 185 |
if hasattr(client, "switch_account_and_mark_failed"):
|
| 186 |
-
client.switch_account_and_mark_failed()
|
|
|
|
|
|
|
|
|
|
| 187 |
else:
|
| 188 |
break # 非 _RetryingClient(测试/直连)不换号,直接失败
|
| 189 |
# 全部账号 fence / 挂起:前面每次超时已把账号换掉并标记冷却,这里统一 502
|
| 190 |
-
|
| 191 |
-
|
|
|
|
| 192 |
|
| 193 |
|
| 194 |
@router.get("/v1/images/models")
|
|
|
|
| 11 |
|
| 12 |
import base64
|
| 13 |
import json
|
| 14 |
+
import logging
|
| 15 |
import re
|
| 16 |
import time
|
| 17 |
import uuid
|
| 18 |
from pathlib import Path
|
| 19 |
from typing import Any
|
| 20 |
|
| 21 |
+
logger = logging.getLogger(__name__)
|
| 22 |
+
|
| 23 |
from fastapi import APIRouter, Depends, HTTPException, Request
|
| 24 |
from pydantic import BaseModel
|
| 25 |
|
|
|
|
| 141 |
"""
|
| 142 |
max_switches = getattr(client, "max_switches", 5) or 1
|
| 143 |
full = ""
|
| 144 |
+
t_start = time.perf_counter()
|
| 145 |
+
acc_name = getattr(client, "_account", None).name if getattr(client, "_account", None) else "?"
|
| 146 |
+
logger.info("[image] attempt 0/%d start account=%s image_model=%s",
|
| 147 |
+
max_switches, acc_name, image_model)
|
| 148 |
for attempt in range(max_switches):
|
| 149 |
parts: list[str] = []
|
| 150 |
kw: dict[str, Any] = {}
|
|
|
|
| 152 |
kw["image_model"] = image_model
|
| 153 |
if image_inputs:
|
| 154 |
kw["image_inputs"] = image_inputs
|
| 155 |
+
t_attempt = time.perf_counter()
|
| 156 |
try:
|
| 157 |
agen = stream_with_retry(client, prompt, None, model_id=model_id, **kw)
|
| 158 |
try:
|
|
|
|
| 176 |
raise HTTPException(status_code=502, detail=ir.error)
|
| 177 |
if ir.kind == "text" and ir.text:
|
| 178 |
parts.append(ir.text)
|
| 179 |
+
# finish 不 break:图片 markdown 在它的之后的 output_text.done 里
|
| 180 |
except TimeoutError:
|
| 181 |
# httpx 总超时(request_timeout=120s)掐断:上游超时/挂死,非账号级。
|
| 182 |
+
# 交给 _RetryingClient 换账号兜底,这里原样 502 不烧号。
|
| 183 |
raise HTTPException(status_code=502, detail="image upstream timeout")
|
| 184 |
except HTTPException:
|
| 185 |
raise
|
|
|
|
| 187 |
raise HTTPException(status_code=502, detail=getattr(e, "detail", None) or str(e)) from e
|
| 188 |
full = "".join(parts)
|
| 189 |
urls = _extract_image_urls(parts)
|
| 190 |
+
dt = time.perf_counter() - t_attempt
|
| 191 |
if urls or "<tool_call>" not in full:
|
| 192 |
+
logger.info("[image_gen] attempt %d/%d DONE dt=%.1fs account=%s urls=%d chars=%d elapsed_total=%.1fs",
|
| 193 |
+
attempt, max_switches, dt, acc_name, len(urls), len(full),
|
| 194 |
+
time.perf_counter() - t_start)
|
| 195 |
return full, urls
|
| 196 |
# fence:换号重试(当前账号标记冷却)。下次循环用新账号的 provider。
|
| 197 |
+
logger.warning("[image_gen] attempt %d/%d FENCED (no media url) dt=%.1fs account=%s chars=%d",
|
| 198 |
+
attempt, max_switches, dt, acc_name, len(full))
|
| 199 |
if hasattr(client, "switch_account_and_mark_failed"):
|
| 200 |
+
new_acc = client.switch_account_and_mark_failed()
|
| 201 |
+
acc_name = getattr(new_acc, "name", None) or "?"
|
| 202 |
+
logger.info("[image_gen] attempt %d/%d switch to account=%s",
|
| 203 |
+
attempt + 1, max_switches, acc_name)
|
| 204 |
else:
|
| 205 |
break # 非 _RetryingClient(测试/直连)不换号,直接失败
|
| 206 |
# 全部账号 fence / 挂起:前面每次超时已把账号换掉并标记冷却,这里统一 502
|
| 207 |
+
logger.warning("[image_gen] ALL attempts exhausted (%.1fs total, accounts fenced/stalled)",
|
| 208 |
+
time.perf_counter() - t_start)
|
| 209 |
+
raise HTTPException(status_code=502, detail=f"all accounts fenced/stalled, no image_url: {full[:400]}")
|
| 210 |
|
| 211 |
|
| 212 |
@router.get("/v1/images/models")
|
app/admin.py
CHANGED
|
@@ -171,6 +171,26 @@ async def usage(
|
|
| 171 |
return usage_store.aggregate(window)
|
| 172 |
|
| 173 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
@router.get("/accounts/{name}")
|
| 175 |
async def get_account(request: Request, name: str, _: None = Depends(verify_admin_key)) -> Account:
|
| 176 |
"""获取单个账号完整信息(含凭据,需 admin 鉴权)。"""
|
|
|
|
| 171 |
return usage_store.aggregate(window)
|
| 172 |
|
| 173 |
|
| 174 |
+
@router.get("/logs")
|
| 175 |
+
async def logs(
|
| 176 |
+
request: Request,
|
| 177 |
+
lines: int = Query(300, ge=1, le=2000, description="返回日志尾部行数"),
|
| 178 |
+
_: None = Depends(verify_admin_key),
|
| 179 |
+
) -> dict[str, Any]:
|
| 180 |
+
"""返回网关日志文件尾部(面板「日志」页实时查看;含生图耗时埋点)。"""
|
| 181 |
+
settings = _settings(request)
|
| 182 |
+
log_path = Path(settings.log_dir) / settings.log_filename
|
| 183 |
+
if not log_path.is_file():
|
| 184 |
+
return {"file": str(log_path), "lines": [], "exists": False}
|
| 185 |
+
try:
|
| 186 |
+
with log_path.open("r", encoding="utf-8", errors="replace") as f:
|
| 187 |
+
# 只读尾部 N 行(文件可能很大)
|
| 188 |
+
tail = f.readlines()[-lines:]
|
| 189 |
+
except OSError:
|
| 190 |
+
return {"file": str(log_path), "lines": [], "exists": True, "error": "read failed"}
|
| 191 |
+
return {"file": str(log_path), "lines": tail, "exists": True, "count": len(tail)}
|
| 192 |
+
|
| 193 |
+
|
| 194 |
@router.get("/accounts/{name}")
|
| 195 |
async def get_account(request: Request, name: str, _: None = Depends(verify_admin_key)) -> Account:
|
| 196 |
"""获取单个账号完整信息(含凭据,需 admin 鉴权)。"""
|
app/static/admin.html
CHANGED
|
@@ -115,6 +115,16 @@
|
|
| 115 |
.filter-bar select:focus, .filter-bar input[type="search"]:focus { border-color: var(--accent); }
|
| 116 |
.filter-count { color: var(--muted); font-size: 12px; white-space: nowrap; }
|
| 117 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
.table-wrap { background: var(--panel); border: 1px solid var(--border); border-radius: var(--radius); overflow: hidden; }
|
| 119 |
.table-scroll { overflow-x: auto; }
|
| 120 |
table { width: 100%; border-collapse: collapse; min-width: 860px; }
|
|
@@ -212,6 +222,7 @@
|
|
| 212 |
<button class="tab active" data-tab="accounts">账号管理</button>
|
| 213 |
<button class="tab" data-tab="usage">用量统计</button>
|
| 214 |
<button class="tab" data-tab="apikeys">API Key</button>
|
|
|
|
| 215 |
</div>
|
| 216 |
|
| 217 |
<!-- ===== 账号管理视图 ===== -->
|
|
@@ -326,6 +337,25 @@
|
|
| 326 |
</div>
|
| 327 |
<p class="muted" style="font-size:12px;margin-top:10px">生成的 Key 用于客户端(Cherry Studio 等)连接 <code>/v1/*</code> 的 <code>Authorization: Bearer <key></code>;config 里的 <code>[gateway] api_key</code> 兼容并存。</p>
|
| 328 |
</div><!-- /view-apikeys -->
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 329 |
</div>
|
| 330 |
</div>
|
| 331 |
|
|
@@ -652,6 +682,38 @@
|
|
| 652 |
}
|
| 653 |
});
|
| 654 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 655 |
async function uploadFiles() {
|
| 656 |
const inp = el('upload-file');
|
| 657 |
const files = inp.files;
|
|
@@ -793,8 +855,11 @@
|
|
| 793 |
el('view-accounts').style.display = curTab === 'accounts' ? '' : 'none';
|
| 794 |
el('view-usage').style.display = curTab === 'usage' ? '' : 'none';
|
| 795 |
el('view-apikeys').style.display = curTab === 'apikeys' ? '' : 'none';
|
|
|
|
| 796 |
if (curTab === 'usage') loadUsage();
|
| 797 |
if (curTab === 'apikeys') loadApiKeys();
|
|
|
|
|
|
|
| 798 |
}));
|
| 799 |
el('usage-window').addEventListener('change', loadUsage);
|
| 800 |
el('usage-refresh').addEventListener('click', loadUsage);
|
|
|
|
| 115 |
.filter-bar select:focus, .filter-bar input[type="search"]:focus { border-color: var(--accent); }
|
| 116 |
.filter-count { color: var(--muted); font-size: 12px; white-space: nowrap; }
|
| 117 |
|
| 118 |
+
.log-box {
|
| 119 |
+
background: #0f1420; border: 1px solid var(--border); border-radius: var(--radius);
|
| 120 |
+
padding: 12px 14px; overflow: auto; max-height: 70vh;
|
| 121 |
+
}
|
| 122 |
+
.log-box pre {
|
| 123 |
+
margin: 0; color: #c7d1e0; font: 12px/1.6 Consolas, "JetBrains Mono", monospace;
|
| 124 |
+
white-space: pre-wrap; word-break: break-all;
|
| 125 |
+
}
|
| 126 |
+
.log-box .lg-warn { color: #e6b84c; }
|
| 127 |
+
.log-box .lg-err { color: #f07474; }
|
| 128 |
.table-wrap { background: var(--panel); border: 1px solid var(--border); border-radius: var(--radius); overflow: hidden; }
|
| 129 |
.table-scroll { overflow-x: auto; }
|
| 130 |
table { width: 100%; border-collapse: collapse; min-width: 860px; }
|
|
|
|
| 222 |
<button class="tab active" data-tab="accounts">账号管理</button>
|
| 223 |
<button class="tab" data-tab="usage">用量统计</button>
|
| 224 |
<button class="tab" data-tab="apikeys">API Key</button>
|
| 225 |
+
<button class="tab" data-tab="logs">日志</button>
|
| 226 |
</div>
|
| 227 |
|
| 228 |
<!-- ===== 账号管理视图 ===== -->
|
|
|
|
| 337 |
</div>
|
| 338 |
<p class="muted" style="font-size:12px;margin-top:10px">生成的 Key 用于客户端(Cherry Studio 等)连接 <code>/v1/*</code> 的 <code>Authorization: Bearer <key></code>;config 里的 <code>[gateway] api_key</code> 兼容并存。</p>
|
| 339 |
</div><!-- /view-apikeys -->
|
| 340 |
+
|
| 341 |
+
<!-- ===== 日志视图 ===== -->
|
| 342 |
+
<div id="view-logs" style="display:none">
|
| 343 |
+
<div class="toolbar">
|
| 344 |
+
<div class="bulk-bar">
|
| 345 |
+
<span class="bulk-info">网关日志(尾部 <b id="lg-count">–</b> 行)</span>
|
| 346 |
+
<button class="sm" id="lg-refresh">↻ 刷新</button>
|
| 347 |
+
<label class="muted" style="font-size:12px;display:flex;align-items:center;gap:4px">
|
| 348 |
+
<input type="checkbox" id="lg-auto" checked> 自动刷新
|
| 349 |
+
</label>
|
| 350 |
+
</div>
|
| 351 |
+
<div class="filter-bar">
|
| 352 |
+
<input type="search" id="lg-search" placeholder="🔍 过滤关键字(image/fence/error…)" autocomplete="off">
|
| 353 |
+
</div>
|
| 354 |
+
</div>
|
| 355 |
+
<div class="log-box">
|
| 356 |
+
<pre id="lg-pre">加载中…</pre>
|
| 357 |
+
</div>
|
| 358 |
+
</div><!-- /view-logs -->
|
| 359 |
</div>
|
| 360 |
</div>
|
| 361 |
|
|
|
|
| 682 |
}
|
| 683 |
});
|
| 684 |
|
| 685 |
+
// ---- 日志 ----
|
| 686 |
+
let logTimer = null;
|
| 687 |
+
let logLines = [];
|
| 688 |
+
let logFilter = '';
|
| 689 |
+
function lgHighlight(line) {
|
| 690 |
+
const escL = esc(line);
|
| 691 |
+
if (/WARNING|WARN|FENCED|fenced|超时|timeout/i.test(line)) return '<span class="lg-warn">'+escL+'</span>';
|
| 692 |
+
if (/ERROR|error|Traceback|502|failed/i.test(line)) return '<span class="lg-err">'+escL+'</span>';
|
| 693 |
+
return escL;
|
| 694 |
+
}
|
| 695 |
+
function lgRender() {
|
| 696 |
+
const rows = logFilter ? logLines.filter(l => l.toLowerCase().includes(logFilter)) : logLines;
|
| 697 |
+
el('lg-count').textContent = rows.length;
|
| 698 |
+
el('lg-pre').innerHTML = rows.map(lgHighlight).join('\n');
|
| 699 |
+
el('lg-pre').scrollTop = el('lg-pre').scrollHeight;
|
| 700 |
+
}
|
| 701 |
+
async function loadLogs() {
|
| 702 |
+
try {
|
| 703 |
+
const d = await api('/admin/logs?lines=300');
|
| 704 |
+
logLines = d.lines || [];
|
| 705 |
+
lgRender();
|
| 706 |
+
} catch (e) { if (e.message!=='key 无效或已失效') { el('lg-pre').textContent = e.message; } }
|
| 707 |
+
}
|
| 708 |
+
function stopLogAuto() { if (logTimer) { clearInterval(logTimer); logTimer = null; } }
|
| 709 |
+
function startLogAuto() {
|
| 710 |
+
stopLogAuto();
|
| 711 |
+
if (el('lg-auto').checked && curTab === 'logs') logTimer = setInterval(loadLogs, 5000);
|
| 712 |
+
}
|
| 713 |
+
el('lg-refresh').addEventListener('click', loadLogs);
|
| 714 |
+
el('lg-search').addEventListener('input', e => { logFilter = e.target.value.trim().toLowerCase(); lgRender(); });
|
| 715 |
+
el('lg-auto').addEventListener('change', startLogAuto);
|
| 716 |
+
|
| 717 |
async function uploadFiles() {
|
| 718 |
const inp = el('upload-file');
|
| 719 |
const files = inp.files;
|
|
|
|
| 855 |
el('view-accounts').style.display = curTab === 'accounts' ? '' : 'none';
|
| 856 |
el('view-usage').style.display = curTab === 'usage' ? '' : 'none';
|
| 857 |
el('view-apikeys').style.display = curTab === 'apikeys' ? '' : 'none';
|
| 858 |
+
el('view-logs').style.display = curTab === 'logs' ? '' : 'none';
|
| 859 |
if (curTab === 'usage') loadUsage();
|
| 860 |
if (curTab === 'apikeys') loadApiKeys();
|
| 861 |
+
if (curTab === 'logs') { loadLogs(); startLogAuto(); }
|
| 862 |
+
else stopLogAuto();
|
| 863 |
}));
|
| 864 |
el('usage-window').addEventListener('change', loadUsage);
|
| 865 |
el('usage-refresh').addEventListener('click', loadUsage);
|