204848 commited on
Commit
8605295
·
1 Parent(s): 804f0ba

feat: 添加HF Space卡死自愈功能,增强代理健康检查逻辑,更新配置文件

Browse files
cf-worker/.wrangler/tmp/deploy-PAi0l6/index.js ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ var __defProp = Object.defineProperty;
2
+ var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
3
+
4
+ // src/index.js
5
+ var LOGIN_PATH = "/admin/login";
6
+ var CHECK_ALL_PATH = "/admin/proxies/check-all";
7
+ var DEFAULT_HF_SPACE_REPO = "a3216/ds2api";
8
+ var RESTART_COOLDOWN_MS = 5 * 60 * 1e3;
9
+ var LAST_RESTART_KV_KEY = "last_restart";
10
+ function vercelHeaders(env) {
11
+ const headers = {};
12
+ const bypass = env.VERCEL_BYPASS_TOKEN;
13
+ if (bypass) {
14
+ headers["x-vercel-protection-bypass"] = bypass;
15
+ }
16
+ return headers;
17
+ }
18
+ __name(vercelHeaders, "vercelHeaders");
19
+ async function login(baseURL, adminKey, extraHeaders) {
20
+ const res = await fetch(`${baseURL}${LOGIN_PATH}`, {
21
+ method: "POST",
22
+ headers: { "Content-Type": "application/json", ...extraHeaders },
23
+ body: JSON.stringify({ admin_key: adminKey, expire_hours: 1 })
24
+ });
25
+ if (!res.ok) {
26
+ const text = await res.text();
27
+ throw new Error(`login failed (${res.status}): ${text}`);
28
+ }
29
+ const data = await res.json();
30
+ return data.token;
31
+ }
32
+ __name(login, "login");
33
+ async function checkAll(baseURL, token, extraHeaders) {
34
+ const res = await fetch(`${baseURL}${CHECK_ALL_PATH}`, {
35
+ method: "POST",
36
+ headers: {
37
+ "Content-Type": "application/json",
38
+ Authorization: `Bearer ${token}`,
39
+ ...extraHeaders
40
+ }
41
+ });
42
+ if (!res.ok) {
43
+ const text = await res.text();
44
+ throw new Error(`check-all failed (${res.status}): ${text}`);
45
+ }
46
+ return res.json();
47
+ }
48
+ __name(checkAll, "checkAll");
49
+ async function runCheck(env) {
50
+ const baseURL = (env.DS2API_BASE_URL || "").replace(/\/+$/, "");
51
+ const adminKey = env.ADMIN_KEY || "";
52
+ if (!baseURL || !adminKey) {
53
+ throw new Error("DS2API_BASE_URL and ADMIN_KEY secrets must be set");
54
+ }
55
+ const extraHeaders = vercelHeaders(env);
56
+ const token = await login(baseURL, adminKey, extraHeaders);
57
+ const result = await checkAll(baseURL, token, extraHeaders);
58
+ return result;
59
+ }
60
+ __name(runCheck, "runCheck");
61
+ async function fetchSpaceRuntime(repo, token) {
62
+ const url = `https://huggingface.co/api/spaces/${repo}/runtime`;
63
+ const res = await fetch(url, {
64
+ headers: { Authorization: `Bearer ${token}` }
65
+ });
66
+ if (!res.ok) {
67
+ const text = await res.text();
68
+ throw new Error(`runtime query failed (${res.status}): ${text}`);
69
+ }
70
+ const data = await res.json();
71
+ return {
72
+ stage: data.stage,
73
+ hardwareCurrent: data.hardware ? data.hardware.current : null
74
+ };
75
+ }
76
+ __name(fetchSpaceRuntime, "fetchSpaceRuntime");
77
+ function isSpaceStuck(runtime) {
78
+ if (!runtime) return true;
79
+ if (runtime.stage !== "RUNNING") return true;
80
+ if (!runtime.hardwareCurrent) return true;
81
+ return false;
82
+ }
83
+ __name(isSpaceStuck, "isSpaceStuck");
84
+ async function restartSpace(repo, token) {
85
+ const url = `https://huggingface.co/api/spaces/${repo}/restart`;
86
+ const res = await fetch(url, {
87
+ method: "POST",
88
+ headers: { Authorization: `Bearer ${token}` }
89
+ });
90
+ if (!res.ok) {
91
+ const text = await res.text();
92
+ throw new Error(`restart failed (${res.status}): ${text}`);
93
+ }
94
+ return true;
95
+ }
96
+ __name(restartSpace, "restartSpace");
97
+ async function getLastRestart(kv) {
98
+ if (!kv) return 0;
99
+ try {
100
+ const v = await kv.get(LAST_RESTART_KV_KEY);
101
+ const n = parseInt(v, 10);
102
+ return Number.isFinite(n) ? n : 0;
103
+ } catch {
104
+ return 0;
105
+ }
106
+ }
107
+ __name(getLastRestart, "getLastRestart");
108
+ async function setLastRestart(kv) {
109
+ if (!kv) return;
110
+ try {
111
+ await kv.put(LAST_RESTART_KV_KEY, String(Date.now()));
112
+ } catch {
113
+ }
114
+ }
115
+ __name(setLastRestart, "setLastRestart");
116
+ async function runSpaceCheck(env) {
117
+ const token = env.HF_TOKEN || "";
118
+ if (!token) {
119
+ throw new Error("HF_TOKEN secret must be set for space self-healing");
120
+ }
121
+ const repo = (env.HF_SPACE_REPO || DEFAULT_HF_SPACE_REPO).trim();
122
+ const runtime = await fetchSpaceRuntime(repo, token);
123
+ if (!isSpaceStuck(runtime)) {
124
+ return {
125
+ ok: true,
126
+ action: "none",
127
+ repo,
128
+ stage: runtime.stage,
129
+ hardwareCurrent: runtime.hardwareCurrent,
130
+ message: "space healthy, no action"
131
+ };
132
+ }
133
+ const last = await getLastRestart(env.SPACE_STATE);
134
+ const elapsed = Date.now() - last;
135
+ if (elapsed < RESTART_COOLDOWN_MS) {
136
+ const waitMs = RESTART_COOLDOWN_MS - elapsed;
137
+ return {
138
+ ok: true,
139
+ action: "cooldown",
140
+ repo,
141
+ stage: runtime.stage,
142
+ hardwareCurrent: runtime.hardwareCurrent,
143
+ message: `space stuck but in restart cooldown, wait ${Math.ceil(waitMs / 1e3)}s`
144
+ };
145
+ }
146
+ await restartSpace(repo, token);
147
+ await setLastRestart(env.SPACE_STATE);
148
+ return {
149
+ ok: true,
150
+ action: "restarted",
151
+ repo,
152
+ stage: runtime.stage,
153
+ hardwareCurrent: runtime.hardwareCurrent,
154
+ message: "space was stuck, restart triggered"
155
+ };
156
+ }
157
+ __name(runSpaceCheck, "runSpaceCheck");
158
+ function isSpaceCheckCron(cron) {
159
+ return cron === "*/10 * * * *";
160
+ }
161
+ __name(isSpaceCheckCron, "isSpaceCheckCron");
162
+ var index_default = {
163
+ async scheduled(event, env, ctx) {
164
+ if (isSpaceCheckCron(event.cron)) {
165
+ ctx.waitUntil(
166
+ runSpaceCheck(env).then((r) => {
167
+ if (r.action === "restarted") {
168
+ console.warn(`[space] ${r.message} repo=${r.repo} stage=${r.stage} hardware=${r.hardwareCurrent}`);
169
+ } else {
170
+ console.log(`[space] ${r.message} repo=${r.repo} stage=${r.stage} hardware=${r.hardwareCurrent}`);
171
+ }
172
+ }).catch((err) => {
173
+ console.error(`[space] check failed: ${err.message}`);
174
+ })
175
+ );
176
+ return;
177
+ }
178
+ ctx.waitUntil(
179
+ runCheck(env).then((result) => {
180
+ const items = result.items || [];
181
+ const healthy = items.filter((i) => i.healthy && !i.disabled).length;
182
+ const banned = items.filter((i) => i.disabled).length;
183
+ console.log(
184
+ `proxy health check done: ${healthy}/${items.length} healthy, ${banned} banned`
185
+ );
186
+ }).catch((err) => {
187
+ console.error(`proxy health check failed: ${err.message}`);
188
+ })
189
+ );
190
+ },
191
+ async fetch(request, env) {
192
+ const url = new URL(request.url);
193
+ if (url.pathname === "/check") {
194
+ try {
195
+ const result = await runCheck(env);
196
+ return Response.json(result);
197
+ } catch (err) {
198
+ return Response.json({ error: err.message }, { status: 500 });
199
+ }
200
+ }
201
+ if (url.pathname === "/check-space") {
202
+ try {
203
+ const result = await runSpaceCheck(env);
204
+ return Response.json(result);
205
+ } catch (err) {
206
+ return Response.json({ error: err.message }, { status: 500 });
207
+ }
208
+ }
209
+ return new Response("not found", { status: 404 });
210
+ }
211
+ };
212
+ export {
213
+ index_default as default
214
+ };
215
+ //# sourceMappingURL=index.js.map
cf-worker/.wrangler/tmp/deploy-PAi0l6/index.js.map ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../../src/index.js"],
4
+ "sourceRoot": "F:\\deepseek\\deepseek2api-main_2\\ds2api\\cf-worker\\.wrangler\\tmp\\deploy-PAi0l6",
5
+ "sourcesContent": ["/**\n * Cloudflare Worker \u2014 ds2api \u4EE3\u7406\u5065\u5EB7\u68C0\u67E5 + HF Space \u5361\u6B7B\u81EA\u6108\n *\n * \u90E8\u7F72\u6B65\u9AA4:\n * 1. cd cf-worker && npm install\n * 2. \u8BBE\u7F6E secrets:\n * wrangler secret put DS2API_BASE_URL\n * wrangler secret put ADMIN_KEY\n * wrangler secret put VERCEL_BYPASS_TOKEN (\u5982\u679C\u5F00\u4E86 Vercel Deployment Protection)\n * wrangler secret put HF_TOKEN (HF API token\uFF0C\u7528\u4E8E Space \u5361\u6B7B\u81EA\u6108)\n * 3. wrangler deploy\n *\n * \u4E24\u4E2A cron\uFF08\u89C1 wrangler.toml\uFF09\uFF1A\n * - 5 \u5C0F\u65F6\u4E00\u6B21\uFF08\"0 0,5,10,15,20 * * *\" \u7B49\u4EF7\u5199\u6CD5\uFF09\uFF1A\u4EE3\u7406\u5065\u5EB7\u68C0\u67E5\uFF08runCheck\uFF09\n * - 10 \u5206\u949F\u4E00\u6B21\uFF1AHF Space \u5361\u6B7B\u68C0\u6D4B+\u81EA\u52A8 restart\uFF08runSpaceCheck\uFF09\n *\n * \u624B\u52A8\u89E6\u53D1\uFF1A\n * GET/POST https://<worker>/check \u2192 \u4EE3\u7406\u5065\u5EB7\u68C0\u67E5\n * GET/POST https://<worker>/check-space \u2192 Space \u5361\u6B7B\u68C0\u6D4B+\u81EA\u6108\n */\n\nconst LOGIN_PATH = \"/admin/login\"\nconst CHECK_ALL_PATH = \"/admin/proxies/check-all\"\n\n// Space \u5361\u6B7B\u81EA\u6108\u76F8\u5173\u5E38\u91CF\u3002namespace/name \u901A\u8FC7 env.HF_SPACE_REPO\n// \uFF08\u683C\u5F0F \"a3216/ds2api\"\uFF09\u914D\u7F6E\uFF1B\u4E0D\u914D\u5219\u7528\u9ED8\u8BA4\u503C\u3002\nconst DEFAULT_HF_SPACE_REPO = \"a3216/ds2api\"\n// \u91CD\u542F\u51B7\u5374\uFF1A\u907F\u514D\u8FDE\u7EED restart \u628A Space \u6253\u7206\u3002\u91CD\u542F\u540E\u81F3\u5C11\u7B49 5 \u5206\u949F\u624D\u5141\u8BB8\u518D\u6B21\u91CD\u542F\u3002\nconst RESTART_COOLDOWN_MS = 5 * 60 * 1000\n// KV \u4E2D\u5B58\u4E0A\u6B21\u91CD\u542F\u65F6\u95F4\u7684 key\u3002\nconst LAST_RESTART_KV_KEY = \"last_restart\"\n\nfunction vercelHeaders(env) {\n const headers = {}\n const bypass = env.VERCEL_BYPASS_TOKEN\n if (bypass) {\n headers[\"x-vercel-protection-bypass\"] = bypass\n }\n return headers\n}\n\nasync function login(baseURL, adminKey, extraHeaders) {\n const res = await fetch(`${baseURL}${LOGIN_PATH}`, {\n method: \"POST\",\n headers: { \"Content-Type\": \"application/json\", ...extraHeaders },\n body: JSON.stringify({ admin_key: adminKey, expire_hours: 1 }),\n })\n if (!res.ok) {\n const text = await res.text()\n throw new Error(`login failed (${res.status}): ${text}`)\n }\n const data = await res.json()\n return data.token\n}\n\nasync function checkAll(baseURL, token, extraHeaders) {\n const res = await fetch(`${baseURL}${CHECK_ALL_PATH}`, {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n Authorization: `Bearer ${token}`,\n ...extraHeaders,\n },\n })\n if (!res.ok) {\n const text = await res.text()\n throw new Error(`check-all failed (${res.status}): ${text}`)\n }\n return res.json()\n}\n\nasync function runCheck(env) {\n const baseURL = (env.DS2API_BASE_URL || \"\").replace(/\\/+$/, \"\")\n const adminKey = env.ADMIN_KEY || \"\"\n\n if (!baseURL || !adminKey) {\n throw new Error(\"DS2API_BASE_URL and ADMIN_KEY secrets must be set\")\n }\n\n const extraHeaders = vercelHeaders(env)\n const token = await login(baseURL, adminKey, extraHeaders)\n const result = await checkAll(baseURL, token, extraHeaders)\n return result\n}\n\n// === HF Space \u5361\u6B7B\u81EA\u6108 ===\n\n// \u67E5\u8BE2 Space runtime \u72B6\u6001\u3002\u8FD4\u56DE { stage, hardwareCurrent }\u3002\nasync function fetchSpaceRuntime(repo, token) {\n const url = `https://huggingface.co/api/spaces/${repo}/runtime`\n const res = await fetch(url, {\n headers: { Authorization: `Bearer ${token}` },\n })\n if (!res.ok) {\n const text = await res.text()\n throw new Error(`runtime query failed (${res.status}): ${text}`)\n }\n const data = await res.json()\n return {\n stage: data.stage,\n hardwareCurrent: data.hardware ? data.hardware.current : null,\n }\n}\n\n// \u5224\u5B9A Space \u662F\u5426\u5361\u6B7B\uFF1A\u975E RUNNING \u6216\u672A\u5206\u914D\u786C\u4EF6\u3002\nfunction isSpaceStuck(runtime) {\n if (!runtime) return true\n if (runtime.stage !== \"RUNNING\") return true\n if (!runtime.hardwareCurrent) return true\n return false\n}\n\n// \u89E6\u53D1 HF Space restart\u3002\u8FD4\u56DE true \u8868\u793A\u8BF7\u6C42\u5DF2\u88AB\u63A5\u53D7\u3002\nasync function restartSpace(repo, token) {\n const url = `https://huggingface.co/api/spaces/${repo}/restart`\n const res = await fetch(url, {\n method: \"POST\",\n headers: { Authorization: `Bearer ${token}` },\n })\n if (!res.ok) {\n const text = await res.text()\n throw new Error(`restart failed (${res.status}): ${text}`)\n }\n return true\n}\n\n// \u4ECE KV \u8BFB\u4E0A\u6B21\u91CD\u542F\u65F6\u95F4\u6233\uFF08ms\uFF09\u3002KV \u672A\u7ED1\u5B9A\u6216\u65E0\u503C\u8FD4\u56DE 0\u3002\nasync function getLastRestart(kv) {\n if (!kv) return 0\n try {\n const v = await kv.get(LAST_RESTART_KV_KEY)\n const n = parseInt(v, 10)\n return Number.isFinite(n) ? n : 0\n } catch {\n return 0\n }\n}\n\nasync function setLastRestart(kv) {\n if (!kv) return\n try {\n await kv.put(LAST_RESTART_KV_KEY, String(Date.now()))\n } catch {\n // KV \u5199\u5931\u8D25\u4E0D\u963B\u65AD\u4E3B\u6D41\u7A0B\u3002\n }\n}\n\n// Space \u5361\u6B7B\u68C0\u6D4B+\u81EA\u6108\u4E3B\u903B\u8F91\u3002\nasync function runSpaceCheck(env) {\n const token = env.HF_TOKEN || \"\"\n if (!token) {\n throw new Error(\"HF_TOKEN secret must be set for space self-healing\")\n }\n const repo = (env.HF_SPACE_REPO || DEFAULT_HF_SPACE_REPO).trim()\n\n const runtime = await fetchSpaceRuntime(repo, token)\n if (!isSpaceStuck(runtime)) {\n return {\n ok: true,\n action: \"none\",\n repo,\n stage: runtime.stage,\n hardwareCurrent: runtime.hardwareCurrent,\n message: \"space healthy, no action\",\n }\n }\n\n // \u5361\u6B7B\uFF0C\u68C0\u67E5\u51B7\u5374\u3002\n const last = await getLastRestart(env.SPACE_STATE)\n const elapsed = Date.now() - last\n if (elapsed < RESTART_COOLDOWN_MS) {\n const waitMs = RESTART_COOLDOWN_MS - elapsed\n return {\n ok: true,\n action: \"cooldown\",\n repo,\n stage: runtime.stage,\n hardwareCurrent: runtime.hardwareCurrent,\n message: `space stuck but in restart cooldown, wait ${Math.ceil(waitMs / 1000)}s`,\n }\n }\n\n // \u89E6\u53D1\u91CD\u542F\u3002\n await restartSpace(repo, token)\n await setLastRestart(env.SPACE_STATE)\n return {\n ok: true,\n action: \"restarted\",\n repo,\n stage: runtime.stage,\n hardwareCurrent: runtime.hardwareCurrent,\n message: \"space was stuck, restart triggered\",\n }\n}\n\n// \u533A\u5206\u4E24\u79CD cron\uFF1A5 \u5C0F\u65F6\u4E00\u6B21\u7684\u4EE3\u7406\u68C0\u67E5 vs 10 \u5206\u949F\u4E00\u6B21\u7684 Space \u68C0\u67E5\u3002\n// scheduled.cron \u5373\u89E6\u53D1\u7684 cron \u8868\u8FBE\u5F0F\uFF08\u5B57\u7B26\u4E32\uFF09\u3002\nfunction isSpaceCheckCron(cron) {\n return cron === \"*/10 * * * *\"\n}\n\nexport default {\n async scheduled(event, env, ctx) {\n if (isSpaceCheckCron(event.cron)) {\n ctx.waitUntil(\n runSpaceCheck(env)\n .then((r) => {\n if (r.action === \"restarted\") {\n console.warn(`[space] ${r.message} repo=${r.repo} stage=${r.stage} hardware=${r.hardwareCurrent}`)\n } else {\n console.log(`[space] ${r.message} repo=${r.repo} stage=${r.stage} hardware=${r.hardwareCurrent}`)\n }\n })\n .catch((err) => {\n console.error(`[space] check failed: ${err.message}`)\n })\n )\n return\n }\n\n // \u9ED8\u8BA4\uFF1A\u4EE3\u7406\u5065\u5EB7\u68C0\u67E5 cron\u3002\n ctx.waitUntil(\n runCheck(env)\n .then((result) => {\n const items = result.items || []\n const healthy = items.filter((i) => i.healthy && !i.disabled).length\n const banned = items.filter((i) => i.disabled).length\n console.log(\n `proxy health check done: ${healthy}/${items.length} healthy, ${banned} banned`\n )\n })\n .catch((err) => {\n console.error(`proxy health check failed: ${err.message}`)\n })\n )\n },\n\n async fetch(request, env) {\n const url = new URL(request.url)\n\n if (url.pathname === \"/check\") {\n try {\n const result = await runCheck(env)\n return Response.json(result)\n } catch (err) {\n return Response.json({ error: err.message }, { status: 500 })\n }\n }\n\n if (url.pathname === \"/check-space\") {\n try {\n const result = await runSpaceCheck(env)\n return Response.json(result)\n } catch (err) {\n return Response.json({ error: err.message }, { status: 500 })\n }\n }\n\n return new Response(\"not found\", { status: 404 })\n },\n}\n"],
6
+ "mappings": ";;;;AAqBA,IAAM,aAAa;AACnB,IAAM,iBAAiB;AAIvB,IAAM,wBAAwB;AAE9B,IAAM,sBAAsB,IAAI,KAAK;AAErC,IAAM,sBAAsB;AAE5B,SAAS,cAAc,KAAK;AAC1B,QAAM,UAAU,CAAC;AACjB,QAAM,SAAS,IAAI;AACnB,MAAI,QAAQ;AACV,YAAQ,4BAA4B,IAAI;AAAA,EAC1C;AACA,SAAO;AACT;AAPS;AAST,eAAe,MAAM,SAAS,UAAU,cAAc;AACpD,QAAM,MAAM,MAAM,MAAM,GAAG,OAAO,GAAG,UAAU,IAAI;AAAA,IACjD,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,oBAAoB,GAAG,aAAa;AAAA,IAC/D,MAAM,KAAK,UAAU,EAAE,WAAW,UAAU,cAAc,EAAE,CAAC;AAAA,EAC/D,CAAC;AACD,MAAI,CAAC,IAAI,IAAI;AACX,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,UAAM,IAAI,MAAM,iBAAiB,IAAI,MAAM,MAAM,IAAI,EAAE;AAAA,EACzD;AACA,QAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,SAAO,KAAK;AACd;AAZe;AAcf,eAAe,SAAS,SAAS,OAAO,cAAc;AACpD,QAAM,MAAM,MAAM,MAAM,GAAG,OAAO,GAAG,cAAc,IAAI;AAAA,IACrD,QAAQ;AAAA,IACR,SAAS;AAAA,MACP,gBAAgB;AAAA,MAChB,eAAe,UAAU,KAAK;AAAA,MAC9B,GAAG;AAAA,IACL;AAAA,EACF,CAAC;AACD,MAAI,CAAC,IAAI,IAAI;AACX,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,UAAM,IAAI,MAAM,qBAAqB,IAAI,MAAM,MAAM,IAAI,EAAE;AAAA,EAC7D;AACA,SAAO,IAAI,KAAK;AAClB;AAde;AAgBf,eAAe,SAAS,KAAK;AAC3B,QAAM,WAAW,IAAI,mBAAmB,IAAI,QAAQ,QAAQ,EAAE;AAC9D,QAAM,WAAW,IAAI,aAAa;AAElC,MAAI,CAAC,WAAW,CAAC,UAAU;AACzB,UAAM,IAAI,MAAM,mDAAmD;AAAA,EACrE;AAEA,QAAM,eAAe,cAAc,GAAG;AACtC,QAAM,QAAQ,MAAM,MAAM,SAAS,UAAU,YAAY;AACzD,QAAM,SAAS,MAAM,SAAS,SAAS,OAAO,YAAY;AAC1D,SAAO;AACT;AAZe;AAiBf,eAAe,kBAAkB,MAAM,OAAO;AAC5C,QAAM,MAAM,qCAAqC,IAAI;AACrD,QAAM,MAAM,MAAM,MAAM,KAAK;AAAA,IAC3B,SAAS,EAAE,eAAe,UAAU,KAAK,GAAG;AAAA,EAC9C,CAAC;AACD,MAAI,CAAC,IAAI,IAAI;AACX,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,UAAM,IAAI,MAAM,yBAAyB,IAAI,MAAM,MAAM,IAAI,EAAE;AAAA,EACjE;AACA,QAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,SAAO;AAAA,IACL,OAAO,KAAK;AAAA,IACZ,iBAAiB,KAAK,WAAW,KAAK,SAAS,UAAU;AAAA,EAC3D;AACF;AAde;AAiBf,SAAS,aAAa,SAAS;AAC7B,MAAI,CAAC,QAAS,QAAO;AACrB,MAAI,QAAQ,UAAU,UAAW,QAAO;AACxC,MAAI,CAAC,QAAQ,gBAAiB,QAAO;AACrC,SAAO;AACT;AALS;AAQT,eAAe,aAAa,MAAM,OAAO;AACvC,QAAM,MAAM,qCAAqC,IAAI;AACrD,QAAM,MAAM,MAAM,MAAM,KAAK;AAAA,IAC3B,QAAQ;AAAA,IACR,SAAS,EAAE,eAAe,UAAU,KAAK,GAAG;AAAA,EAC9C,CAAC;AACD,MAAI,CAAC,IAAI,IAAI;AACX,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,UAAM,IAAI,MAAM,mBAAmB,IAAI,MAAM,MAAM,IAAI,EAAE;AAAA,EAC3D;AACA,SAAO;AACT;AAXe;AAcf,eAAe,eAAe,IAAI;AAChC,MAAI,CAAC,GAAI,QAAO;AAChB,MAAI;AACF,UAAM,IAAI,MAAM,GAAG,IAAI,mBAAmB;AAC1C,UAAM,IAAI,SAAS,GAAG,EAAE;AACxB,WAAO,OAAO,SAAS,CAAC,IAAI,IAAI;AAAA,EAClC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AATe;AAWf,eAAe,eAAe,IAAI;AAChC,MAAI,CAAC,GAAI;AACT,MAAI;AACF,UAAM,GAAG,IAAI,qBAAqB,OAAO,KAAK,IAAI,CAAC,CAAC;AAAA,EACtD,QAAQ;AAAA,EAER;AACF;AAPe;AAUf,eAAe,cAAc,KAAK;AAChC,QAAM,QAAQ,IAAI,YAAY;AAC9B,MAAI,CAAC,OAAO;AACV,UAAM,IAAI,MAAM,oDAAoD;AAAA,EACtE;AACA,QAAM,QAAQ,IAAI,iBAAiB,uBAAuB,KAAK;AAE/D,QAAM,UAAU,MAAM,kBAAkB,MAAM,KAAK;AACnD,MAAI,CAAC,aAAa,OAAO,GAAG;AAC1B,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,QAAQ;AAAA,MACR;AAAA,MACA,OAAO,QAAQ;AAAA,MACf,iBAAiB,QAAQ;AAAA,MACzB,SAAS;AAAA,IACX;AAAA,EACF;AAGA,QAAM,OAAO,MAAM,eAAe,IAAI,WAAW;AACjD,QAAM,UAAU,KAAK,IAAI,IAAI;AAC7B,MAAI,UAAU,qBAAqB;AACjC,UAAM,SAAS,sBAAsB;AACrC,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,QAAQ;AAAA,MACR;AAAA,MACA,OAAO,QAAQ;AAAA,MACf,iBAAiB,QAAQ;AAAA,MACzB,SAAS,6CAA6C,KAAK,KAAK,SAAS,GAAI,CAAC;AAAA,IAChF;AAAA,EACF;AAGA,QAAM,aAAa,MAAM,KAAK;AAC9B,QAAM,eAAe,IAAI,WAAW;AACpC,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,QAAQ;AAAA,IACR;AAAA,IACA,OAAO,QAAQ;AAAA,IACf,iBAAiB,QAAQ;AAAA,IACzB,SAAS;AAAA,EACX;AACF;AA7Ce;AAiDf,SAAS,iBAAiB,MAAM;AAC9B,SAAO,SAAS;AAClB;AAFS;AAIT,IAAO,gBAAQ;AAAA,EACb,MAAM,UAAU,OAAO,KAAK,KAAK;AAC/B,QAAI,iBAAiB,MAAM,IAAI,GAAG;AAChC,UAAI;AAAA,QACF,cAAc,GAAG,EACd,KAAK,CAAC,MAAM;AACX,cAAI,EAAE,WAAW,aAAa;AAC5B,oBAAQ,KAAK,WAAW,EAAE,OAAO,SAAS,EAAE,IAAI,UAAU,EAAE,KAAK,aAAa,EAAE,eAAe,EAAE;AAAA,UACnG,OAAO;AACL,oBAAQ,IAAI,WAAW,EAAE,OAAO,SAAS,EAAE,IAAI,UAAU,EAAE,KAAK,aAAa,EAAE,eAAe,EAAE;AAAA,UAClG;AAAA,QACF,CAAC,EACA,MAAM,CAAC,QAAQ;AACd,kBAAQ,MAAM,yBAAyB,IAAI,OAAO,EAAE;AAAA,QACtD,CAAC;AAAA,MACL;AACA;AAAA,IACF;AAGA,QAAI;AAAA,MACF,SAAS,GAAG,EACT,KAAK,CAAC,WAAW;AAChB,cAAM,QAAQ,OAAO,SAAS,CAAC;AAC/B,cAAM,UAAU,MAAM,OAAO,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,QAAQ,EAAE;AAC9D,cAAM,SAAS,MAAM,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE;AAC/C,gBAAQ;AAAA,UACN,4BAA4B,OAAO,IAAI,MAAM,MAAM,aAAa,MAAM;AAAA,QACxE;AAAA,MACF,CAAC,EACA,MAAM,CAAC,QAAQ;AACd,gBAAQ,MAAM,8BAA8B,IAAI,OAAO,EAAE;AAAA,MAC3D,CAAC;AAAA,IACL;AAAA,EACF;AAAA,EAEA,MAAM,MAAM,SAAS,KAAK;AACxB,UAAM,MAAM,IAAI,IAAI,QAAQ,GAAG;AAE/B,QAAI,IAAI,aAAa,UAAU;AAC7B,UAAI;AACF,cAAM,SAAS,MAAM,SAAS,GAAG;AACjC,eAAO,SAAS,KAAK,MAAM;AAAA,MAC7B,SAAS,KAAK;AACZ,eAAO,SAAS,KAAK,EAAE,OAAO,IAAI,QAAQ,GAAG,EAAE,QAAQ,IAAI,CAAC;AAAA,MAC9D;AAAA,IACF;AAEA,QAAI,IAAI,aAAa,gBAAgB;AACnC,UAAI;AACF,cAAM,SAAS,MAAM,cAAc,GAAG;AACtC,eAAO,SAAS,KAAK,MAAM;AAAA,MAC7B,SAAS,KAAK;AACZ,eAAO,SAAS,KAAK,EAAE,OAAO,IAAI,QAAQ,GAAG,EAAE,QAAQ,IAAI,CAAC;AAAA,MAC9D;AAAA,IACF;AAEA,WAAO,IAAI,SAAS,aAAa,EAAE,QAAQ,IAAI,CAAC;AAAA,EAClD;AACF;",
7
+ "names": []
8
+ }
cf-worker/src/index.js CHANGED
@@ -1,5 +1,5 @@
1
  /**
2
- * Cloudflare Worker — 定时刷新 ds2api 代理健康状态
3
  *
4
  * 部署步骤:
5
  * 1. cd cf-worker && npm install
@@ -7,15 +7,29 @@
7
  * wrangler secret put DS2API_BASE_URL
8
  * wrangler secret put ADMIN_KEY
9
  * wrangler secret put VERCEL_BYPASS_TOKEN (如果开了 Vercel Deployment Protection)
 
10
  * 3. wrangler deploy
11
  *
12
- * Cron 5 小时触发一次 (与 wrangler.toml 中的 crons 对应)。
13
- * 也可以手动 GET/POST https://<worker>/check 触发。
 
 
 
 
 
14
  */
15
 
16
  const LOGIN_PATH = "/admin/login"
17
  const CHECK_ALL_PATH = "/admin/proxies/check-all"
18
 
 
 
 
 
 
 
 
 
19
  function vercelHeaders(env) {
20
  const headers = {}
21
  const bypass = env.VERCEL_BYPASS_TOKEN
@@ -69,8 +83,142 @@ async function runCheck(env) {
69
  return result
70
  }
71
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
72
  export default {
73
  async scheduled(event, env, ctx) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
74
  ctx.waitUntil(
75
  runCheck(env)
76
  .then((result) => {
@@ -89,15 +237,25 @@ export default {
89
 
90
  async fetch(request, env) {
91
  const url = new URL(request.url)
92
- if (url.pathname !== "/check") {
93
- return new Response("not found", { status: 404 })
 
 
 
 
 
 
94
  }
95
 
96
- try {
97
- const result = await runCheck(env)
98
- return Response.json(result)
99
- } catch (err) {
100
- return Response.json({ error: err.message }, { status: 500 })
 
 
101
  }
 
 
102
  },
103
  }
 
1
  /**
2
+ * Cloudflare Worker — ds2api 代理健康检查 + HF Space 卡死自愈
3
  *
4
  * 部署步骤:
5
  * 1. cd cf-worker && npm install
 
7
  * wrangler secret put DS2API_BASE_URL
8
  * wrangler secret put ADMIN_KEY
9
  * wrangler secret put VERCEL_BYPASS_TOKEN (如果开了 Vercel Deployment Protection)
10
+ * wrangler secret put HF_TOKEN (HF API token,用于 Space 卡死自愈)
11
  * 3. wrangler deploy
12
  *
13
+ * 两个 cron(见 wrangler.toml):
14
+ * - 5 小时一次("0 0,5,10,15,20 * * *" 等价写法):代理健康检查(runCheck)
15
+ * - 10 分钟一次:HF Space 卡死检测+自动 restart(runSpaceCheck)
16
+ *
17
+ * 手动触发:
18
+ * GET/POST https://<worker>/check → 代理健康检查
19
+ * GET/POST https://<worker>/check-space → Space 卡死检测+自愈
20
  */
21
 
22
  const LOGIN_PATH = "/admin/login"
23
  const CHECK_ALL_PATH = "/admin/proxies/check-all"
24
 
25
+ // Space 卡死自愈相关常量。namespace/name 通过 env.HF_SPACE_REPO
26
+ // (格式 "a3216/ds2api")配置;不配则用默认值。
27
+ const DEFAULT_HF_SPACE_REPO = "a3216/ds2api"
28
+ // 重启冷却:避免连续 restart 把 Space 打爆。重启后至少等 5 分钟才允许再次重启。
29
+ const RESTART_COOLDOWN_MS = 5 * 60 * 1000
30
+ // KV 中存上次重启时间的 key。
31
+ const LAST_RESTART_KV_KEY = "last_restart"
32
+
33
  function vercelHeaders(env) {
34
  const headers = {}
35
  const bypass = env.VERCEL_BYPASS_TOKEN
 
83
  return result
84
  }
85
 
86
+ // === HF Space 卡死自愈 ===
87
+
88
+ // 查询 Space runtime 状态。返回 { stage, hardwareCurrent }。
89
+ async function fetchSpaceRuntime(repo, token) {
90
+ const url = `https://huggingface.co/api/spaces/${repo}/runtime`
91
+ const res = await fetch(url, {
92
+ headers: { Authorization: `Bearer ${token}` },
93
+ })
94
+ if (!res.ok) {
95
+ const text = await res.text()
96
+ throw new Error(`runtime query failed (${res.status}): ${text}`)
97
+ }
98
+ const data = await res.json()
99
+ return {
100
+ stage: data.stage,
101
+ hardwareCurrent: data.hardware ? data.hardware.current : null,
102
+ }
103
+ }
104
+
105
+ // 判定 Space 是否卡死:非 RUNNING 或未分配硬件。
106
+ function isSpaceStuck(runtime) {
107
+ if (!runtime) return true
108
+ if (runtime.stage !== "RUNNING") return true
109
+ if (!runtime.hardwareCurrent) return true
110
+ return false
111
+ }
112
+
113
+ // 触发 HF Space restart。返回 true 表示请求已被接受。
114
+ async function restartSpace(repo, token) {
115
+ const url = `https://huggingface.co/api/spaces/${repo}/restart`
116
+ const res = await fetch(url, {
117
+ method: "POST",
118
+ headers: { Authorization: `Bearer ${token}` },
119
+ })
120
+ if (!res.ok) {
121
+ const text = await res.text()
122
+ throw new Error(`restart failed (${res.status}): ${text}`)
123
+ }
124
+ return true
125
+ }
126
+
127
+ // 从 KV 读上次重启时间戳(ms)。KV 未绑定或无值返回 0。
128
+ async function getLastRestart(kv) {
129
+ if (!kv) return 0
130
+ try {
131
+ const v = await kv.get(LAST_RESTART_KV_KEY)
132
+ const n = parseInt(v, 10)
133
+ return Number.isFinite(n) ? n : 0
134
+ } catch {
135
+ return 0
136
+ }
137
+ }
138
+
139
+ async function setLastRestart(kv) {
140
+ if (!kv) return
141
+ try {
142
+ await kv.put(LAST_RESTART_KV_KEY, String(Date.now()))
143
+ } catch {
144
+ // KV 写失败不阻断主流程。
145
+ }
146
+ }
147
+
148
+ // Space 卡死检测+自愈主逻辑。
149
+ async function runSpaceCheck(env) {
150
+ const token = env.HF_TOKEN || ""
151
+ if (!token) {
152
+ throw new Error("HF_TOKEN secret must be set for space self-healing")
153
+ }
154
+ const repo = (env.HF_SPACE_REPO || DEFAULT_HF_SPACE_REPO).trim()
155
+
156
+ const runtime = await fetchSpaceRuntime(repo, token)
157
+ if (!isSpaceStuck(runtime)) {
158
+ return {
159
+ ok: true,
160
+ action: "none",
161
+ repo,
162
+ stage: runtime.stage,
163
+ hardwareCurrent: runtime.hardwareCurrent,
164
+ message: "space healthy, no action",
165
+ }
166
+ }
167
+
168
+ // 卡死,检查冷却。
169
+ const last = await getLastRestart(env.SPACE_STATE)
170
+ const elapsed = Date.now() - last
171
+ if (elapsed < RESTART_COOLDOWN_MS) {
172
+ const waitMs = RESTART_COOLDOWN_MS - elapsed
173
+ return {
174
+ ok: true,
175
+ action: "cooldown",
176
+ repo,
177
+ stage: runtime.stage,
178
+ hardwareCurrent: runtime.hardwareCurrent,
179
+ message: `space stuck but in restart cooldown, wait ${Math.ceil(waitMs / 1000)}s`,
180
+ }
181
+ }
182
+
183
+ // 触发重启。
184
+ await restartSpace(repo, token)
185
+ await setLastRestart(env.SPACE_STATE)
186
+ return {
187
+ ok: true,
188
+ action: "restarted",
189
+ repo,
190
+ stage: runtime.stage,
191
+ hardwareCurrent: runtime.hardwareCurrent,
192
+ message: "space was stuck, restart triggered",
193
+ }
194
+ }
195
+
196
+ // 区分两种 cron:5 小时一次的代理检查 vs 10 分钟一次的 Space 检查。
197
+ // scheduled.cron 即触发的 cron 表达式(字符串)。
198
+ function isSpaceCheckCron(cron) {
199
+ return cron === "*/10 * * * *"
200
+ }
201
+
202
  export default {
203
  async scheduled(event, env, ctx) {
204
+ if (isSpaceCheckCron(event.cron)) {
205
+ ctx.waitUntil(
206
+ runSpaceCheck(env)
207
+ .then((r) => {
208
+ if (r.action === "restarted") {
209
+ console.warn(`[space] ${r.message} repo=${r.repo} stage=${r.stage} hardware=${r.hardwareCurrent}`)
210
+ } else {
211
+ console.log(`[space] ${r.message} repo=${r.repo} stage=${r.stage} hardware=${r.hardwareCurrent}`)
212
+ }
213
+ })
214
+ .catch((err) => {
215
+ console.error(`[space] check failed: ${err.message}`)
216
+ })
217
+ )
218
+ return
219
+ }
220
+
221
+ // 默认:代理健康检查 cron。
222
  ctx.waitUntil(
223
  runCheck(env)
224
  .then((result) => {
 
237
 
238
  async fetch(request, env) {
239
  const url = new URL(request.url)
240
+
241
+ if (url.pathname === "/check") {
242
+ try {
243
+ const result = await runCheck(env)
244
+ return Response.json(result)
245
+ } catch (err) {
246
+ return Response.json({ error: err.message }, { status: 500 })
247
+ }
248
  }
249
 
250
+ if (url.pathname === "/check-space") {
251
+ try {
252
+ const result = await runSpaceCheck(env)
253
+ return Response.json(result)
254
+ } catch (err) {
255
+ return Response.json({ error: err.message }, { status: 500 })
256
+ }
257
  }
258
+
259
+ return new Response("not found", { status: 404 })
260
  },
261
  }
cf-worker/wrangler.toml CHANGED
@@ -2,8 +2,20 @@ name = "ds2api-health-check"
2
  main = "src/index.js"
3
  compatibility_date = "2024-12-01"
4
 
 
 
 
5
  [triggers]
6
- crons = ["0 */5 * * *"]
7
 
8
  [vars]
9
  # DS2API_BASE_URL = "https://your-app.vercel.app"
 
 
 
 
 
 
 
 
 
 
2
  main = "src/index.js"
3
  compatibility_date = "2024-12-01"
4
 
5
+ # 两个 cron:
6
+ # - "0 */5 * * *" 每 5 小时:代理健康检查(runCheck)
7
+ # - "*/10 * * * *" 每 10 分钟:HF Space 卡死检测+自动 restart(runSpaceCheck)
8
  [triggers]
9
+ crons = ["0 */5 * * *", "*/10 * * * *"]
10
 
11
  [vars]
12
  # DS2API_BASE_URL = "https://your-app.vercel.app"
13
+ # HF_SPACE_REPO = "a3216/ds2api" # 可选,覆盖默认监控的 Space
14
+
15
+ # KV 用于存上次 Space restart 时间,做重启冷却(避免连续 restart 打爆 Space)。
16
+ # 部署前需先创建 KV namespace:
17
+ # wrangler kv namespace create SPACE_STATE
18
+ # 然后把下面的 id 替换为返回的 id。
19
+ [[kv_namespaces]]
20
+ binding = "SPACE_STATE"
21
+ id = "da57e26ab14248c79649397afc12bc73"