icebear icebear0828 Claude Opus 4.6 commited on
Commit
9d6278d
·
unverified ·
1 Parent(s): d40d20b

fix: detect empty Codex responses and auto-retry with account rotation (#18)

Browse files

* fix: detect empty Codex responses and auto-retry with account rotation

Codex API intermittently returns HTTP 200 with valid SSE events but zero
text deltas. Add EmptyResponseError detection in all three translation
layers (OpenAI, Anthropic, Gemini) and a retry loop in proxy-handler
that switches accounts on empty responses (up to 3 attempts).

- Non-streaming: catch EmptyResponseError → release account → acquire
new account → retry with fresh CodexApi (max 2 retries)
- Streaming: inject "[Error] Codex returned an empty response" text
chunk when response.completed fires with no content
- All retries exhausted → 502 with clear error message

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: prevent double-release of account in empty-response retry loop

The outer catch block used the original entryId, which was already
released when EmptyResponseError switched to a new account. Introduce
activeEntryId to track the currently-active account across retries.
Also set EmptyResponseError.name for proper serialization.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: icebear0828 <icebear0828@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

CHANGELOG.md CHANGED
@@ -8,6 +8,7 @@
8
 
9
  ### Added
10
 
 
11
  - 自动提取 Chromium 版本:`extract-fingerprint.ts` 从 `package.json` 读取 Electron 版本,通过 `electron-to-chromium` 映射为 Chromium 大版本,`apply-update.ts` 自动更新 `chromium_version` 和 TLS impersonate profile
12
  - 动态模型列表:后台从 Codex 后端自动获取模型目录,与静态 YAML 合并(`src/models/model-store.ts`、`src/models/model-fetcher.ts`)
13
  - `/debug/models` 诊断端点,展示模型来源(static/backend)与刷新状态
 
8
 
9
  ### Added
10
 
11
+ - 空响应检测 + 自动换号重试:Codex API 返回 HTTP 200 但无内容时,非流式自动切换账号重试(最多 3 次),流式注入错误提示文本
12
  - 自动提取 Chromium 版本:`extract-fingerprint.ts` 从 `package.json` 读取 Electron 版本,通过 `electron-to-chromium` 映射为 Chromium 大版本,`apply-update.ts` 自动更新 `chromium_version` 和 TLS impersonate profile
13
  - 动态模型列表:后台从 Codex 后端自动获取模型目录,与静态 YAML 合并(`src/models/model-store.ts`、`src/models/model-fetcher.ts`)
14
  - `/debug/models` 诊断端点,展示模型来源(static/backend)与刷新状态
src/routes/shared/proxy-handler.ts CHANGED
@@ -12,6 +12,7 @@ import { stream } from "hono/streaming";
12
  import { randomUUID } from "crypto";
13
  import { CodexApi, CodexApiError } from "../../proxy/codex-api.js";
14
  import type { CodexResponsesRequest } from "../../proxy/codex-api.js";
 
15
  import type { AccountPool } from "../../auth/account-pool.js";
16
  import type { SessionManager } from "../../session/manager.js";
17
  import type { CookieJar } from "../../proxy/cookie-jar.js";
@@ -72,6 +73,8 @@ export async function handleProxyRequest(
72
 
73
  const { entryId, token, accountId } = acquired;
74
  const codexApi = new CodexApi(token, accountId, cookieJar, entryId);
 
 
75
 
76
  // 2. Session lookup for multi-turn
77
  const existingSession = sessionManager.findSession(req.sessionMessages);
@@ -145,51 +148,101 @@ export async function handleProxyRequest(
145
  }
146
  });
147
  } else {
148
- try {
149
- const result = await fmt.collectTranslator(
150
- codexApi,
151
- rawResponse,
152
- req.model,
153
- );
154
- if (result.responseId) {
155
- const taskId = `task-${randomUUID()}`;
156
- sessionManager.storeSession(
157
- taskId,
158
- "turn-1",
159
- req.sessionMessages,
160
  );
161
- sessionManager.updateResponseId(taskId, result.responseId);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
162
  }
163
- accountPool.release(entryId, result.usage);
164
- return c.json(result.response);
165
- } catch (collectErr) {
166
- accountPool.release(entryId);
167
- const msg = collectErr instanceof Error ? collectErr.message : "Unknown error";
168
- c.status(502);
169
- return c.json(fmt.formatError(502, msg));
170
  }
171
  }
172
  } catch (err) {
173
  // 5. Error handling with format-specific responses
174
  if (err instanceof CodexApiError) {
175
  console.error(
176
- `[${fmt.tag}] Account ${entryId} | Codex API error:`,
177
  err.message,
178
  );
179
  if (err.status === 429) {
180
  // P1-6: Count 429s as requests via encapsulated API (no direct entry mutation)
181
- accountPool.markRateLimited(entryId, { countRequest: true });
182
  c.status(429);
183
  return c.json(fmt.format429(err.message));
184
  }
185
- accountPool.release(entryId);
186
  const code = (
187
  err.status >= 400 && err.status < 600 ? err.status : 502
188
  ) as StatusCode;
189
  c.status(code);
190
  return c.json(fmt.formatError(code, err.message));
191
  }
192
- accountPool.release(entryId);
193
  throw err;
194
  }
195
  }
 
12
  import { randomUUID } from "crypto";
13
  import { CodexApi, CodexApiError } from "../../proxy/codex-api.js";
14
  import type { CodexResponsesRequest } from "../../proxy/codex-api.js";
15
+ import { EmptyResponseError } from "../../translation/codex-event-extractor.js";
16
  import type { AccountPool } from "../../auth/account-pool.js";
17
  import type { SessionManager } from "../../session/manager.js";
18
  import type { CookieJar } from "../../proxy/cookie-jar.js";
 
73
 
74
  const { entryId, token, accountId } = acquired;
75
  const codexApi = new CodexApi(token, accountId, cookieJar, entryId);
76
+ // Tracks which account the outer catch should release (updated by retry loop)
77
+ let activeEntryId = entryId;
78
 
79
  // 2. Session lookup for multi-turn
80
  const existingSession = sessionManager.findSession(req.sessionMessages);
 
148
  }
149
  });
150
  } else {
151
+ // Non-streaming: retry loop for empty responses (switch accounts)
152
+ const MAX_EMPTY_RETRIES = 2;
153
+ let currentEntryId = entryId;
154
+ let currentCodexApi = codexApi;
155
+ let currentRawResponse = rawResponse;
156
+
157
+ for (let attempt = 1; ; attempt++) {
158
+ try {
159
+ const result = await fmt.collectTranslator(
160
+ currentCodexApi,
161
+ currentRawResponse,
162
+ req.model,
163
  );
164
+ if (result.responseId) {
165
+ const taskId = `task-${randomUUID()}`;
166
+ sessionManager.storeSession(
167
+ taskId,
168
+ "turn-1",
169
+ req.sessionMessages,
170
+ );
171
+ sessionManager.updateResponseId(taskId, result.responseId);
172
+ }
173
+ accountPool.release(currentEntryId, result.usage);
174
+ return c.json(result.response);
175
+ } catch (collectErr) {
176
+ if (collectErr instanceof EmptyResponseError && attempt <= MAX_EMPTY_RETRIES) {
177
+ console.warn(
178
+ `[${fmt.tag}] Account ${currentEntryId} | Empty response (attempt ${attempt}/${MAX_EMPTY_RETRIES + 1}), switching account...`,
179
+ );
180
+ accountPool.release(currentEntryId, collectErr.usage);
181
+
182
+ // Acquire a new account
183
+ const newAcquired = accountPool.acquire();
184
+ if (!newAcquired) {
185
+ console.warn(`[${fmt.tag}] No available account for retry`);
186
+ c.status(502);
187
+ return c.json(fmt.formatError(502, "Codex returned an empty response and no other accounts are available for retry"));
188
+ }
189
+
190
+ currentEntryId = newAcquired.entryId;
191
+ activeEntryId = currentEntryId;
192
+ currentCodexApi = new CodexApi(newAcquired.token, newAcquired.accountId, cookieJar, newAcquired.entryId);
193
+ try {
194
+ currentRawResponse = await withRetry(
195
+ () => currentCodexApi.createResponse(req.codexRequest, abortController.signal),
196
+ { tag: fmt.tag },
197
+ );
198
+ } catch (retryErr) {
199
+ accountPool.release(currentEntryId);
200
+ if (retryErr instanceof CodexApiError) {
201
+ const code = (retryErr.status >= 400 && retryErr.status < 600 ? retryErr.status : 502) as StatusCode;
202
+ c.status(code);
203
+ return c.json(fmt.formatError(code, retryErr.message));
204
+ }
205
+ throw retryErr;
206
+ }
207
+ continue;
208
+ }
209
+
210
+ // Not an empty response error, or retries exhausted
211
+ accountPool.release(currentEntryId);
212
+ if (collectErr instanceof EmptyResponseError) {
213
+ console.warn(
214
+ `[${fmt.tag}] Account ${currentEntryId} | Empty response (attempt ${attempt}/${MAX_EMPTY_RETRIES + 1}), all retries exhausted`,
215
+ );
216
+ c.status(502);
217
+ return c.json(fmt.formatError(502, "Codex returned empty responses across all available accounts"));
218
+ }
219
+ const msg = collectErr instanceof Error ? collectErr.message : "Unknown error";
220
+ c.status(502);
221
+ return c.json(fmt.formatError(502, msg));
222
  }
 
 
 
 
 
 
 
223
  }
224
  }
225
  } catch (err) {
226
  // 5. Error handling with format-specific responses
227
  if (err instanceof CodexApiError) {
228
  console.error(
229
+ `[${fmt.tag}] Account ${activeEntryId} | Codex API error:`,
230
  err.message,
231
  );
232
  if (err.status === 429) {
233
  // P1-6: Count 429s as requests via encapsulated API (no direct entry mutation)
234
+ accountPool.markRateLimited(activeEntryId, { countRequest: true });
235
  c.status(429);
236
  return c.json(fmt.format429(err.message));
237
  }
238
+ accountPool.release(activeEntryId);
239
  const code = (
240
  err.status >= 400 && err.status < 600 ? err.status : 502
241
  ) as StatusCode;
242
  c.status(code);
243
  return c.json(fmt.formatError(code, err.message));
244
  }
245
+ accountPool.release(activeEntryId);
246
  throw err;
247
  }
248
  }
src/translation/codex-event-extractor.ts CHANGED
@@ -33,6 +33,16 @@ export interface FunctionCallDone {
33
  arguments: string;
34
  }
35
 
 
 
 
 
 
 
 
 
 
 
36
  export interface ExtractedEvent {
37
  typed: TypedCodexEvent;
38
  responseId?: string;
 
33
  arguments: string;
34
  }
35
 
36
+ export class EmptyResponseError extends Error {
37
+ constructor(
38
+ public readonly responseId: string | null,
39
+ public readonly usage: UsageInfo | undefined,
40
+ ) {
41
+ super("Codex returned an empty response");
42
+ this.name = "EmptyResponseError";
43
+ }
44
+ }
45
+
46
  export interface ExtractedEvent {
47
  typed: TypedCodexEvent;
48
  responseId?: string;
src/translation/codex-to-anthropic.ts CHANGED
@@ -16,7 +16,7 @@ import type {
16
  AnthropicMessagesResponse,
17
  AnthropicUsage,
18
  } from "../types/anthropic.js";
19
- import { iterateCodexEvents } from "./codex-event-extractor.js";
20
 
21
  export interface AnthropicUsageInfo {
22
  input_tokens: number;
@@ -43,6 +43,7 @@ export async function* streamCodexToAnthropic(
43
  let outputTokens = 0;
44
  let inputTokens = 0;
45
  let hasToolCalls = false;
 
46
  let contentIndex = 0;
47
  let textBlockStarted = false;
48
  const callIdsWithDeltas = new Set<string>();
@@ -100,6 +101,7 @@ export async function* streamCodexToAnthropic(
100
  // Handle function call start → close text block, open tool_use block
101
  if (evt.functionCallStart) {
102
  hasToolCalls = true;
 
103
 
104
  // Close text block if still open
105
  if (textBlockStarted) {
@@ -156,6 +158,7 @@ export async function* streamCodexToAnthropic(
156
  switch (evt.typed.type) {
157
  case "response.output_text.delta": {
158
  if (evt.textDelta) {
 
159
  // Reopen a text block if the previous one was closed (e.g. after tool calls)
160
  if (!textBlockStarted) {
161
  yield formatSSE("content_block_start", {
@@ -180,6 +183,14 @@ export async function* streamCodexToAnthropic(
180
  outputTokens = evt.usage.output_tokens;
181
  onUsage?.({ input_tokens: inputTokens, output_tokens: outputTokens });
182
  }
 
 
 
 
 
 
 
 
183
  break;
184
  }
185
  }
@@ -252,6 +263,11 @@ export async function collectCodexToAnthropicResponse(
252
  }
253
  }
254
 
 
 
 
 
 
255
  const hasToolCalls = toolUseBlocks.length > 0;
256
  const content: AnthropicContentBlock[] = [];
257
  if (fullText) {
 
16
  AnthropicMessagesResponse,
17
  AnthropicUsage,
18
  } from "../types/anthropic.js";
19
+ import { iterateCodexEvents, EmptyResponseError } from "./codex-event-extractor.js";
20
 
21
  export interface AnthropicUsageInfo {
22
  input_tokens: number;
 
43
  let outputTokens = 0;
44
  let inputTokens = 0;
45
  let hasToolCalls = false;
46
+ let hasContent = false;
47
  let contentIndex = 0;
48
  let textBlockStarted = false;
49
  const callIdsWithDeltas = new Set<string>();
 
101
  // Handle function call start → close text block, open tool_use block
102
  if (evt.functionCallStart) {
103
  hasToolCalls = true;
104
+ hasContent = true;
105
 
106
  // Close text block if still open
107
  if (textBlockStarted) {
 
158
  switch (evt.typed.type) {
159
  case "response.output_text.delta": {
160
  if (evt.textDelta) {
161
+ hasContent = true;
162
  // Reopen a text block if the previous one was closed (e.g. after tool calls)
163
  if (!textBlockStarted) {
164
  yield formatSSE("content_block_start", {
 
183
  outputTokens = evt.usage.output_tokens;
184
  onUsage?.({ input_tokens: inputTokens, output_tokens: outputTokens });
185
  }
186
+ // Inject error text if stream completed with no content
187
+ if (!hasContent && textBlockStarted) {
188
+ yield formatSSE("content_block_delta", {
189
+ type: "content_block_delta",
190
+ index: contentIndex,
191
+ delta: { type: "text_delta", text: "[Error] Codex returned an empty response. Please retry." },
192
+ });
193
+ }
194
  break;
195
  }
196
  }
 
263
  }
264
  }
265
 
266
+ // Detect empty response (HTTP 200 but no content)
267
+ if (!fullText && toolUseBlocks.length === 0 && outputTokens === 0) {
268
+ throw new EmptyResponseError(responseId, { input_tokens: inputTokens, output_tokens: outputTokens });
269
+ }
270
+
271
  const hasToolCalls = toolUseBlocks.length > 0;
272
  const content: AnthropicContentBlock[] = [];
273
  if (fullText) {
src/translation/codex-to-gemini.ts CHANGED
@@ -15,7 +15,7 @@ import type {
15
  GeminiUsageMetadata,
16
  GeminiPart,
17
  } from "../types/gemini.js";
18
- import { iterateCodexEvents } from "./codex-event-extractor.js";
19
 
20
  export interface GeminiUsageInfo {
21
  input_tokens: number;
@@ -35,6 +35,7 @@ export async function* streamCodexToGemini(
35
  ): AsyncGenerator<string> {
36
  let inputTokens = 0;
37
  let outputTokens = 0;
 
38
 
39
  for await (const evt of iterateCodexEvents(codexApi, rawResponse)) {
40
  if (evt.responseId) onResponseId?.(evt.responseId);
@@ -60,6 +61,7 @@ export async function* streamCodexToGemini(
60
 
61
  // Function call done → emit as a candidate with functionCall part
62
  if (evt.functionCallDone) {
 
63
  let args: Record<string, unknown> = {};
64
  try {
65
  args = JSON.parse(evt.functionCallDone.arguments) as Record<string, unknown>;
@@ -88,6 +90,7 @@ export async function* streamCodexToGemini(
88
  switch (evt.typed.type) {
89
  case "response.output_text.delta": {
90
  if (evt.textDelta) {
 
91
  const chunk: GeminiGenerateContentResponse = {
92
  candidates: [
93
  {
@@ -112,6 +115,23 @@ export async function* streamCodexToGemini(
112
  onUsage?.({ input_tokens: inputTokens, output_tokens: outputTokens });
113
  }
114
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
115
  // Final chunk with finishReason and usage
116
  const finalChunk: GeminiGenerateContentResponse = {
117
  candidates: [
@@ -189,6 +209,11 @@ export async function collectCodexToGeminiResponse(
189
  totalTokenCount: inputTokens + outputTokens,
190
  };
191
 
 
 
 
 
 
192
  // Build response parts: text + function calls
193
  const parts: GeminiPart[] = [];
194
  if (fullText) {
 
15
  GeminiUsageMetadata,
16
  GeminiPart,
17
  } from "../types/gemini.js";
18
+ import { iterateCodexEvents, EmptyResponseError } from "./codex-event-extractor.js";
19
 
20
  export interface GeminiUsageInfo {
21
  input_tokens: number;
 
35
  ): AsyncGenerator<string> {
36
  let inputTokens = 0;
37
  let outputTokens = 0;
38
+ let hasContent = false;
39
 
40
  for await (const evt of iterateCodexEvents(codexApi, rawResponse)) {
41
  if (evt.responseId) onResponseId?.(evt.responseId);
 
61
 
62
  // Function call done → emit as a candidate with functionCall part
63
  if (evt.functionCallDone) {
64
+ hasContent = true;
65
  let args: Record<string, unknown> = {};
66
  try {
67
  args = JSON.parse(evt.functionCallDone.arguments) as Record<string, unknown>;
 
90
  switch (evt.typed.type) {
91
  case "response.output_text.delta": {
92
  if (evt.textDelta) {
93
+ hasContent = true;
94
  const chunk: GeminiGenerateContentResponse = {
95
  candidates: [
96
  {
 
115
  onUsage?.({ input_tokens: inputTokens, output_tokens: outputTokens });
116
  }
117
 
118
+ // Inject error text if stream completed with no content
119
+ if (!hasContent) {
120
+ const emptyErrChunk: GeminiGenerateContentResponse = {
121
+ candidates: [
122
+ {
123
+ content: {
124
+ parts: [{ text: "[Error] Codex returned an empty response. Please retry." }],
125
+ role: "model",
126
+ },
127
+ index: 0,
128
+ },
129
+ ],
130
+ modelVersion: model,
131
+ };
132
+ yield `data: ${JSON.stringify(emptyErrChunk)}\n\n`;
133
+ }
134
+
135
  // Final chunk with finishReason and usage
136
  const finalChunk: GeminiGenerateContentResponse = {
137
  candidates: [
 
209
  totalTokenCount: inputTokens + outputTokens,
210
  };
211
 
212
+ // Detect empty response (HTTP 200 but no content)
213
+ if (!fullText && functionCallParts.length === 0 && outputTokens === 0) {
214
+ throw new EmptyResponseError(responseId, { input_tokens: inputTokens, output_tokens: outputTokens });
215
+ }
216
+
217
  // Build response parts: text + function calls
218
  const parts: GeminiPart[] = [];
219
  if (fullText) {
src/translation/codex-to-openai.ts CHANGED
@@ -18,7 +18,7 @@ import type {
18
  ChatCompletionToolCall,
19
  ChatCompletionChunkToolCall,
20
  } from "../types/openai.js";
21
- import { iterateCodexEvents, type UsageInfo } from "./codex-event-extractor.js";
22
 
23
  export type { UsageInfo };
24
 
@@ -42,6 +42,7 @@ export async function* streamCodexToOpenAI(
42
  const chunkId = `chatcmpl-${randomUUID().replace(/-/g, "").slice(0, 24)}`;
43
  const created = Math.floor(Date.now() / 1000);
44
  let hasToolCalls = false;
 
45
  // Track tool call indices by call_id
46
  const toolCallIndexMap = new Map<string, number>();
47
  let nextToolCallIndex = 0;
@@ -101,6 +102,7 @@ export async function* streamCodexToOpenAI(
101
  // Handle function call events
102
  if (evt.functionCallStart) {
103
  hasToolCalls = true;
 
104
  const idx = nextToolCallIndex++;
105
  toolCallIndexMap.set(evt.functionCallStart.callId, idx);
106
  const toolCall: ChatCompletionChunkToolCall = {
@@ -183,6 +185,7 @@ export async function* streamCodexToOpenAI(
183
  switch (evt.typed.type) {
184
  case "response.output_text.delta": {
185
  if (evt.textDelta) {
 
186
  yield formatSSE({
187
  id: chunkId,
188
  object: "chat.completion.chunk",
@@ -202,6 +205,22 @@ export async function* streamCodexToOpenAI(
202
 
203
  case "response.completed": {
204
  if (evt.usage) onUsage?.(evt.usage);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
205
  yield formatSSE({
206
  id: chunkId,
207
  object: "chat.completion.chunk",
@@ -265,6 +284,11 @@ export async function collectCodexResponse(
265
  }
266
  }
267
 
 
 
 
 
 
268
  const hasToolCalls = toolCalls.length > 0;
269
  const message: ChatCompletionResponse["choices"][0]["message"] = {
270
  role: "assistant",
 
18
  ChatCompletionToolCall,
19
  ChatCompletionChunkToolCall,
20
  } from "../types/openai.js";
21
+ import { iterateCodexEvents, EmptyResponseError, type UsageInfo } from "./codex-event-extractor.js";
22
 
23
  export type { UsageInfo };
24
 
 
42
  const chunkId = `chatcmpl-${randomUUID().replace(/-/g, "").slice(0, 24)}`;
43
  const created = Math.floor(Date.now() / 1000);
44
  let hasToolCalls = false;
45
+ let hasContent = false;
46
  // Track tool call indices by call_id
47
  const toolCallIndexMap = new Map<string, number>();
48
  let nextToolCallIndex = 0;
 
102
  // Handle function call events
103
  if (evt.functionCallStart) {
104
  hasToolCalls = true;
105
+ hasContent = true;
106
  const idx = nextToolCallIndex++;
107
  toolCallIndexMap.set(evt.functionCallStart.callId, idx);
108
  const toolCall: ChatCompletionChunkToolCall = {
 
185
  switch (evt.typed.type) {
186
  case "response.output_text.delta": {
187
  if (evt.textDelta) {
188
+ hasContent = true;
189
  yield formatSSE({
190
  id: chunkId,
191
  object: "chat.completion.chunk",
 
205
 
206
  case "response.completed": {
207
  if (evt.usage) onUsage?.(evt.usage);
208
+ // Inject error text if stream completed with no content
209
+ if (!hasContent) {
210
+ yield formatSSE({
211
+ id: chunkId,
212
+ object: "chat.completion.chunk",
213
+ created,
214
+ model,
215
+ choices: [
216
+ {
217
+ index: 0,
218
+ delta: { content: "[Error] Codex returned an empty response. Please retry." },
219
+ finish_reason: null,
220
+ },
221
+ ],
222
+ });
223
+ }
224
  yield formatSSE({
225
  id: chunkId,
226
  object: "chat.completion.chunk",
 
284
  }
285
  }
286
 
287
+ // Detect empty response (HTTP 200 but no content)
288
+ if (!fullText && toolCalls.length === 0 && completionTokens === 0) {
289
+ throw new EmptyResponseError(responseId, { input_tokens: promptTokens, output_tokens: completionTokens });
290
+ }
291
+
292
  const hasToolCalls = toolCalls.length > 0;
293
  const message: ChatCompletionResponse["choices"][0]["message"] = {
294
  role: "assistant",