| package cooldown |
|
|
| import ( |
| "context" |
| "testing" |
| "time" |
|
|
| "ccLoad/internal/model" |
| "ccLoad/internal/storage" |
| "ccLoad/internal/testutil" |
| "ccLoad/internal/util" |
| ) |
|
|
| |
| func TestNewManager(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
|
|
| manager := NewManager(store, nil) |
| if manager == nil { |
| t.Fatal("NewManager should not return nil") |
| } |
| if manager.store == nil { |
| t.Error("Manager.store should not be nil") |
| } |
| } |
|
|
| |
| func TestHandleError_ClientError(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| |
| cfg := createTestChannel(t, store, "test-client-error") |
|
|
| testCases := []struct { |
| name string |
| statusCode int |
| errorBody []byte |
| }{ |
| {"406不可接受", 406, []byte(`{"error":"not acceptable"}`)}, |
| |
| |
| } |
|
|
| for _, tc := range testCases { |
| t.Run(tc.name, func(t *testing.T) { |
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| StatusCode: tc.statusCode, |
| ErrorBody: tc.errorBody, |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
|
|
| if action != ActionReturnClient { |
| t.Errorf("Expected ActionReturnClient for %d, got %v", tc.statusCode, action) |
| } |
|
|
| |
| channelCfg, _ := store.GetConfig(ctx, cfg.ID) |
| if channelCfg.CooldownUntil > 0 { |
| t.Errorf("Client error should not trigger cooldown") |
| } |
| }) |
| } |
| } |
|
|
| |
| func TestHandleError_KeyLevelError(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| |
| cfg := createTestChannel(t, store, "test-key-error") |
| keys := make([]*model.APIKey, 3) |
| for i := 0; i < 3; i++ { |
| keys[i] = &model.APIKey{ |
| ChannelID: cfg.ID, |
| KeyIndex: i, |
| APIKey: "sk-key-" + string(rune('0'+i)), |
| KeyStrategy: model.KeyStrategySequential, |
| } |
| } |
| _ = store.CreateAPIKeysBatch(ctx, keys) |
|
|
| testCases := []struct { |
| name string |
| statusCode int |
| errorBody []byte |
| }{ |
| {"401未授权", 401, []byte(`{"error":{"type":"authentication_error"}}`)}, |
| {"403禁止访问", 403, []byte(`{"error":{"type":"permission_error"}}`)}, |
| {"429限流", 429, []byte(`{"error":{"type":"rate_limit_error"}}`)}, |
| } |
|
|
| for _, tc := range testCases { |
| t.Run(tc.name, func(t *testing.T) { |
| keyIndex := 0 |
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: keyIndex, |
| StatusCode: tc.statusCode, |
| ErrorBody: tc.errorBody, |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
|
|
| if action != ActionRetryKey { |
| t.Errorf("Expected ActionRetryKey for %d, got %v", tc.statusCode, action) |
| } |
|
|
| |
| cooldownUntil, exists := getKeyCooldownUntil(ctx, store, cfg.ID, keyIndex) |
| if !exists || cooldownUntil.Before(time.Now()) { |
| t.Errorf("Key should be cooled down for status %d", tc.statusCode) |
| } |
|
|
| |
| channelCfg, _ := store.GetConfig(ctx, cfg.ID) |
| if channelCfg.CooldownUntil > 0 && time.Unix(channelCfg.CooldownUntil, 0).After(time.Now()) { |
| t.Errorf("Channel should not be cooled down for key-level error") |
| } |
| }) |
| } |
| } |
|
|
| |
| func TestHandleError_ChannelLevelError(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| cfg := createTestChannel(t, store, "test-channel-error") |
|
|
| testCases := []struct { |
| name string |
| statusCode int |
| errorBody []byte |
| }{ |
| {"404未找到", 404, []byte(`{"error":"not found"}`)}, |
| {"405方法不允许", 405, []byte(`{"error":"method not allowed"}`)}, |
| {"500内部错误", 500, []byte(`{"error":"internal server error"}`)}, |
| {"502网关错误", 502, []byte(`{"error":"bad gateway"}`)}, |
| {"503服务不可用", 503, []byte(`{"error":"service unavailable"}`)}, |
| {"504网关超时", 504, []byte(`{"error":"gateway timeout"}`)}, |
| } |
|
|
| for _, tc := range testCases { |
| t.Run(tc.name, func(t *testing.T) { |
| |
| _ = store.ResetChannelCooldown(ctx, cfg.ID) |
|
|
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: -1, |
| StatusCode: tc.statusCode, |
| ErrorBody: tc.errorBody, |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
|
|
| if action != ActionRetryChannel { |
| t.Errorf("Expected ActionRetryChannel for %d, got %v", tc.statusCode, action) |
| } |
|
|
| |
| channelCfg, _ := store.GetConfig(ctx, cfg.ID) |
| if channelCfg.CooldownUntil == 0 || time.Unix(channelCfg.CooldownUntil, 0).Before(time.Now()) { |
| t.Errorf("Channel should be cooled down for status %d", tc.statusCode) |
| } |
| }) |
| } |
| } |
|
|
| |
| func TestHandleError_SingleKeyUpgrade(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| |
| cfg := createTestChannel(t, store, "test-single-key") |
| _ = store.CreateAPIKeysBatch(ctx, []*model.APIKey{{ |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| APIKey: "sk-only-key", |
| KeyStrategy: model.KeyStrategySequential, |
| }}) |
|
|
| |
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| StatusCode: 401, |
| ErrorBody: []byte(`{"error":{"type":"authentication_error"}}`), |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
|
|
| |
| if action != ActionRetryChannel { |
| t.Errorf("Expected ActionRetryChannel for single-key channel, got %v", action) |
| } |
|
|
| |
| channelCfg, _ := store.GetConfig(ctx, cfg.ID) |
| if channelCfg.CooldownUntil == 0 { |
| t.Error("Single-key channel should be cooled down at channel level") |
| } |
| } |
|
|
| |
| func TestHandleError_NetworkError(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| cfg := createTestChannel(t, store, "test-network-error") |
|
|
| testCases := []struct { |
| name string |
| statusCode int |
| expectedAction Action |
| description string |
| }{ |
| { |
| name: "首字节超时(598)", |
| statusCode: 598, |
| expectedAction: ActionRetryChannel, |
| description: "First byte timeout should trigger channel-level cooldown", |
| }, |
| { |
| name: "网关超时(504)", |
| statusCode: 504, |
| expectedAction: ActionRetryChannel, |
| description: "Gateway timeout should trigger channel-level cooldown", |
| }, |
| { |
| name: "其他网络错误(502)", |
| statusCode: 502, |
| expectedAction: ActionRetryChannel, |
| description: "Other network errors should be channel-level", |
| }, |
| } |
|
|
| |
| netKeys := make([]*model.APIKey, 2) |
| for i := 0; i < 2; i++ { |
| netKeys[i] = &model.APIKey{ |
| ChannelID: cfg.ID, |
| KeyIndex: i, |
| APIKey: "sk-net-key-" + string(rune('0'+i)), |
| KeyStrategy: model.KeyStrategySequential, |
| } |
| } |
| _ = store.CreateAPIKeysBatch(ctx, netKeys) |
|
|
| for _, tc := range testCases { |
| t.Run(tc.name, func(t *testing.T) { |
| |
| _ = store.ResetChannelCooldown(ctx, cfg.ID) |
|
|
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| StatusCode: tc.statusCode, |
| ErrorBody: nil, |
| IsNetworkError: true, |
| Headers: nil, |
| }) |
|
|
| if action != tc.expectedAction { |
| t.Errorf("%s: expected %v, got %v", tc.description, tc.expectedAction, action) |
| } |
| }) |
| } |
| } |
|
|
| |
| func TestClearChannelCooldown(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| cfg := createTestChannel(t, store, "test-clear-channel") |
|
|
| |
| _ = manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: -1, |
| StatusCode: 500, |
| ErrorBody: nil, |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
|
|
| |
| channelCfg, _ := store.GetConfig(ctx, cfg.ID) |
| if channelCfg.CooldownUntil == 0 { |
| t.Fatal("Channel should be cooled down") |
| } |
|
|
| |
| err := manager.ClearChannelCooldown(ctx, cfg.ID) |
| if err != nil { |
| t.Fatalf("ClearChannelCooldown failed: %v", err) |
| } |
|
|
| |
| channelCfg, _ = store.GetConfig(ctx, cfg.ID) |
| if channelCfg.CooldownUntil != 0 { |
| t.Error("Channel cooldown should be cleared") |
| } |
| } |
|
|
| |
| func TestClearKeyCooldown(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| cfg := createTestChannel(t, store, "test-clear-key") |
| _ = store.CreateAPIKeysBatch(ctx, []*model.APIKey{ |
| { |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| APIKey: "sk-test-clear", |
| KeyStrategy: model.KeyStrategySequential, |
| }, |
| { |
| ChannelID: cfg.ID, |
| KeyIndex: 1, |
| APIKey: "sk-test-clear-2", |
| KeyStrategy: model.KeyStrategySequential, |
| }, |
| }) |
|
|
| |
| _ = manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| StatusCode: 401, |
| ErrorBody: []byte(`{"error":{"type":"authentication_error"}}`), |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
|
|
| |
| cooldownUntil, exists := getKeyCooldownUntil(ctx, store, cfg.ID, 0) |
| if !exists || cooldownUntil.Before(time.Now()) { |
| t.Fatal("Key should be cooled down") |
| } |
|
|
| |
| err := manager.ClearKeyCooldown(ctx, cfg.ID, 0) |
| if err != nil { |
| t.Fatalf("ClearKeyCooldown failed: %v", err) |
| } |
|
|
| |
| _, exists = getKeyCooldownUntil(ctx, store, cfg.ID, 0) |
| if exists { |
| t.Error("Key cooldown should be cleared") |
| } |
| } |
|
|
| |
| func TestHandleError_EdgeCases(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| t.Run("不存在的渠道", func(t *testing.T) { |
| |
| |
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: 99999, |
| KeyIndex: 0, |
| StatusCode: 500, |
| ErrorBody: nil, |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
| |
| if action != ActionRetryChannel { |
| t.Errorf("Expected ActionRetryChannel when cooldown fails, got %v", action) |
| } |
| }) |
|
|
| t.Run("负数keyIndex", func(t *testing.T) { |
| cfg := createTestChannel(t, store, "test-negative-key") |
| |
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: -1, |
| StatusCode: 500, |
| ErrorBody: nil, |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
| if action != ActionRetryChannel { |
| t.Errorf("Expected ActionRetryChannel for channel-level error") |
| } |
| }) |
|
|
| t.Run("nil错误体", func(t *testing.T) { |
| cfg := createTestChannel(t, store, "test-nil-body") |
| |
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: -1, |
| StatusCode: 500, |
| ErrorBody: nil, |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
| if action != ActionRetryChannel { |
| t.Error("Should classify 500 as channel-level even with nil body") |
| } |
| }) |
|
|
| t.Run("空错误体", func(t *testing.T) { |
| cfg := createTestChannel(t, store, "test-empty-body") |
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: -1, |
| StatusCode: 503, |
| ErrorBody: []byte{}, |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
| if action != ActionRetryChannel { |
| t.Error("Should classify 503 as channel-level") |
| } |
| }) |
| } |
|
|
| |
| |
| func TestHandleError_RateLimitClassification(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| |
| cfg := createTestChannel(t, store, "test-429-classification") |
| rateKeys := make([]*model.APIKey, 3) |
| for i := 0; i < 3; i++ { |
| rateKeys[i] = &model.APIKey{ |
| ChannelID: cfg.ID, |
| KeyIndex: i, |
| APIKey: "sk-ratelimit-" + string(rune('0'+i)), |
| KeyStrategy: model.KeyStrategySequential, |
| } |
| } |
| _ = store.CreateAPIKeysBatch(ctx, rateKeys) |
|
|
| testCases := []struct { |
| name string |
| headers map[string][]string |
| responseBody []byte |
| expectedAction Action |
| description string |
| }{ |
| { |
| name: "429-Retry-After大于60秒", |
| headers: map[string][]string{ |
| "Retry-After": {"120"}, |
| }, |
| responseBody: []byte(`{"error":{"type":"rate_limit_error"}}`), |
| expectedAction: ActionRetryChannel, |
| description: "Retry-After > 60s indicates account/IP level rate limit", |
| }, |
| { |
| name: "429-Retry-After小于60秒", |
| headers: map[string][]string{ |
| "Retry-After": {"30"}, |
| }, |
| responseBody: []byte(`{"error":{"type":"rate_limit_error"}}`), |
| expectedAction: ActionRetryKey, |
| description: "Retry-After <= 60s indicates key-level rate limit", |
| }, |
| { |
| name: "429-Retry-After为HTTP日期", |
| headers: map[string][]string{ |
| "Retry-After": {"Wed, 29 Oct 2025 12:00:00 GMT"}, |
| }, |
| responseBody: []byte(`{"error":{"type":"rate_limit_error"}}`), |
| expectedAction: ActionRetryChannel, |
| description: "HTTP date format typically indicates long-term rate limit", |
| }, |
| { |
| name: "429-X-RateLimit-Scope-global", |
| headers: map[string][]string{ |
| "X-Ratelimit-Scope": {"global"}, |
| }, |
| responseBody: []byte(`{"error":{"type":"rate_limit_error"}}`), |
| expectedAction: ActionRetryChannel, |
| description: "Global scope indicates channel-level rate limit", |
| }, |
| { |
| name: "429-X-RateLimit-Scope-ip", |
| headers: map[string][]string{ |
| "X-Ratelimit-Scope": {"ip"}, |
| }, |
| responseBody: []byte(`{"error":{"type":"rate_limit_error"}}`), |
| expectedAction: ActionRetryChannel, |
| description: "IP scope indicates channel-level rate limit", |
| }, |
| { |
| name: "429-X-RateLimit-Scope-account", |
| headers: map[string][]string{ |
| "X-Ratelimit-Scope": {"account"}, |
| }, |
| responseBody: []byte(`{"error":{"type":"rate_limit_error"}}`), |
| expectedAction: ActionRetryChannel, |
| description: "Account scope indicates channel-level rate limit", |
| }, |
| { |
| name: "429-响应体包含ip-rate-limit", |
| headers: map[string][]string{ |
| "Content-Type": {"application/json"}, |
| }, |
| responseBody: []byte(`{"error":{"message":"IP rate limit exceeded"}}`), |
| expectedAction: ActionRetryChannel, |
| description: "Response body with 'ip rate limit' indicates channel-level", |
| }, |
| { |
| name: "429-响应体包含account-rate-limit", |
| headers: map[string][]string{ |
| "Content-Type": {"application/json"}, |
| }, |
| responseBody: []byte(`{"error":{"message":"Account rate limit exceeded"}}`), |
| expectedAction: ActionRetryChannel, |
| description: "Response body with 'account rate limit' indicates channel-level", |
| }, |
| { |
| name: "429-响应体包含global-rate-limit", |
| headers: map[string][]string{ |
| "Content-Type": {"application/json"}, |
| }, |
| responseBody: []byte(`{"error":{"message":"Global rate limit exceeded"}}`), |
| expectedAction: ActionRetryChannel, |
| description: "Response body with 'global rate limit' indicates channel-level", |
| }, |
| { |
| name: "429-无特殊headers和响应体", |
| headers: map[string][]string{ |
| "Content-Type": {"application/json"}, |
| }, |
| responseBody: []byte(`{"error":{"type":"rate_limit_error"}}`), |
| expectedAction: ActionRetryKey, |
| description: "Default to key-level when no special indicators present", |
| }, |
| { |
| name: "429-nil-headers", |
| headers: nil, |
| responseBody: []byte(`{"error":{"type":"rate_limit_error"}}`), |
| expectedAction: ActionRetryKey, |
| description: "Nil headers should default to key-level", |
| }, |
| } |
|
|
| for _, tc := range testCases { |
| t.Run(tc.name, func(t *testing.T) { |
| |
| _ = store.ResetChannelCooldown(ctx, cfg.ID) |
| for i := 0; i < 3; i++ { |
| _ = store.ResetKeyCooldown(ctx, cfg.ID, i) |
| } |
|
|
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| StatusCode: 429, |
| ErrorBody: tc.responseBody, |
| IsNetworkError: false, |
| Headers: tc.headers, |
| }) |
|
|
| if action != tc.expectedAction { |
| t.Errorf("%s: expected %v, got %v", tc.description, tc.expectedAction, action) |
| } |
|
|
| |
| switch tc.expectedAction { |
| case ActionRetryChannel: |
| channelCfg, _ := store.GetConfig(ctx, cfg.ID) |
| if channelCfg.CooldownUntil == 0 || time.Unix(channelCfg.CooldownUntil, 0).Before(time.Now()) { |
| t.Errorf("Channel should be cooled down for %s", tc.name) |
| } |
| case ActionRetryKey: |
| cooldownUntil, exists := getKeyCooldownUntil(ctx, store, cfg.ID, 0) |
| if !exists || cooldownUntil.Before(time.Now()) { |
| t.Errorf("Key should be cooled down for %s", tc.name) |
| } |
| } |
|
|
| t.Logf("[INFO] %s: %s", tc.name, tc.description) |
| }) |
| } |
| } |
|
|
| func TestHandleError_Structured429QuotaCooldown(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| cfg := createTestChannel(t, store, "test-structured-429-quota") |
| keys := make([]*model.APIKey, 2) |
| for i := 0; i < 2; i++ { |
| keys[i] = &model.APIKey{ |
| ChannelID: cfg.ID, |
| KeyIndex: i, |
| APIKey: "sk-quota-" + string(rune('0'+i)), |
| KeyStrategy: model.KeyStrategySequential, |
| } |
| } |
| _ = store.CreateAPIKeysBatch(ctx, keys) |
|
|
| t.Run("DAILY_LIMIT_EXCEEDED cools key until next local day", func(t *testing.T) { |
| before := time.Now() |
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| StatusCode: 429, |
| ErrorBody: []byte(`{"code":"USAGE_LIMIT_EXCEEDED","message":"error: code=429 reason=\"DAILY_LIMIT_EXCEEDED\" message=\"daily usage limit exceeded\" metadata=map[]"}`), |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
| after := time.Now() |
|
|
| if action != ActionRetryKey { |
| t.Fatalf("expected ActionRetryKey, got %v", action) |
| } |
|
|
| cooldownUntil, exists := getKeyCooldownUntil(ctx, store, cfg.ID, 0) |
| if !exists { |
| t.Fatal("expected key cooldown") |
| } |
|
|
| if !sameTimeSecond(cooldownUntil, nextLocalMidnight(before)) && |
| !sameTimeSecond(cooldownUntil, nextLocalMidnight(after)) { |
| t.Fatalf("cooldownUntil=%s, want next local midnight from %s or %s", |
| cooldownUntil.Format(time.RFC3339), |
| before.Format(time.RFC3339), |
| after.Format(time.RFC3339)) |
| } |
| }) |
|
|
| t.Run("API_KEY_QUOTA_EXHAUSTED cools key for thirty minutes", func(t *testing.T) { |
| before := time.Now() |
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: 1, |
| StatusCode: 429, |
| ErrorBody: []byte(`{"code":"API_KEY_QUOTA_EXHAUSTED","message":"API key 额度已用完"}`), |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
|
|
| if action != ActionRetryKey { |
| t.Fatalf("expected ActionRetryKey, got %v", action) |
| } |
|
|
| cooldownUntil, exists := getKeyCooldownUntil(ctx, store, cfg.ID, 1) |
| if !exists { |
| t.Fatal("expected key cooldown") |
| } |
|
|
| duration := cooldownUntil.Sub(before) |
| if duration < 29*time.Minute+55*time.Second || duration > 30*time.Minute+5*time.Second { |
| t.Fatalf("cooldown duration=%v, want about 30m", duration) |
| } |
| }) |
| } |
|
|
| func TestHandleError_FreeTierBudgetExceededWrappedIn500CoolsKeyThirtyMinutes(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| cfg := createTestChannel(t, store, "test-free-tier-budget-exceeded") |
| _ = store.CreateAPIKeysBatch(ctx, []*model.APIKey{ |
| { |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| APIKey: "sk-free-tier-0", |
| KeyStrategy: model.KeyStrategySequential, |
| }, |
| { |
| ChannelID: cfg.ID, |
| KeyIndex: 1, |
| APIKey: "sk-free-tier-1", |
| KeyStrategy: model.KeyStrategySequential, |
| }, |
| }) |
|
|
| before := time.Now() |
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| StatusCode: 500, |
| ErrorBody: []byte(`{"type":"error","error":{"type":"api_error","message":"403 {\"error\":{\"code\":\"FREE_TIER_BUDGET_EXCEEDED\",\"message\":\"Free tier monthly spend limit exceeded. Please upgrade to a paid plan to continue using this service.\"}}"}}`), |
| }) |
|
|
| if action != ActionRetryKey { |
| t.Fatalf("expected ActionRetryKey, got %v", action) |
| } |
|
|
| cooldownUntil, exists := getKeyCooldownUntil(ctx, store, cfg.ID, 0) |
| if !exists { |
| t.Fatal("expected key cooldown") |
| } |
|
|
| duration := cooldownUntil.Sub(before) |
| if duration < 29*time.Minute+55*time.Second || duration > 30*time.Minute+5*time.Second { |
| t.Fatalf("cooldown duration=%v, want about 30m", duration) |
| } |
|
|
| channelCfg, err := store.GetConfig(ctx, cfg.ID) |
| if err != nil { |
| t.Fatalf("get config: %v", err) |
| } |
| if channelCfg.CooldownUntil > 0 && time.Unix(channelCfg.CooldownUntil, 0).After(time.Now()) { |
| t.Fatalf("channel should not be cooled for wrapped free tier quota error") |
| } |
| } |
|
|
| func TestHandleError_FreeTierBudgetExceededSSEErrorCoolsKeyThirtyMinutes(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| cfg := createTestChannel(t, store, "test-free-tier-budget-exceeded-sse") |
| _ = store.CreateAPIKeysBatch(ctx, []*model.APIKey{ |
| { |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| APIKey: "sk-free-tier-sse-0", |
| KeyStrategy: model.KeyStrategySequential, |
| }, |
| { |
| ChannelID: cfg.ID, |
| KeyIndex: 1, |
| APIKey: "sk-free-tier-sse-1", |
| KeyStrategy: model.KeyStrategySequential, |
| }, |
| }) |
|
|
| before := time.Now() |
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| StatusCode: util.StatusSSEError, |
| ErrorBody: []byte(`{"type":"error","error":{"type":"api_error","message":"403 {\"error\":{\"code\":\"FREE_TIER_BUDGET_EXCEEDED\",\"message\":\"Free tier monthly spend limit exceeded. Please upgrade to a paid plan to continue using this service.\"}}"}}`), |
| }) |
|
|
| if action != ActionRetryKey { |
| t.Fatalf("expected ActionRetryKey, got %v", action) |
| } |
|
|
| cooldownUntil, exists := getKeyCooldownUntil(ctx, store, cfg.ID, 0) |
| if !exists { |
| t.Fatal("expected key cooldown") |
| } |
|
|
| duration := cooldownUntil.Sub(before) |
| if duration < 29*time.Minute+55*time.Second || duration > 30*time.Minute+5*time.Second { |
| t.Fatalf("cooldown duration=%v, want about 30m", duration) |
| } |
|
|
| channelCfg, err := store.GetConfig(ctx, cfg.ID) |
| if err != nil { |
| t.Fatalf("get config: %v", err) |
| } |
| if channelCfg.CooldownUntil > 0 && time.Unix(channelCfg.CooldownUntil, 0).After(time.Now()) { |
| t.Fatalf("channel should not be cooled for SSE free tier quota error") |
| } |
| } |
|
|
| func TestHandleError_Structured429QuotaSingleKeyStaysKeyCooldown(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| cfg := createTestChannel(t, store, "test-single-key-structured-429-quota") |
| _ = store.CreateAPIKeysBatch(ctx, []*model.APIKey{{ |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| APIKey: "sk-single-quota", |
| KeyStrategy: model.KeyStrategySequential, |
| }}) |
|
|
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| StatusCode: 429, |
| ErrorBody: []byte(`{"code":"API_KEY_QUOTA_EXHAUSTED","message":"API key 额度已用完"}`), |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
|
|
| if action != ActionRetryKey { |
| t.Fatalf("expected ActionRetryKey, got %v", action) |
| } |
|
|
| cooldownUntil, exists := getKeyCooldownUntil(ctx, store, cfg.ID, 0) |
| if !exists || cooldownUntil.Before(time.Now()) { |
| t.Fatal("expected key cooldown") |
| } |
|
|
| channelCfg, err := store.GetConfig(ctx, cfg.ID) |
| if err != nil { |
| t.Fatalf("get config: %v", err) |
| } |
| if channelCfg.CooldownUntil > 0 && time.Unix(channelCfg.CooldownUntil, 0).After(time.Now()) { |
| t.Fatalf("channel should not be cooled for structured key quota error") |
| } |
| } |
|
|
| func TestHandleError_ChineseRelativeQuotaCooldown(t *testing.T) { |
| store, cleanup := setupTestStore(t) |
| defer cleanup() |
| manager := NewManager(store, nil) |
| ctx := context.Background() |
|
|
| cfg := createTestChannel(t, store, "test-chinese-relative-quota") |
| _ = store.CreateAPIKeysBatch(ctx, []*model.APIKey{ |
| { |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| APIKey: "sk-relative-quota-0", |
| KeyStrategy: model.KeyStrategySequential, |
| }, |
| { |
| ChannelID: cfg.ID, |
| KeyIndex: 1, |
| APIKey: "sk-relative-quota-1", |
| KeyStrategy: model.KeyStrategySequential, |
| }, |
| }) |
|
|
| before := time.Now() |
| action := manager.HandleError(ctx, ErrorInput{ |
| ChannelID: cfg.ID, |
| KeyIndex: 0, |
| StatusCode: 402, |
| ErrorBody: []byte(`{"error":"已达到用量上限,将在明天凌晨3点13分(北京时间)恢复"}`), |
| IsNetworkError: false, |
| Headers: nil, |
| }) |
| after := time.Now() |
|
|
| if action != ActionRetryKey { |
| t.Fatalf("expected ActionRetryKey, got %v", action) |
| } |
|
|
| cooldownUntil, exists := getKeyCooldownUntil(ctx, store, cfg.ID, 0) |
| if !exists { |
| t.Fatal("expected key cooldown") |
| } |
|
|
| beforeExpected := nextBeijingTime(before, 1, 3, 13) |
| afterExpected := nextBeijingTime(after, 1, 3, 13) |
| if !sameTimeSecond(cooldownUntil, beforeExpected) && |
| !sameTimeSecond(cooldownUntil, afterExpected) { |
| t.Fatalf("cooldownUntil=%s, want %s or %s", |
| cooldownUntil.Format(time.RFC3339), |
| beforeExpected.Format(time.RFC3339), |
| afterExpected.Format(time.RFC3339)) |
| } |
| } |
|
|
| |
|
|
| func nextLocalMidnight(now time.Time) time.Time { |
| y, m, d := now.In(time.Local).Date() |
| return time.Date(y, m, d+1, 0, 0, 0, 0, time.Local) |
| } |
|
|
| func nextBeijingTime(now time.Time, days int, hour int, minute int) time.Time { |
| loc := time.FixedZone("Asia/Shanghai", 8*60*60) |
| local := now.In(loc) |
| y, m, d := local.Date() |
| return time.Date(y, m, d+days, hour, minute, 0, 0, loc) |
| } |
|
|
| func sameTimeSecond(a, b time.Time) bool { |
| return a.Sub(b).Abs() <= 2*time.Second |
| } |
|
|
| |
| func getKeyCooldownUntil(ctx context.Context, store storage.Store, channelID int64, keyIndex int) (time.Time, bool) { |
| cooldowns, err := store.GetAllKeyCooldowns(ctx) |
| if err != nil { |
| return time.Time{}, false |
| } |
| channelCooldowns, ok := cooldowns[channelID] |
| if !ok { |
| return time.Time{}, false |
| } |
| until, ok := channelCooldowns[keyIndex] |
| return until, ok |
| } |
|
|
| func setupTestStore(t *testing.T) (storage.Store, func()) { |
| return testutil.SetupTestStore(t) |
| } |
|
|
| func createTestChannel(t *testing.T, store storage.Store, name string) *model.Config { |
| t.Helper() |
|
|
| cfg := &model.Config{ |
| Name: name, |
| URL: "https://api.example.com", |
| Priority: 10, |
| ModelEntries: []model.ModelEntry{ |
| {Model: "test-model", RedirectModel: ""}, |
| }, |
| Enabled: true, |
| } |
|
|
| created, err := store.CreateConfig(context.Background(), cfg) |
| if err != nil { |
| t.Fatalf("Failed to create test channel: %v", err) |
| } |
|
|
| return created |
| } |
|
|