| package util |
|
|
| import ( |
| "context" |
| "encoding/json" |
| "errors" |
| "net" |
| "net/http" |
| "regexp" |
| "strconv" |
| "strings" |
| "time" |
|
|
| "ccLoad/internal/protocol" |
| ) |
|
|
| |
| |
|
|
| |
| var ErrUpstreamFirstByteTimeout = errors.New("upstream first byte timeout") |
|
|
| |
| var ErrUpstreamEmptyResponse = errors.New("upstream returned empty response") |
|
|
| |
| |
| var resetTime1308Regex = regexp.MustCompile(`\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}`) |
|
|
| |
| var beijingTomorrowResetRegex = regexp.MustCompile(`明天\s*(?:凌晨|早上|上午)?\s*(\d{1,2})\s*点\s*(?:(\d{1,2})\s*分)?`) |
|
|
| |
| const ( |
| |
| |
| StatusClientClosedRequest = 499 |
|
|
| |
| |
| StatusQuotaExceeded = 596 |
|
|
| |
| |
| StatusSSEError = 597 |
|
|
| |
| StatusFirstByteTimeout = 598 |
|
|
| |
| |
| StatusStreamIncomplete = 599 |
| ) |
|
|
| |
| const ( |
| |
| RetryAfterThresholdSeconds = 60 |
| |
| RateLimitScopeGlobal = "global" |
| RateLimitScopeIP = "ip" |
| RateLimitScopeAccount = "account" |
| ) |
|
|
| |
| type ErrorLevel int |
|
|
| const ( |
| |
| ErrorLevelNone ErrorLevel = iota |
| |
| ErrorLevelKey |
| |
| ErrorLevelChannel |
| |
| ErrorLevelClient |
| ) |
|
|
| |
| |
| |
| |
| type StatusCodeMeta struct { |
| Level ErrorLevel |
| } |
|
|
| |
| type HTTPResponseClassification struct { |
| Level ErrorLevel |
| KeyCooldownUntil time.Time |
| HasKeyCooldownUntil bool |
| KeyCooldownReason string |
| } |
|
|
| |
| |
| type sseErrorResponse struct { |
| Type string `json:"type"` |
| Error struct { |
| Type string `json:"type"` |
| Code string `json:"code"` |
| Message string `json:"message"` |
| } `json:"error"` |
| } |
|
|
| type structuredQuotaErrorResponse struct { |
| Code string `json:"code"` |
| Message string `json:"message"` |
| Error json.RawMessage `json:"error"` |
| } |
|
|
| type structuredQuotaErrorObject struct { |
| Type string `json:"type"` |
| Code string `json:"code"` |
| Message string `json:"message"` |
| } |
|
|
| |
| |
| func (r *sseErrorResponse) ErrorType() string { |
| if r.Error.Type != "" { |
| return r.Error.Type |
| } |
| return r.Error.Code |
| } |
|
|
| |
| |
| var statusCodeMetaMap = map[int]StatusCodeMeta{ |
| |
| |
| |
| 499: {ErrorLevelChannel}, |
|
|
| |
| |
| 401: {ErrorLevelKey}, |
| 402: {ErrorLevelKey}, |
| 403: {ErrorLevelKey}, |
| 429: {ErrorLevelKey}, |
|
|
| |
| 444: {ErrorLevelChannel}, |
| 500: {ErrorLevelChannel}, |
| 502: {ErrorLevelChannel}, |
| 503: {ErrorLevelChannel}, |
| 504: {ErrorLevelChannel}, |
| 520: {ErrorLevelChannel}, |
| 521: {ErrorLevelChannel}, |
| 524: {ErrorLevelChannel}, |
|
|
| |
| StatusQuotaExceeded: {ErrorLevelKey}, |
| StatusSSEError: {ErrorLevelKey}, |
| StatusFirstByteTimeout: {ErrorLevelChannel}, |
| StatusStreamIncomplete: {ErrorLevelChannel}, |
|
|
| |
| |
| 408: {ErrorLevelClient}, |
| |
| |
| 405: {ErrorLevelChannel}, |
| 406: {ErrorLevelClient}, |
| 410: {ErrorLevelClient}, |
| 413: {ErrorLevelClient}, |
| 414: {ErrorLevelClient}, |
| 415: {ErrorLevelClient}, |
| 416: {ErrorLevelClient}, |
| 417: {ErrorLevelClient}, |
| } |
|
|
| |
| func GetStatusCodeMeta(status int) StatusCodeMeta { |
| if meta, ok := statusCodeMetaMap[status]; ok { |
| return meta |
| } |
| |
| if status >= 500 { |
| return StatusCodeMeta{ErrorLevelChannel} |
| } |
| if status >= 400 { |
| |
| |
| |
| return StatusCodeMeta{ErrorLevelKey} |
| } |
| return StatusCodeMeta{ErrorLevelClient} |
| } |
|
|
| |
| |
| |
| |
| |
| func ClientStatusFor(status int) int { |
| if status <= 0 { |
| return http.StatusBadGateway |
| } |
|
|
| |
| switch status { |
| case StatusQuotaExceeded: |
| return http.StatusTooManyRequests |
| case StatusSSEError: |
| return http.StatusBadGateway |
| case StatusFirstByteTimeout: |
| return http.StatusGatewayTimeout |
| case StatusStreamIncomplete: |
| return http.StatusBadGateway |
| } |
|
|
| |
| return status |
| } |
|
|
| |
| |
| func ClassifyHTTPStatus(statusCode int) ErrorLevel { |
| if statusCode >= 200 && statusCode < 300 { |
| return ErrorLevelNone |
| } |
| return GetStatusCodeMeta(statusCode).Level |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| func ClassifyHTTPResponseWithMeta(statusCode int, headers map[string][]string, responseBody []byte) HTTPResponseClassification { |
| |
| |
| if resetTime, has1308 := ParseResetTimeFrom1308Error(responseBody); has1308 { |
| return HTTPResponseClassification{ |
| Level: ErrorLevelKey, |
| KeyCooldownUntil: resetTime, |
| HasKeyCooldownUntil: true, |
| KeyCooldownReason: "1308", |
| } |
| } |
|
|
| if cooldownUntil, reason, ok := parseStructuredQuotaCooldown(responseBody, time.Now()); ok { |
| return HTTPResponseClassification{ |
| Level: ErrorLevelKey, |
| KeyCooldownUntil: cooldownUntil, |
| HasKeyCooldownUntil: true, |
| KeyCooldownReason: reason, |
| } |
| } |
|
|
| |
| |
| |
| if statusCode == StatusSSEError { |
| return HTTPResponseClassification{Level: classifySSEError(responseBody)} |
| } |
|
|
| |
| if statusCode == 429 { |
| if headers != nil { |
| return HTTPResponseClassification{Level: classifyRateLimitError(headers, responseBody)} |
| } |
| return HTTPResponseClassification{Level: ErrorLevelKey} |
| } |
|
|
| |
| if statusCode == 400 { |
| return HTTPResponseClassification{Level: classify400Error(responseBody)} |
| } |
|
|
| |
| if statusCode == 404 { |
| return HTTPResponseClassification{Level: classify404Error(responseBody)} |
| } |
|
|
| |
| if statusCode != 401 && statusCode != 403 { |
| return HTTPResponseClassification{Level: ClassifyHTTPStatus(statusCode)} |
| } |
|
|
| |
| if len(responseBody) == 0 { |
| return HTTPResponseClassification{Level: ErrorLevelKey} |
| } |
|
|
| bodyLower := strings.ToLower(string(responseBody)) |
|
|
| |
| |
| channelErrorPatterns := []string{ |
| |
| "account suspended", |
| "account disabled", |
| "account banned", |
| "service disabled", |
|
|
| |
| |
| |
| |
| } |
|
|
| for _, pattern := range channelErrorPatterns { |
| if strings.Contains(bodyLower, pattern) { |
| return HTTPResponseClassification{Level: ErrorLevelChannel} |
| } |
| } |
|
|
| |
| |
| |
| return HTTPResponseClassification{Level: ErrorLevelKey} |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| func classifyRateLimitError(headers map[string][]string, responseBody []byte) ErrorLevel { |
| |
| if retryAfterValues, ok := headers["Retry-After"]; ok && len(retryAfterValues) > 0 { |
| retryAfter := retryAfterValues[0] |
|
|
| |
| |
| if seconds, err := strconv.Atoi(retryAfter); err == nil { |
| |
| |
| if seconds > RetryAfterThresholdSeconds { |
| return ErrorLevelChannel |
| } |
| } |
| |
| if _, err := time.Parse(time.RFC1123, retryAfter); err == nil { |
| return ErrorLevelChannel |
| } |
| } |
|
|
| |
| if scopeValues, ok := headers["X-Ratelimit-Scope"]; ok && len(scopeValues) > 0 { |
| scope := strings.ToLower(scopeValues[0]) |
| |
| if scope == RateLimitScopeGlobal || scope == RateLimitScopeIP || scope == RateLimitScopeAccount { |
| return ErrorLevelChannel |
| } |
| } |
|
|
| |
| if len(responseBody) > 0 { |
| bodyLower := strings.ToLower(string(responseBody)) |
|
|
| |
| channelPatterns := []string{ |
| "ip rate limit", |
| "account rate limit", |
| "global rate limit", |
| "organization limit", |
| } |
|
|
| for _, pattern := range channelPatterns { |
| if strings.Contains(bodyLower, pattern) { |
| return ErrorLevelChannel |
| } |
| } |
| } |
|
|
| |
| |
| return ErrorLevelKey |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| func classifySSEError(responseBody []byte) ErrorLevel { |
| if len(responseBody) == 0 { |
| return ErrorLevelKey |
| } |
|
|
| |
| |
| |
| |
| var errResp sseErrorResponse |
|
|
| if err := json.Unmarshal(responseBody, &errResp); err != nil { |
| return ErrorLevelKey |
| } |
|
|
| |
| switch errResp.ErrorType() { |
| case "api_error", "overloaded_error", "service_unavailable_error", "server_is_overloaded", "1305": |
| |
| return ErrorLevelChannel |
| case "rate_limit_error", "authentication_error", "invalid_request_error", "1308", "1310": |
| |
| return ErrorLevelKey |
| default: |
| |
| return ErrorLevelKey |
| } |
| } |
|
|
| func parseStructuredQuotaCooldown(responseBody []byte, now time.Time) (time.Time, string, bool) { |
| if len(responseBody) == 0 { |
| return time.Time{}, "", false |
| } |
|
|
| code, message, ok := parseStructuredQuotaError(responseBody) |
| if !ok { |
| return time.Time{}, "", false |
| } |
|
|
| messageUpper := strings.ToUpper(message) |
|
|
| switch { |
| case code == "API_KEY_QUOTA_EXHAUSTED": |
| return now.Add(30 * time.Minute), "API_KEY_QUOTA_EXHAUSTED", true |
| case code == "FREE_TIER_BUDGET_EXCEEDED" || strings.Contains(messageUpper, "FREE_TIER_BUDGET_EXCEEDED"): |
| return now.Add(30 * time.Minute), "FREE_TIER_BUDGET_EXCEEDED", true |
| case code == "DAILY_LIMIT_EXCEEDED" || |
| (code == "USAGE_LIMIT_EXCEEDED" && strings.Contains(messageUpper, "DAILY_LIMIT_EXCEEDED")): |
| return nextLocalMidnight(now), "DAILY_LIMIT_EXCEEDED", true |
| case strings.Contains(message, "用量上限"): |
| if until, ok := parseBeijingTomorrowResetTime(message, now); ok { |
| return until, "BEIJING_RELATIVE_QUOTA_RESET", true |
| } |
| return time.Time{}, "", false |
| default: |
| return time.Time{}, "", false |
| } |
| } |
|
|
| func parseStructuredQuotaError(responseBody []byte) (string, string, bool) { |
| var errResp structuredQuotaErrorResponse |
| if err := json.Unmarshal(responseBody, &errResp); err != nil { |
| return "", "", false |
| } |
|
|
| code := strings.ToUpper(strings.TrimSpace(errResp.Code)) |
| message := errResp.Message |
|
|
| if len(errResp.Error) > 0 { |
| var errorText string |
| if err := json.Unmarshal(errResp.Error, &errorText); err == nil { |
| if message == "" { |
| message = errorText |
| } |
| } else { |
| var errorObj structuredQuotaErrorObject |
| if err := json.Unmarshal(errResp.Error, &errorObj); err == nil { |
| if code == "" { |
| code = strings.ToUpper(strings.TrimSpace(errorObj.Code)) |
| } |
| if code == "" { |
| code = strings.ToUpper(strings.TrimSpace(errorObj.Type)) |
| } |
| if message == "" { |
| message = errorObj.Message |
| } |
| } |
| } |
| } |
|
|
| return code, message, code != "" || message != "" |
| } |
|
|
| func nextLocalMidnight(now time.Time) time.Time { |
| local := now.In(time.Local) |
| y, m, d := local.Date() |
| return time.Date(y, m, d+1, 0, 0, 0, 0, time.Local) |
| } |
|
|
| func parseBeijingTomorrowResetTime(message string, now time.Time) (time.Time, bool) { |
| if !strings.Contains(message, "北京时间") { |
| return time.Time{}, false |
| } |
|
|
| matches := beijingTomorrowResetRegex.FindStringSubmatch(message) |
| if matches == nil { |
| return time.Time{}, false |
| } |
|
|
| hour, err := strconv.Atoi(matches[1]) |
| if err != nil || hour < 0 || hour > 23 { |
| return time.Time{}, false |
| } |
|
|
| minute := 0 |
| if len(matches) > 2 && matches[2] != "" { |
| minute, err = strconv.Atoi(matches[2]) |
| if err != nil || minute < 0 || minute > 59 { |
| return time.Time{}, false |
| } |
| } |
|
|
| loc := time.FixedZone("Asia/Shanghai", 8*60*60) |
| local := now.In(loc) |
| y, mon, d := local.Date() |
| return time.Date(y, mon, d+1, hour, minute, 0, 0, loc), true |
| } |
|
|
| |
| |
| func classify400Error(responseBody []byte) ErrorLevel { |
| if len(responseBody) == 0 { |
| return ErrorLevelChannel |
| } |
| bodyLower := strings.ToLower(string(responseBody)) |
|
|
| |
| if strings.Contains(bodyLower, "invalid_api_key") || |
| strings.Contains(bodyLower, "api key") { |
| return ErrorLevelKey |
| } |
|
|
| |
| return ErrorLevelChannel |
| } |
|
|
| |
| |
| |
| |
| func classify404Error(responseBody []byte) ErrorLevel { |
| if len(responseBody) == 0 { |
| return ErrorLevelChannel |
| } |
| bodyLower := strings.ToLower(string(responseBody)) |
|
|
| |
| if strings.Contains(bodyLower, "model_not_found") || |
| strings.Contains(bodyLower, "does not exist") { |
| return ErrorLevelClient |
| } |
|
|
| |
| |
| return ErrorLevelChannel |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| func ParseResetTimeFrom1308Error(responseBody []byte) (time.Time, bool) { |
| |
| |
| |
| |
| var errResp sseErrorResponse |
|
|
| if err := json.Unmarshal(responseBody, &errResp); err != nil { |
| return time.Time{}, false |
| } |
|
|
| |
| errorType := errResp.ErrorType() |
| if errorType != "1308" && errorType != "1310" { |
| return time.Time{}, false |
| } |
|
|
| |
| |
| timeStr := resetTime1308Regex.FindString(errResp.Error.Message) |
| if timeStr == "" { |
| return time.Time{}, false |
| } |
|
|
| |
| resetTime, err := time.ParseInLocation("2006-01-02 15:04:05", timeStr, time.Local) |
| if err != nil { |
| return time.Time{}, false |
| } |
|
|
| return resetTime, true |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| func ClassifyError(err error) (statusCode int, errorLevel ErrorLevel, shouldRetry bool) { |
| if err == nil { |
| return 200, ErrorLevelNone, false |
| } |
|
|
| |
| if errors.Is(err, ErrUpstreamFirstByteTimeout) { |
| return StatusFirstByteTimeout, ErrorLevelChannel, true |
| } |
|
|
| |
| if errors.Is(err, ErrUpstreamEmptyResponse) { |
| return http.StatusBadGateway, ErrorLevelChannel, true |
| } |
|
|
| |
| if errors.Is(err, protocol.ErrUnsupportedRequestShape) { |
| return http.StatusBadRequest, ErrorLevelClient, false |
| } |
|
|
| |
| if errors.Is(err, context.Canceled) { |
| return 499, ErrorLevelClient, false |
| } |
|
|
| |
| if errors.Is(err, context.DeadlineExceeded) { |
| return 504, ErrorLevelChannel, true |
| } |
|
|
| |
| var netErr net.Error |
| if errors.As(err, &netErr) { |
| if netErr.Timeout() { |
| return 504, ErrorLevelChannel, true |
| } |
| } |
|
|
| |
| return classifyErrorByString(err.Error()) |
| } |
|
|
| |
| |
| func classifyErrorByString(errStr string) (int, ErrorLevel, bool) { |
| errLower := strings.ToLower(errStr) |
|
|
| |
| if strings.Contains(errLower, "broken pipe") { |
| return 499, ErrorLevelClient, false |
| } |
|
|
| |
| |
| if strings.Contains(errLower, "connection reset by peer") { |
| return 502, ErrorLevelChannel, true |
| } |
|
|
| |
| |
| if strings.Contains(errLower, "empty response") && |
| strings.Contains(errLower, "content-length: 0") { |
| return 502, ErrorLevelChannel, true |
| } |
|
|
| |
| if strings.Contains(errLower, "connection refused") { |
| return 502, ErrorLevelChannel, true |
| } |
|
|
| |
| |
| |
| if strings.Contains(errLower, "http2: response body closed") || |
| strings.Contains(errLower, "stream error:") { |
| return 502, ErrorLevelChannel, true |
| } |
|
|
| |
| if strings.Contains(errLower, "no such host") || |
| strings.Contains(errLower, "host unreachable") || |
| strings.Contains(errLower, "network unreachable") || |
| strings.Contains(errLower, "connection timeout") || |
| strings.Contains(errLower, "no route to host") { |
| return 502, ErrorLevelChannel, true |
| } |
|
|
| |
| |
| |
| return 502, ErrorLevelChannel, true |
| } |
|
|