backend/internal/application/gateway/failure_test.go

package gateway

import (
	"errors"
	"io"
	"net/http"
	"strings"
	"testing"

	accountdomain "github.com/chenyme/grok2api/backend/internal/domain/account"
	"github.com/chenyme/grok2api/backend/internal/infra/provider"
)

type responseHeaderTimeoutTestError struct{}

func (responseHeaderTimeoutTestError) Error() string {
	return "http2: timeout awaiting response headers"
}
func (responseHeaderTimeoutTestError) Timeout() bool   { return true }
func (responseHeaderTimeoutTestError) Temporary() bool { return true }

func TestTransportUpstreamFailureClassifiesResponseHeaderTimeout(t *testing.T) {
	failure := newTransportUpstreamFailure(responseHeaderTimeoutTestError{}, 42, "build")
	if failure.HTTPStatus != http.StatusGatewayTimeout || failure.Code != "upstream_header_timeout" || failure.PublicMessage != "等待上游响应头超时" || failure.AuditCode() != "upstream_header_timeout" {
		t.Fatalf("failure = %#v", failure)
	}
	if stage := transportStage(responseHeaderTimeoutTestError{}); stage != "response_header_timeout" {
		t.Fatalf("stage = %q", stage)
	}
	if isRetryableTransportFailure(accountdomain.ProviderBuild, responseHeaderTimeoutTestError{}) {
		t.Fatal("a Build response-header timeout must not switch accounts")
	}
	if !isRetryableTransportFailure(accountdomain.ProviderWeb, responseHeaderTimeoutTestError{}) {
		t.Fatal("the Build-specific retry veto must not change Web failover")
	}
	if !isRetryableTransportFailure(accountdomain.ProviderBuild, errors.New("connection reset by peer")) {
		t.Fatal("ordinary pre-response transport failures must retain failover behavior")
	}
}

func TestHTTPUpstreamFailureClassifiesBuildForbiddenBodies(t *testing.T) {
	tests := []struct {
		name                   string
		status                 int
		body                   string
		accountScoped          bool
		permanentAccountDenial bool
		safetyRejection        bool
		requestScopedForbidden bool
		quotaExhausted         bool
		freeQuotaExhausted     bool
		modelQuotaExhausted    bool
		accountBlocked         bool
		spendingLimitBlocked   bool
		upstreamCode           string
	}{
		{
			name: "blocked account", status: http.StatusForbidden, body: `{"code":"unauthorized:blocked-user","error":"User is blocked"}`,
			accountScoped: true, accountBlocked: true, upstreamCode: "unauthorized:blocked-user",
		},
		{
			name: "top-level permanent chat denial", status: http.StatusForbidden, body: `{"status_code":403,"code":"permission-denied","error":"Access to the chat endpoint is denied. Please update the permissions."}`,
			accountScoped: true, permanentAccountDenial: true, upstreamCode: "permission-denied",
		},
		{
			// Web chat nested error; numeric code alone must not decide AccountBlocked.
			name: "web nested blocked-user message", body: `{"error":{"code":7,"message":"User is blocked [WKE=unauthorized:blocked-user]","details":[]}}`,
			accountScoped: true, accountBlocked: true, upstreamCode: "", // numeric JSON code is not stringified by extractors
		},
		{
			name: "forbidden without block text", body: `{"error":{"code":7,"message":"Something went wrong","details":[]}}`,
		},
		{
			name: "top-level permanent chat denial", body: `{"status_code":403,"code":"permission-denied","error":"Access to the chat endpoint is denied. Please update the permissions."}`,
			accountScoped: true, permanentAccountDenial: true, upstreamCode: "permission-denied",
		},
		{
			name: "403 spending limit", status: http.StatusForbidden, body: `{"code":"personal-team-blocked:spending-limit","error":"quota exhausted"}`,
			accountScoped: true, quotaExhausted: true, spendingLimitBlocked: true, upstreamCode: "personal-team-blocked:spending-limit",
		},
		{
			name: "402 spending limit", status: http.StatusPaymentRequired, body: `{"code":"personal-team-blocked:spending-limit","error":"quota exhausted"}`,
			accountScoped: true, quotaExhausted: true, spendingLimitBlocked: true, upstreamCode: "personal-team-blocked:spending-limit",
		},
		{
			name: "unknown policy rejection", status: http.StatusForbidden, body: `{"error":"upstream policy rejected request"}`,
			requestScopedForbidden: true,
		},
		{
			name: "free model quota", status: http.StatusForbidden, body: `{"error":"You've used all the included free usage for model grok-build"}`,
			accountScoped: true, quotaExhausted: true, freeQuotaExhausted: true, modelQuotaExhausted: true,
		},
		{
			name: "safety usage guidelines", body: `{"code":"permission-denied","error":"Content violates usage guidelines. SAFETY_CHECK_TYPE_VIOLENCE"}`,
			safetyRejection: true, upstreamCode: "permission-denied",
		},
		{
			name: "explicit policy rejection", body: `{"code":"permission-denied","error":"request rejected by policy"}`,
			requestScopedForbidden: true,
			upstreamCode:           "permission-denied",
		},
		{
			name: "bare permission-denied", body: `{"code":"permission_denied","error":"denied"}`,
			upstreamCode: "permission_denied",
		},
		{
			name: "invalid argument code", body: `{"code":"invalid-argument","error":"unsupported request field"}`,
			requestScopedForbidden: true, upstreamCode: "invalid-argument",
		},
		{
			name: "request-level access denied sentence", body: `{"code":"operation-denied","error":"Access denied because this operation is unavailable under ZDR"}`,
			requestScopedForbidden: true, upstreamCode: "operation-denied",
		},
		{
			name: "legacy 403 credit exhaustion", body: `{"code":"permission-denied","error":"You have run out of credits"}`,
			accountScoped: true, quotaExhausted: true, upstreamCode: "permission-denied",
		},
		{
			name: "nested usage balance exhaustion", body: `{"error":{"type":"billing_error","message":"Grok Build usage balance exhausted"}}`,
			accountScoped: true, quotaExhausted: true,
		},
		{
			name: "exact access denied", body: `{"code":"operation-denied","error":"Access denied."}`,
			accountScoped: true, permanentAccountDenial: true, upstreamCode: "operation-denied",
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			status := test.status
			if status == 0 {
				status = http.StatusForbidden
			}
			failure := newHTTPUpstreamFailure(status, []byte(test.body), 42, "build")
			if failure.HTTPStatus != status || failure.AccountScoped != test.accountScoped || failure.AccountBlocked != test.accountBlocked || failure.PermanentAccountDenial != test.permanentAccountDenial || failure.SafetyRejection != test.safetyRejection || failure.RequestScopedForbidden != test.requestScopedForbidden || failure.QuotaExhausted != test.quotaExhausted || failure.FreeQuotaExhausted != test.freeQuotaExhausted || failure.ModelQuotaExhausted != test.modelQuotaExhausted || failure.SpendingLimitBlocked != test.spendingLimitBlocked || failure.UpstreamCode != test.upstreamCode {
				t.Fatalf("failure = %#v", failure)
			}
			if test.upstreamCode == "permission-denied" && (failure.ClientCredentialErrorCode() != "permission-denied" || failure.AuditCode() != "upstream_forbidden_permission_denied") {
				t.Fatalf("public=%q audit=%q", failure.ClientCredentialErrorCode(), failure.AuditCode())
			}
		})
	}
}

func TestNonAccountFailureFingerprintStopsAtLimit(t *testing.T) {
	fingerprints := map[string]int{}
	for _, status := range []int{
		http.StatusUnauthorized,
		http.StatusPaymentRequired,
		http.StatusForbidden,
		http.StatusTooManyRequests,
	} {
		accountScoped := &UpstreamFailure{HTTPStatus: status, AccountScoped: true, Fingerprint: http.StatusText(status)}
		for i := 0; i < nonAccountFailureFingerprintLimit+5; i++ {
			if shouldStopForNonAccountFingerprint(fingerprints, accountScoped) {
				t.Fatalf("account-scoped %d stopped credential traversal at iteration %d", status, i)
			}
		}
	}
	if len(fingerprints) != 0 {
		t.Fatalf("account-scoped failures should not count fingerprints: %#v", fingerprints)
	}

	// 未知 403、Team 限流不计指纹，应持续换号。
	unknown403 := &UpstreamFailure{HTTPStatus: http.StatusForbidden, Fingerprint: "403:unknown"}
	teamLimit := &UpstreamFailure{HTTPStatus: http.StatusTooManyRequests, Fingerprint: "429:team_model_rate_limit"}
	for i := 0; i < nonAccountFailureFingerprintLimit+5; i++ {
		if shouldStopForNonAccountFingerprint(fingerprints, unknown403) {
			t.Fatalf("unknown 403 must not stop at iteration %d", i)
		}
		if shouldStopForNonAccountFingerprint(fingerprints, teamLimit) {
			t.Fatalf("team rate limit must not stop at iteration %d", i)
		}
	}
	if len(fingerprints) != 0 {
		t.Fatalf("excluded failure types should not count fingerprints: %#v", fingerprints)
	}

	network := &UpstreamFailure{Fingerprint: "upstream_timeout"}
	for i := 1; i < nonAccountFailureFingerprintLimit; i++ {
		if shouldStopForNonAccountFingerprint(fingerprints, network) {
			t.Fatalf("stopped early at count %d", i)
		}
	}
	if !shouldStopForNonAccountFingerprint(fingerprints, network) {
		t.Fatalf("should stop after %d non-account failures", nonAccountFailureFingerprintLimit)
	}
	if fingerprints["upstream_timeout"] != nonAccountFailureFingerprintLimit {
		t.Fatalf("fingerprint count = %d, want %d", fingerprints["upstream_timeout"], nonAccountFailureFingerprintLimit)
	}
}

func TestHTTPUpstreamFailureLeavesPaymentRecoveryKindToBilling(t *testing.T) {
	failure := newHTTPUpstreamFailure(http.StatusPaymentRequired, []byte(`{
		"code":"personal-team-blocked:spending-limit",
		"error":"You have run out of credits"
	}`), 42, "build")
	if !failure.AccountScoped || !failure.QuotaExhausted || failure.FreeQuotaExhausted || failure.UpstreamCode != "personal-team-blocked:spending-limit" {
		t.Fatalf("failure = %#v", failure)
	}
}

func TestRetryableResponseHonorsUpstreamRetryVeto(t *testing.T) {
	response := &provider.Response{
		StatusCode: http.StatusInternalServerError,
		Header:     http.Header{"X-Should-Retry": {"false"}},
		Body:       io.NopCloser(strings.NewReader(`{"error":"invalid request history"}`)),
	}
	if isRetryableResponse(response, accountdomain.ProviderBuild) {
		t.Fatal("x-should-retry:false 必须禁止换账号重试")
	}
	response.Header.Set("X-Should-Retry", "true")
	if !isRetryableResponse(response, accountdomain.ProviderBuild) {
		t.Fatal("x-should-retry:true 不应覆盖现有状态码重试策略")
	}
	response.Header.Set("X-Should-Retry", "unknown")
	if !isRetryableResponse(response, accountdomain.ProviderBuild) {
		t.Fatal("未知 x-should-retry 值应按未提供处理")
	}
}

func TestPaymentRequiredAlwaysRetriesDespiteUpstreamVeto(t *testing.T) {
	response := &provider.Response{
		StatusCode: http.StatusPaymentRequired,
		Header:     http.Header{"X-Should-Retry": {"false"}},
		Body:       io.NopCloser(strings.NewReader(`{"code":"personal-team-blocked:spending-limit","error":"You have run out of credits"}`)),
	}
	if !isRetryableResponse(response, accountdomain.ProviderBuild) {
		t.Fatal("402 spending-limit must force account rotation even when X-Should-Retry is false")
	}
	if isRetryableResponse(response, accountdomain.ProviderWeb) {
		t.Fatal("non-Build 402 must continue honoring X-Should-Retry:false")
	}
}

func TestBuildForbiddenAlwaysEntersAccountFailureHandling(t *testing.T) {
	response := &provider.Response{
		StatusCode: http.StatusForbidden,
		Header:     http.Header{"X-Should-Retry": {"false"}},
		Body:       io.NopCloser(strings.NewReader(`{"code":"permission-denied"}`)),
	}
	if !isRetryableResponse(response, accountdomain.ProviderBuild) {
		t.Fatal("Build 403 must enter account failure handling even when X-Should-Retry is false")
	}
	if isRetryableResponse(response, accountdomain.ProviderWeb) {
		t.Fatal("non-Build 403 must continue honoring X-Should-Retry:false")
	}
}

func TestBuildForbiddenReauthPolicyMatchesExactErrorCodes(t *testing.T) {
	service := &Service{}
	service.UpdateBuildForbiddenReauthPolicy(true, []string{"permission-denied", "team-access-denied"})

	for _, code := range []string{"permission-denied", "permission_denied", "TEAM-ACCESS-DENIED"} {
		failure := &UpstreamFailure{HTTPStatus: http.StatusForbidden, UpstreamCode: code, AccountScoped: true}
		if !service.shouldInvalidateBuildForbidden(failure) {
			t.Fatalf("configured code %q did not match", code)
		}
	}
	for _, failure := range []*UpstreamFailure{
		{HTTPStatus: http.StatusForbidden, UpstreamCode: "permission-denied"},
		{HTTPStatus: http.StatusForbidden, UpstreamCode: "permission-denied", AccountScoped: true, RequestScopedForbidden: true},
		{HTTPStatus: http.StatusForbidden, UpstreamCode: "unconfigured-denial", AccountScoped: true},
		{HTTPStatus: http.StatusUnauthorized, UpstreamCode: "permission-denied", AccountScoped: true},
		{HTTPStatus: http.StatusInternalServerError, UpstreamCode: "permission-denied", AccountScoped: true},
	} {
		if service.shouldInvalidateBuildForbidden(failure) {
			t.Fatalf("unconfigured or ineligible failure matched: %#v", failure)
		}
	}

	service.UpdateBuildForbiddenReauthPolicy(false, []string{"permission-denied"})
	if service.shouldInvalidateBuildForbidden(&UpstreamFailure{HTTPStatus: http.StatusForbidden, UpstreamCode: "permission-denied", AccountScoped: true}) {
		t.Fatal("disabled policy matched an error code")
	}
}

func TestHTTPUpstreamFailureClassifiesSubscriptionFreeUsageAsAccountQuota(t *testing.T) {
	body := `{"code":"subscription:free-usage-exhausted","error":"tokens (actual/limit): 10/10"}`
	failure := newHTTPUpstreamFailure(http.StatusTooManyRequests, []byte(body), 7, "build")
	if !failure.AccountScoped || !failure.FreeQuotaExhausted || failure.ModelQuotaExhausted || !failure.QuotaExhausted {
		t.Fatalf("failure = %#v", failure)
	}
}

func TestHTTPUpstreamFailureKeepsExplicitModelFreeUsageScoped(t *testing.T) {
	body := `{"error":"You've used all the included free usage for model grok-build"}`
	failure := newHTTPUpstreamFailure(http.StatusTooManyRequests, []byte(body), 7, "build")
	if !failure.AccountScoped || !failure.FreeQuotaExhausted || !failure.ModelQuotaExhausted || !failure.QuotaExhausted {
		t.Fatalf("failure = %#v", failure)
	}
}

func TestBuildRateLimitForcesAccountFailoverDespiteRetryVeto(t *testing.T) {
	response := &provider.Response{
		StatusCode: http.StatusTooManyRequests,
		Header:     http.Header{"X-Should-Retry": {"false"}},
		Body:       io.NopCloser(strings.NewReader(`{"code":"subscription:free-usage-exhausted"}`)),
	}
	if !isRetryableResponse(response, accountdomain.ProviderBuild) {
		t.Fatal("Build free-usage 429 must force account rotation even when X-Should-Retry is false")
	}
	if isRetryableResponse(response, accountdomain.ProviderWeb) {
		t.Fatal("non-Build 429 must continue honoring X-Should-Retry:false")
	}
}

func TestBuildForbiddenReauthPolicyIgnoresSafetyRejection(t *testing.T) {
	service := &Service{}
	service.UpdateBuildForbiddenReauthPolicy(true, []string{"permission-denied"})
	failure := &UpstreamFailure{HTTPStatus: http.StatusForbidden, UpstreamCode: "permission-denied", AccountScoped: true, SafetyRejection: true}
	if service.shouldInvalidateBuildForbidden(failure) {
		t.Fatal("safety rejection must not match permission-denied invalidation policy")
	}
}

func TestTerminalRequestForbiddenRequiresExplicitRequestSignal(t *testing.T) {
	bare := &UpstreamFailure{HTTPStatus: http.StatusForbidden, UpstreamCode: "permission-denied"}
	if isTerminalRequestForbidden(accountdomain.ProviderBuild, bare) {
		t.Fatal("bare permission-denied must remain on credential traversal")
	}

	requestScoped := &UpstreamFailure{HTTPStatus: http.StatusForbidden, UpstreamCode: "invalid-argument", RequestScopedForbidden: true}
	if !isTerminalRequestForbidden(accountdomain.ProviderBuild, requestScoped) {
		t.Fatal("explicit Build request rejection must be terminal")
	}
	for _, providerValue := range []accountdomain.Provider{accountdomain.ProviderWeb, accountdomain.ProviderConsole} {
		if isTerminalRequestForbidden(providerValue, requestScoped) {
			t.Fatalf("%s request classification must retain existing egress recovery", providerValue)
		}
	}

	safety := &UpstreamFailure{HTTPStatus: http.StatusForbidden, UpstreamCode: "permission-denied", SafetyRejection: true}
	for _, providerValue := range []accountdomain.Provider{accountdomain.ProviderBuild, accountdomain.ProviderWeb, accountdomain.ProviderConsole} {
		if !isTerminalRequestForbidden(providerValue, safety) {
			t.Fatalf("%s safety rejection must remain terminal", providerValue)
		}
	}
}