luckfun233
feat: Add jitter and random offset to token refresh intervals for better distribution
04c88a2 | package auth | |
| import ( | |
| "context" | |
| "crypto/sha256" | |
| "encoding/hex" | |
| "errors" | |
| "math/rand" | |
| "net/http" | |
| "strings" | |
| "sync" | |
| "time" | |
| "ds2api/internal/account" | |
| "ds2api/internal/config" | |
| ) | |
| type ctxKey string | |
| const authCtxKey ctxKey = "auth_context" | |
| var ( | |
| ErrUnauthorized = errors.New("unauthorized: missing auth token") | |
| ErrNoAccount = errors.New("no accounts configured or all accounts are busy") | |
| ) | |
| type RequestAuth struct { | |
| UseConfigToken bool | |
| DeepSeekToken string | |
| CallerID string | |
| AccountID string | |
| TargetAccount string | |
| Account config.Account | |
| TriedAccounts map[string]bool | |
| resolver *Resolver | |
| } | |
| type LoginFunc func(ctx context.Context, acc config.Account) (string, error) | |
| type Resolver struct { | |
| Store *config.Store | |
| Pool *account.Pool | |
| Login LoginFunc | |
| mu sync.Mutex | |
| tokenRefreshedAt map[string]time.Time | |
| } | |
| func NewResolver(store *config.Store, pool *account.Pool, login LoginFunc) *Resolver { | |
| return &Resolver{ | |
| Store: store, | |
| Pool: pool, | |
| Login: login, | |
| tokenRefreshedAt: map[string]time.Time{}, | |
| } | |
| } | |
| func (r *Resolver) Determine(req *http.Request) (*RequestAuth, error) { | |
| callerKey := extractCallerToken(req) | |
| if callerKey == "" { | |
| return nil, ErrUnauthorized | |
| } | |
| callerID := callerTokenID(callerKey) | |
| ctx := req.Context() | |
| if !r.Store.HasAPIKey(callerKey) { | |
| return &RequestAuth{ | |
| UseConfigToken: false, | |
| DeepSeekToken: callerKey, | |
| CallerID: callerID, | |
| resolver: r, | |
| TriedAccounts: map[string]bool{}, | |
| }, nil | |
| } | |
| target := strings.TrimSpace(req.Header.Get("X-Ds2-Target-Account")) | |
| a, err := r.acquireManagedRequestAuth(ctx, callerID, target) | |
| if err != nil { | |
| return nil, err | |
| } | |
| return a, nil | |
| } | |
| func (r *Resolver) acquireManagedRequestAuth(ctx context.Context, callerID, target string) (*RequestAuth, error) { | |
| tried := map[string]bool{} | |
| var lastEnsureErr error | |
| for { | |
| if target == "" && len(tried) >= len(r.Store.Accounts()) { | |
| if lastEnsureErr != nil { | |
| return nil, lastEnsureErr | |
| } | |
| return nil, ErrNoAccount | |
| } | |
| acc, ok := r.Pool.AcquireWait(ctx, target, tried) | |
| if !ok { | |
| if lastEnsureErr != nil { | |
| return nil, lastEnsureErr | |
| } | |
| return nil, ErrNoAccount | |
| } | |
| a := &RequestAuth{ | |
| UseConfigToken: true, | |
| CallerID: callerID, | |
| AccountID: acc.Identifier(), | |
| TargetAccount: target, | |
| Account: acc, | |
| TriedAccounts: tried, | |
| resolver: r, | |
| } | |
| if err := r.ensureManagedToken(ctx, a); err != nil { | |
| lastEnsureErr = err | |
| tried[a.AccountID] = true | |
| r.Pool.Release(a.AccountID) | |
| if target != "" { | |
| return nil, err | |
| } | |
| continue | |
| } | |
| return a, nil | |
| } | |
| } | |
| // DetermineCaller resolves caller identity without acquiring any pooled account. | |
| // Use this for local-cache lookup routes that only need tenant isolation. | |
| func (r *Resolver) DetermineCaller(req *http.Request) (*RequestAuth, error) { | |
| callerKey := extractCallerToken(req) | |
| if callerKey == "" { | |
| return nil, ErrUnauthorized | |
| } | |
| callerID := callerTokenID(callerKey) | |
| a := &RequestAuth{ | |
| UseConfigToken: false, | |
| CallerID: callerID, | |
| resolver: r, | |
| TriedAccounts: map[string]bool{}, | |
| } | |
| if r == nil || r.Store == nil || !r.Store.HasAPIKey(callerKey) { | |
| a.DeepSeekToken = callerKey | |
| } | |
| return a, nil | |
| } | |
| func WithAuth(ctx context.Context, a *RequestAuth) context.Context { | |
| return context.WithValue(ctx, authCtxKey, a) | |
| } | |
| func FromContext(ctx context.Context) (*RequestAuth, bool) { | |
| v := ctx.Value(authCtxKey) | |
| a, ok := v.(*RequestAuth) | |
| return a, ok | |
| } | |
| func (r *Resolver) loginAndPersist(ctx context.Context, a *RequestAuth) error { | |
| token, err := r.Login(ctx, a.Account) | |
| if err != nil { | |
| // Detect USER_IS_BANNED: auto-ban the account and promote standby accounts. | |
| if strings.Contains(strings.ToUpper(err.Error()), "USER_IS_BANNED") { | |
| r.handleBannedAccount(a.AccountID) | |
| } | |
| return err | |
| } | |
| a.Account.Token = token | |
| a.DeepSeekToken = token | |
| r.markTokenRefreshedNow(a.AccountID) | |
| return r.Store.UpdateAccountToken(a.AccountID, token) | |
| } | |
| // handleBannedAccount marks an account as banned and promotes standby accounts | |
| // based on the segment-based rule: if active normal accounts fall below half of | |
| // total normal accounts (per segment), promote standby accounts. | |
| func (r *Resolver) handleBannedAccount(accountID string) { | |
| if r.Store == nil || r.Pool == nil { | |
| return | |
| } | |
| acc, err := r.Store.SetAccountBanned(accountID, true) | |
| if err != nil { | |
| config.Logger.Error("[banned] failed to mark account as banned", "account", accountID, "error", err) | |
| return | |
| } | |
| // Only auto-promote if the banned account was a normal (non-standby) account. | |
| if acc.Role == "standby" { | |
| return | |
| } | |
| config.Logger.Warn("[banned] account banned, checking standby promotion", "account", accountID) | |
| activeNormal := r.Store.ActiveNormalAccounts() | |
| totalNormal := r.Store.TotalNormalAccounts() | |
| // Segment rule: promote when active normal accounts < half of total normal accounts. | |
| // e.g. 2 normal -> both must be banned; 4 normal -> promote when only 1 active. | |
| threshold := (totalNormal + 1) / 2 // ceiling of totalNormal/2 | |
| if activeNormal < threshold { | |
| releaseCount := r.Store.RuntimeBackupReleaseCount() | |
| promoted := r.Store.PromoteStandbyAccounts(releaseCount) | |
| if len(promoted) > 0 { | |
| config.Logger.Info("[banned] promoted standby accounts", "count", len(promoted), "accounts", promoted) | |
| r.Pool.Reset() | |
| } | |
| } | |
| } | |
| func (r *Resolver) RefreshToken(ctx context.Context, a *RequestAuth) bool { | |
| if !a.UseConfigToken || a.AccountID == "" { | |
| return false | |
| } | |
| _ = r.Store.UpdateAccountToken(a.AccountID, "") | |
| a.Account.Token = "" | |
| if err := r.loginAndPersist(ctx, a); err != nil { | |
| config.Logger.Error("[refresh_token] failed", "account", a.AccountID, "error", err) | |
| return false | |
| } | |
| return true | |
| } | |
| func (r *Resolver) MarkTokenInvalid(a *RequestAuth) { | |
| if !a.UseConfigToken || a.AccountID == "" { | |
| return | |
| } | |
| a.Account.Token = "" | |
| a.DeepSeekToken = "" | |
| r.clearTokenRefreshMark(a.AccountID) | |
| _ = r.Store.UpdateAccountToken(a.AccountID, "") | |
| } | |
| func (r *Resolver) SwitchAccount(ctx context.Context, a *RequestAuth) bool { | |
| if !a.UseConfigToken { | |
| return false | |
| } | |
| if strings.TrimSpace(a.TargetAccount) != "" { | |
| return false | |
| } | |
| if a.TriedAccounts == nil { | |
| a.TriedAccounts = map[string]bool{} | |
| } | |
| if a.AccountID != "" { | |
| a.TriedAccounts[a.AccountID] = true | |
| r.Pool.Release(a.AccountID) | |
| } | |
| for { | |
| acc, ok := r.Pool.Acquire("", a.TriedAccounts) | |
| if !ok { | |
| return false | |
| } | |
| a.Account = acc | |
| a.AccountID = acc.Identifier() | |
| if err := r.ensureManagedToken(ctx, a); err != nil { | |
| a.TriedAccounts[a.AccountID] = true | |
| r.Pool.Release(a.AccountID) | |
| continue | |
| } | |
| return true | |
| } | |
| } | |
| func (a *RequestAuth) SwitchAccount(ctx context.Context) bool { | |
| if a == nil || a.resolver == nil { | |
| return false | |
| } | |
| return a.resolver.SwitchAccount(ctx, a) | |
| } | |
| func (r *Resolver) Release(a *RequestAuth) { | |
| if a == nil || !a.UseConfigToken || a.AccountID == "" { | |
| return | |
| } | |
| r.Pool.Release(a.AccountID) | |
| } | |
| func extractCallerToken(req *http.Request) string { | |
| authHeader := strings.TrimSpace(req.Header.Get("Authorization")) | |
| if strings.HasPrefix(strings.ToLower(authHeader), "bearer ") { | |
| token := strings.TrimSpace(authHeader[7:]) | |
| if token != "" { | |
| return token | |
| } | |
| } | |
| if key := strings.TrimSpace(req.Header.Get("x-api-key")); key != "" { | |
| return key | |
| } | |
| // Gemini/Google clients commonly send API key via x-goog-api-key. | |
| if key := strings.TrimSpace(req.Header.Get("x-goog-api-key")); key != "" { | |
| return key | |
| } | |
| // Gemini AI Studio compatibility: allow query key fallback only when no | |
| // header-based credential is present. | |
| if key := strings.TrimSpace(req.URL.Query().Get("key")); key != "" { | |
| return key | |
| } | |
| return strings.TrimSpace(req.URL.Query().Get("api_key")) | |
| } | |
| func callerTokenID(token string) string { | |
| token = strings.TrimSpace(token) | |
| if token == "" { | |
| return "" | |
| } | |
| sum := sha256.Sum256([]byte(token)) | |
| return "caller:" + hex.EncodeToString(sum[:8]) | |
| } | |
| func (r *Resolver) ensureManagedToken(ctx context.Context, a *RequestAuth) error { | |
| if strings.TrimSpace(a.Account.Token) == "" { | |
| return r.loginAndPersist(ctx, a) | |
| } | |
| if r.shouldForceRefresh(a.AccountID) { | |
| if err := r.loginAndPersist(ctx, a); err != nil { | |
| return err | |
| } | |
| return nil | |
| } | |
| a.DeepSeekToken = a.Account.Token | |
| return nil | |
| } | |
| func (r *Resolver) shouldForceRefresh(accountID string) bool { | |
| if r == nil || r.Store == nil { | |
| return false | |
| } | |
| if strings.TrimSpace(accountID) == "" { | |
| return false | |
| } | |
| intervalHours := r.Store.RuntimeTokenRefreshIntervalHours() | |
| if intervalHours <= 0 { | |
| return false | |
| } | |
| now := time.Now() | |
| r.mu.Lock() | |
| defer r.mu.Unlock() | |
| last, ok := r.tokenRefreshedAt[accountID] | |
| if !ok || last.IsZero() { | |
| r.tokenRefreshedAt[accountID] = now | |
| return false | |
| } | |
| // Add jitter: ±15% random variation to avoid all accounts refreshing at the same time. | |
| // For a 6h interval, this gives a range of ~5h06m to ~6h54m. | |
| baseInterval := time.Duration(intervalHours) * time.Hour | |
| jitterRange := baseInterval / 10 // ±10% → total range is 80%-120% of base | |
| jitter := time.Duration(rand.Int63n(int64(2*jitterRange+1))) - jitterRange | |
| effectiveInterval := baseInterval + jitter | |
| if effectiveInterval < time.Hour { | |
| effectiveInterval = time.Hour | |
| } | |
| return now.Sub(last) >= effectiveInterval | |
| } | |
| func (r *Resolver) markTokenRefreshedNow(accountID string) { | |
| if strings.TrimSpace(accountID) == "" { | |
| return | |
| } | |
| // Add a random offset (0-30min) to spread out future refreshes across accounts. | |
| // This prevents all accounts from refreshing at the same wall-clock time. | |
| offset := time.Duration(rand.Int63n(int64(30*time.Minute))) | |
| r.mu.Lock() | |
| defer r.mu.Unlock() | |
| r.tokenRefreshedAt[accountID] = time.Now().Add(-offset) | |
| } | |
| func (r *Resolver) clearTokenRefreshMark(accountID string) { | |
| if strings.TrimSpace(accountID) == "" { | |
| return | |
| } | |
| r.mu.Lock() | |
| defer r.mu.Unlock() | |
| delete(r.tokenRefreshedAt, accountID) | |
| } | |