|
|
const { v4: uuidv4 } = require('uuid') |
|
|
const crypto = require('crypto') |
|
|
const ProxyHelper = require('../utils/proxyHelper') |
|
|
const axios = require('axios') |
|
|
const redis = require('../models/redis') |
|
|
const config = require('../../config/config') |
|
|
const logger = require('../utils/logger') |
|
|
const { maskToken } = require('../utils/tokenMask') |
|
|
const { |
|
|
logRefreshStart, |
|
|
logRefreshSuccess, |
|
|
logRefreshError, |
|
|
logTokenUsage, |
|
|
logRefreshSkipped |
|
|
} = require('../utils/tokenRefreshLogger') |
|
|
const tokenRefreshService = require('./tokenRefreshService') |
|
|
const LRUCache = require('../utils/lruCache') |
|
|
const { formatDateWithTimezone, getISOStringWithTimezone } = require('../utils/dateHelper') |
|
|
|
|
|
class ClaudeAccountService { |
|
|
constructor() { |
|
|
this.claudeApiUrl = 'https://console.anthropic.com/v1/oauth/token' |
|
|
this.claudeOauthClientId = '9d1c250a-e61b-44d9-88ed-5944d1962f5e' |
|
|
let maxWarnings = parseInt(process.env.CLAUDE_5H_WARNING_MAX_NOTIFICATIONS || '', 10) |
|
|
|
|
|
if (Number.isNaN(maxWarnings) && config.claude?.fiveHourWarning) { |
|
|
maxWarnings = parseInt(config.claude.fiveHourWarning.maxNotificationsPerWindow, 10) |
|
|
} |
|
|
|
|
|
if (Number.isNaN(maxWarnings) || maxWarnings < 1) { |
|
|
maxWarnings = 1 |
|
|
} |
|
|
|
|
|
this.maxFiveHourWarningsPerWindow = Math.min(maxWarnings, 10) |
|
|
|
|
|
|
|
|
this.ENCRYPTION_ALGORITHM = 'aes-256-cbc' |
|
|
this.ENCRYPTION_SALT = 'salt' |
|
|
|
|
|
|
|
|
|
|
|
this._encryptionKeyCache = null |
|
|
|
|
|
|
|
|
this._decryptCache = new LRUCache(500) |
|
|
|
|
|
|
|
|
setInterval( |
|
|
() => { |
|
|
this._decryptCache.cleanup() |
|
|
logger.info('🧹 Claude decrypt cache cleanup completed', this._decryptCache.getStats()) |
|
|
}, |
|
|
10 * 60 * 1000 |
|
|
) |
|
|
} |
|
|
|
|
|
|
|
|
async createAccount(options = {}) { |
|
|
const { |
|
|
name = 'Unnamed Account', |
|
|
description = '', |
|
|
email = '', |
|
|
password = '', |
|
|
refreshToken = '', |
|
|
claudeAiOauth = null, |
|
|
proxy = null, |
|
|
isActive = true, |
|
|
accountType = 'shared', |
|
|
platform = 'claude', |
|
|
priority = 50, |
|
|
schedulable = true, |
|
|
subscriptionInfo = null, |
|
|
autoStopOnWarning = false, |
|
|
useUnifiedUserAgent = false, |
|
|
useUnifiedClientId = false, |
|
|
unifiedClientId = '', |
|
|
expiresAt = null |
|
|
} = options |
|
|
|
|
|
const accountId = uuidv4() |
|
|
|
|
|
let accountData |
|
|
|
|
|
if (claudeAiOauth) { |
|
|
|
|
|
accountData = { |
|
|
id: accountId, |
|
|
name, |
|
|
description, |
|
|
email: this._encryptSensitiveData(email), |
|
|
password: this._encryptSensitiveData(password), |
|
|
claudeAiOauth: this._encryptSensitiveData(JSON.stringify(claudeAiOauth)), |
|
|
accessToken: this._encryptSensitiveData(claudeAiOauth.accessToken), |
|
|
refreshToken: this._encryptSensitiveData(claudeAiOauth.refreshToken), |
|
|
expiresAt: claudeAiOauth.expiresAt.toString(), |
|
|
scopes: claudeAiOauth.scopes.join(' '), |
|
|
proxy: proxy ? JSON.stringify(proxy) : '', |
|
|
isActive: isActive.toString(), |
|
|
accountType, |
|
|
platform, |
|
|
priority: priority.toString(), |
|
|
createdAt: new Date().toISOString(), |
|
|
lastUsedAt: '', |
|
|
lastRefreshAt: '', |
|
|
status: 'active', |
|
|
errorMessage: '', |
|
|
schedulable: schedulable.toString(), |
|
|
autoStopOnWarning: autoStopOnWarning.toString(), |
|
|
useUnifiedUserAgent: useUnifiedUserAgent.toString(), |
|
|
useUnifiedClientId: useUnifiedClientId.toString(), |
|
|
unifiedClientId: unifiedClientId || '', |
|
|
|
|
|
subscriptionInfo: subscriptionInfo |
|
|
? JSON.stringify(subscriptionInfo) |
|
|
: claudeAiOauth.subscriptionInfo |
|
|
? JSON.stringify(claudeAiOauth.subscriptionInfo) |
|
|
: '', |
|
|
|
|
|
subscriptionExpiresAt: expiresAt || '' |
|
|
} |
|
|
} else { |
|
|
|
|
|
accountData = { |
|
|
id: accountId, |
|
|
name, |
|
|
description, |
|
|
email: this._encryptSensitiveData(email), |
|
|
password: this._encryptSensitiveData(password), |
|
|
refreshToken: this._encryptSensitiveData(refreshToken), |
|
|
accessToken: '', |
|
|
expiresAt: '', |
|
|
scopes: '', |
|
|
proxy: proxy ? JSON.stringify(proxy) : '', |
|
|
isActive: isActive.toString(), |
|
|
accountType, |
|
|
platform, |
|
|
priority: priority.toString(), |
|
|
createdAt: new Date().toISOString(), |
|
|
lastUsedAt: '', |
|
|
lastRefreshAt: '', |
|
|
status: 'created', |
|
|
errorMessage: '', |
|
|
schedulable: schedulable.toString(), |
|
|
autoStopOnWarning: autoStopOnWarning.toString(), |
|
|
useUnifiedUserAgent: useUnifiedUserAgent.toString(), |
|
|
|
|
|
subscriptionInfo: subscriptionInfo ? JSON.stringify(subscriptionInfo) : '', |
|
|
|
|
|
subscriptionExpiresAt: expiresAt || '' |
|
|
} |
|
|
} |
|
|
|
|
|
await redis.setClaudeAccount(accountId, accountData) |
|
|
|
|
|
logger.success(`🏢 Created Claude account: ${name} (${accountId})`) |
|
|
|
|
|
|
|
|
if (claudeAiOauth && claudeAiOauth.accessToken) { |
|
|
|
|
|
const hasProfileScope = claudeAiOauth.scopes && claudeAiOauth.scopes.includes('user:profile') |
|
|
|
|
|
if (hasProfileScope) { |
|
|
try { |
|
|
const agent = this._createProxyAgent(proxy) |
|
|
await this.fetchAndUpdateAccountProfile(accountId, claudeAiOauth.accessToken, agent) |
|
|
logger.info(`📊 Successfully fetched profile info for new account: ${name}`) |
|
|
} catch (profileError) { |
|
|
logger.warn(`⚠️ Failed to fetch profile info for new account: ${profileError.message}`) |
|
|
} |
|
|
} else { |
|
|
logger.info(`⏩ Skipping profile fetch for account ${name} (no user:profile scope)`) |
|
|
} |
|
|
} |
|
|
|
|
|
return { |
|
|
id: accountId, |
|
|
name, |
|
|
description, |
|
|
email, |
|
|
isActive, |
|
|
proxy, |
|
|
accountType, |
|
|
platform, |
|
|
priority, |
|
|
status: accountData.status, |
|
|
createdAt: accountData.createdAt, |
|
|
expiresAt: accountData.expiresAt, |
|
|
subscriptionExpiresAt: |
|
|
accountData.subscriptionExpiresAt && accountData.subscriptionExpiresAt !== '' |
|
|
? accountData.subscriptionExpiresAt |
|
|
: null, |
|
|
scopes: claudeAiOauth ? claudeAiOauth.scopes : [], |
|
|
autoStopOnWarning, |
|
|
useUnifiedUserAgent, |
|
|
useUnifiedClientId, |
|
|
unifiedClientId |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async refreshAccountToken(accountId) { |
|
|
let lockAcquired = false |
|
|
|
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
|
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
const refreshToken = this._decryptSensitiveData(accountData.refreshToken) |
|
|
|
|
|
if (!refreshToken) { |
|
|
throw new Error('No refresh token available - manual token update required') |
|
|
} |
|
|
|
|
|
|
|
|
lockAcquired = await tokenRefreshService.acquireRefreshLock(accountId, 'claude') |
|
|
|
|
|
if (!lockAcquired) { |
|
|
|
|
|
logger.info( |
|
|
`🔒 Token refresh already in progress for account: ${accountData.name} (${accountId})` |
|
|
) |
|
|
logRefreshSkipped(accountId, accountData.name, 'claude', 'already_locked') |
|
|
|
|
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, 2000)) |
|
|
|
|
|
|
|
|
const updatedData = await redis.getClaudeAccount(accountId) |
|
|
if (updatedData && updatedData.accessToken) { |
|
|
const accessToken = this._decryptSensitiveData(updatedData.accessToken) |
|
|
return { |
|
|
success: true, |
|
|
accessToken, |
|
|
expiresAt: updatedData.expiresAt |
|
|
} |
|
|
} |
|
|
|
|
|
throw new Error('Token refresh in progress by another process') |
|
|
} |
|
|
|
|
|
|
|
|
logRefreshStart(accountId, accountData.name, 'claude', 'manual_refresh') |
|
|
logger.info(`🔄 Starting token refresh for account: ${accountData.name} (${accountId})`) |
|
|
|
|
|
|
|
|
const agent = this._createProxyAgent(accountData.proxy) |
|
|
|
|
|
const axiosConfig = { |
|
|
headers: { |
|
|
'Content-Type': 'application/json', |
|
|
Accept: 'application/json, text/plain, */*', |
|
|
'User-Agent': 'claude-cli/1.0.56 (external, cli)', |
|
|
'Accept-Language': 'en-US,en;q=0.9', |
|
|
Referer: 'https://claude.ai/', |
|
|
Origin: 'https://claude.ai' |
|
|
}, |
|
|
timeout: 30000 |
|
|
} |
|
|
|
|
|
if (agent) { |
|
|
axiosConfig.httpAgent = agent |
|
|
axiosConfig.httpsAgent = agent |
|
|
axiosConfig.proxy = false |
|
|
} |
|
|
|
|
|
const response = await axios.post( |
|
|
this.claudeApiUrl, |
|
|
{ |
|
|
grant_type: 'refresh_token', |
|
|
refresh_token: refreshToken, |
|
|
client_id: this.claudeOauthClientId |
|
|
}, |
|
|
axiosConfig |
|
|
) |
|
|
|
|
|
if (response.status === 200) { |
|
|
|
|
|
logger.authDetail('Token refresh response', response.data) |
|
|
|
|
|
|
|
|
logger.info('📊 Token refresh response (analyzing for subscription info):', { |
|
|
status: response.status, |
|
|
hasData: !!response.data, |
|
|
dataKeys: response.data ? Object.keys(response.data) : [] |
|
|
}) |
|
|
|
|
|
const { access_token, refresh_token, expires_in } = response.data |
|
|
|
|
|
|
|
|
if ( |
|
|
response.data.subscription || |
|
|
response.data.plan || |
|
|
response.data.tier || |
|
|
response.data.account_type |
|
|
) { |
|
|
const subscriptionInfo = { |
|
|
subscription: response.data.subscription, |
|
|
plan: response.data.plan, |
|
|
tier: response.data.tier, |
|
|
accountType: response.data.account_type, |
|
|
features: response.data.features, |
|
|
limits: response.data.limits |
|
|
} |
|
|
logger.info('🎯 Found subscription info in refresh response:', subscriptionInfo) |
|
|
|
|
|
|
|
|
accountData.subscriptionInfo = JSON.stringify(subscriptionInfo) |
|
|
} |
|
|
|
|
|
|
|
|
accountData.accessToken = this._encryptSensitiveData(access_token) |
|
|
accountData.refreshToken = this._encryptSensitiveData(refresh_token) |
|
|
accountData.expiresAt = (Date.now() + expires_in * 1000).toString() |
|
|
accountData.lastRefreshAt = new Date().toISOString() |
|
|
accountData.status = 'active' |
|
|
accountData.errorMessage = '' |
|
|
|
|
|
await redis.setClaudeAccount(accountId, accountData) |
|
|
|
|
|
|
|
|
|
|
|
const hasProfileScope = accountData.scopes && accountData.scopes.includes('user:profile') |
|
|
|
|
|
if (hasProfileScope) { |
|
|
try { |
|
|
await this.fetchAndUpdateAccountProfile(accountId, access_token, agent) |
|
|
} catch (profileError) { |
|
|
logger.warn(`⚠️ Failed to fetch profile info after refresh: ${profileError.message}`) |
|
|
} |
|
|
} else { |
|
|
logger.debug( |
|
|
`⏩ Skipping profile fetch after refresh for account ${accountId} (no user:profile scope)` |
|
|
) |
|
|
} |
|
|
|
|
|
|
|
|
logRefreshSuccess(accountId, accountData.name, 'claude', { |
|
|
accessToken: access_token, |
|
|
refreshToken: refresh_token, |
|
|
expiresAt: accountData.expiresAt, |
|
|
scopes: accountData.scopes |
|
|
}) |
|
|
|
|
|
logger.success( |
|
|
`🔄 Refreshed token for account: ${accountData.name} (${accountId}) - Access Token: ${maskToken(access_token)}` |
|
|
) |
|
|
|
|
|
return { |
|
|
success: true, |
|
|
accessToken: access_token, |
|
|
expiresAt: accountData.expiresAt |
|
|
} |
|
|
} else { |
|
|
throw new Error(`Token refresh failed with status: ${response.status}`) |
|
|
} |
|
|
} catch (error) { |
|
|
|
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (accountData) { |
|
|
logRefreshError(accountId, accountData.name, 'claude', error) |
|
|
accountData.status = 'error' |
|
|
accountData.errorMessage = error.message |
|
|
await redis.setClaudeAccount(accountId, accountData) |
|
|
|
|
|
|
|
|
try { |
|
|
const webhookNotifier = require('../utils/webhookNotifier') |
|
|
await webhookNotifier.sendAccountAnomalyNotification({ |
|
|
accountId, |
|
|
accountName: accountData.name, |
|
|
platform: 'claude-oauth', |
|
|
status: 'error', |
|
|
errorCode: 'CLAUDE_OAUTH_ERROR', |
|
|
reason: `Token refresh failed: ${error.message}` |
|
|
}) |
|
|
} catch (webhookError) { |
|
|
logger.error('Failed to send webhook notification:', webhookError) |
|
|
} |
|
|
} |
|
|
|
|
|
logger.error(`❌ Failed to refresh token for account ${accountId}:`, error) |
|
|
|
|
|
throw error |
|
|
} finally { |
|
|
|
|
|
if (lockAcquired) { |
|
|
await tokenRefreshService.releaseRefreshLock(accountId, 'claude') |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async getAccount(accountId) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
|
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
return null |
|
|
} |
|
|
|
|
|
return accountData |
|
|
} catch (error) { |
|
|
logger.error('❌ Failed to get Claude account:', error) |
|
|
return null |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async getValidAccessToken(accountId) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
|
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
if (accountData.isActive !== 'true') { |
|
|
throw new Error('Account is disabled') |
|
|
} |
|
|
|
|
|
|
|
|
const expiresAt = parseInt(accountData.expiresAt) |
|
|
const now = Date.now() |
|
|
const isExpired = !expiresAt || now >= expiresAt - 60000 |
|
|
|
|
|
|
|
|
logTokenUsage(accountId, accountData.name, 'claude', accountData.expiresAt, isExpired) |
|
|
|
|
|
if (isExpired) { |
|
|
logger.info(`🔄 Token expired/expiring for account ${accountId}, attempting refresh...`) |
|
|
try { |
|
|
const refreshResult = await this.refreshAccountToken(accountId) |
|
|
return refreshResult.accessToken |
|
|
} catch (refreshError) { |
|
|
logger.warn(`⚠️ Token refresh failed for account ${accountId}: ${refreshError.message}`) |
|
|
|
|
|
const currentToken = this._decryptSensitiveData(accountData.accessToken) |
|
|
if (currentToken) { |
|
|
logger.info(`🔄 Using current token for account ${accountId} (refresh failed)`) |
|
|
return currentToken |
|
|
} |
|
|
throw refreshError |
|
|
} |
|
|
} |
|
|
|
|
|
const accessToken = this._decryptSensitiveData(accountData.accessToken) |
|
|
|
|
|
if (!accessToken) { |
|
|
throw new Error('No access token available') |
|
|
} |
|
|
|
|
|
|
|
|
accountData.lastUsedAt = new Date().toISOString() |
|
|
await this.updateSessionWindow(accountId, accountData) |
|
|
await redis.setClaudeAccount(accountId, accountData) |
|
|
|
|
|
return accessToken |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to get valid access token for account ${accountId}:`, error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async getAllAccounts() { |
|
|
try { |
|
|
const accounts = await redis.getAllClaudeAccounts() |
|
|
|
|
|
|
|
|
const processedAccounts = await Promise.all( |
|
|
accounts.map(async (account) => { |
|
|
|
|
|
const rateLimitInfo = await this.getAccountRateLimitInfo(account.id) |
|
|
|
|
|
|
|
|
const sessionWindowInfo = await this.getSessionWindowInfo(account.id) |
|
|
|
|
|
|
|
|
const claudeUsage = this.buildClaudeUsageSnapshot(account) |
|
|
|
|
|
|
|
|
const scopes = account.scopes && account.scopes.trim() ? account.scopes.split(' ') : [] |
|
|
const isOAuth = scopes.includes('user:profile') && scopes.includes('user:inference') |
|
|
const authType = isOAuth ? 'oauth' : 'setup-token' |
|
|
|
|
|
return { |
|
|
id: account.id, |
|
|
name: account.name, |
|
|
description: account.description, |
|
|
email: account.email ? this._maskEmail(this._decryptSensitiveData(account.email)) : '', |
|
|
isActive: account.isActive === 'true', |
|
|
proxy: account.proxy ? JSON.parse(account.proxy) : null, |
|
|
status: account.status, |
|
|
errorMessage: account.errorMessage, |
|
|
accountType: account.accountType || 'shared', |
|
|
priority: parseInt(account.priority) || 50, |
|
|
platform: account.platform || 'claude', |
|
|
authType, |
|
|
createdAt: account.createdAt, |
|
|
lastUsedAt: account.lastUsedAt, |
|
|
lastRefreshAt: account.lastRefreshAt, |
|
|
expiresAt: account.expiresAt || null, |
|
|
subscriptionExpiresAt: |
|
|
account.subscriptionExpiresAt && account.subscriptionExpiresAt !== '' |
|
|
? account.subscriptionExpiresAt |
|
|
: null, |
|
|
|
|
|
|
|
|
scopes: account.scopes && account.scopes.trim() ? account.scopes.split(' ') : [], |
|
|
|
|
|
hasRefreshToken: !!account.refreshToken, |
|
|
|
|
|
subscriptionInfo: account.subscriptionInfo |
|
|
? JSON.parse(account.subscriptionInfo) |
|
|
: null, |
|
|
|
|
|
rateLimitStatus: rateLimitInfo |
|
|
? { |
|
|
isRateLimited: rateLimitInfo.isRateLimited, |
|
|
rateLimitedAt: rateLimitInfo.rateLimitedAt, |
|
|
minutesRemaining: rateLimitInfo.minutesRemaining |
|
|
} |
|
|
: null, |
|
|
|
|
|
sessionWindow: sessionWindowInfo || { |
|
|
hasActiveWindow: false, |
|
|
windowStart: null, |
|
|
windowEnd: null, |
|
|
progress: 0, |
|
|
remainingTime: null, |
|
|
lastRequestTime: null |
|
|
}, |
|
|
|
|
|
claudeUsage: claudeUsage || null, |
|
|
|
|
|
schedulable: account.schedulable !== 'false', |
|
|
|
|
|
autoStopOnWarning: account.autoStopOnWarning === 'true', |
|
|
|
|
|
fiveHourAutoStopped: account.fiveHourAutoStopped === 'true', |
|
|
fiveHourStoppedAt: account.fiveHourStoppedAt || null, |
|
|
|
|
|
useUnifiedUserAgent: account.useUnifiedUserAgent === 'true', |
|
|
|
|
|
useUnifiedClientId: account.useUnifiedClientId === 'true', |
|
|
unifiedClientId: account.unifiedClientId || '', |
|
|
|
|
|
stoppedReason: account.stoppedReason || null |
|
|
} |
|
|
}) |
|
|
) |
|
|
|
|
|
return processedAccounts |
|
|
} catch (error) { |
|
|
logger.error('❌ Failed to get Claude accounts:', error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async getAccountOverview(accountId) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
|
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
return null |
|
|
} |
|
|
|
|
|
const [sessionWindowInfo, rateLimitInfo] = await Promise.all([ |
|
|
this.getSessionWindowInfo(accountId), |
|
|
this.getAccountRateLimitInfo(accountId) |
|
|
]) |
|
|
|
|
|
const sessionWindow = sessionWindowInfo || { |
|
|
hasActiveWindow: false, |
|
|
windowStart: null, |
|
|
windowEnd: null, |
|
|
progress: 0, |
|
|
remainingTime: null, |
|
|
lastRequestTime: accountData.lastRequestTime || null, |
|
|
sessionWindowStatus: accountData.sessionWindowStatus || null |
|
|
} |
|
|
|
|
|
const rateLimitStatus = rateLimitInfo |
|
|
? { |
|
|
isRateLimited: !!rateLimitInfo.isRateLimited, |
|
|
rateLimitedAt: rateLimitInfo.rateLimitedAt || null, |
|
|
minutesRemaining: rateLimitInfo.minutesRemaining || 0, |
|
|
rateLimitEndAt: rateLimitInfo.rateLimitEndAt || null |
|
|
} |
|
|
: { |
|
|
isRateLimited: false, |
|
|
rateLimitedAt: null, |
|
|
minutesRemaining: 0, |
|
|
rateLimitEndAt: null |
|
|
} |
|
|
|
|
|
return { |
|
|
id: accountData.id, |
|
|
accountType: accountData.accountType || 'shared', |
|
|
platform: accountData.platform || 'claude', |
|
|
isActive: accountData.isActive === 'true', |
|
|
schedulable: accountData.schedulable !== 'false', |
|
|
sessionWindow, |
|
|
rateLimitStatus |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to build Claude account overview for ${accountId}:`, error) |
|
|
return null |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async updateAccount(accountId, updates) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
|
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
const allowedUpdates = [ |
|
|
'name', |
|
|
'description', |
|
|
'email', |
|
|
'password', |
|
|
'refreshToken', |
|
|
'proxy', |
|
|
'isActive', |
|
|
'claudeAiOauth', |
|
|
'accountType', |
|
|
'priority', |
|
|
'schedulable', |
|
|
'subscriptionInfo', |
|
|
'autoStopOnWarning', |
|
|
'useUnifiedUserAgent', |
|
|
'useUnifiedClientId', |
|
|
'unifiedClientId', |
|
|
'subscriptionExpiresAt' |
|
|
] |
|
|
const updatedData = { ...accountData } |
|
|
let shouldClearAutoStopFields = false |
|
|
|
|
|
|
|
|
const oldRefreshToken = this._decryptSensitiveData(accountData.refreshToken) |
|
|
|
|
|
for (const [field, value] of Object.entries(updates)) { |
|
|
if (allowedUpdates.includes(field)) { |
|
|
if (['email', 'password', 'refreshToken'].includes(field)) { |
|
|
updatedData[field] = this._encryptSensitiveData(value) |
|
|
} else if (field === 'proxy') { |
|
|
updatedData[field] = value ? JSON.stringify(value) : '' |
|
|
} else if (field === 'priority') { |
|
|
updatedData[field] = value.toString() |
|
|
} else if (field === 'subscriptionInfo') { |
|
|
|
|
|
updatedData[field] = typeof value === 'string' ? value : JSON.stringify(value) |
|
|
} else if (field === 'subscriptionExpiresAt') { |
|
|
|
|
|
updatedData[field] = value ? value.toString() : '' |
|
|
} else if (field === 'claudeAiOauth') { |
|
|
|
|
|
if (value) { |
|
|
updatedData.claudeAiOauth = this._encryptSensitiveData(JSON.stringify(value)) |
|
|
updatedData.accessToken = this._encryptSensitiveData(value.accessToken) |
|
|
updatedData.refreshToken = this._encryptSensitiveData(value.refreshToken) |
|
|
updatedData.expiresAt = value.expiresAt.toString() |
|
|
updatedData.scopes = value.scopes.join(' ') |
|
|
updatedData.status = 'active' |
|
|
updatedData.errorMessage = '' |
|
|
updatedData.lastRefreshAt = new Date().toISOString() |
|
|
} |
|
|
} else { |
|
|
updatedData[field] = value !== null && value !== undefined ? value.toString() : '' |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
if (updates.refreshToken && !oldRefreshToken && updates.refreshToken.trim()) { |
|
|
const newExpiresAt = Date.now() + 10 * 60 * 1000 |
|
|
updatedData.expiresAt = newExpiresAt.toString() |
|
|
logger.info( |
|
|
`🔄 New refresh token added for account ${accountId}, setting expiry to 10 minutes` |
|
|
) |
|
|
} |
|
|
|
|
|
|
|
|
if (updates.claudeAiOauth && updates.claudeAiOauth.refreshToken && !oldRefreshToken) { |
|
|
|
|
|
const providedExpiry = parseInt(updates.claudeAiOauth.expiresAt) |
|
|
const now = Date.now() |
|
|
const oneHour = 60 * 60 * 1000 |
|
|
|
|
|
if (providedExpiry - now > oneHour) { |
|
|
const newExpiresAt = now + 10 * 60 * 1000 |
|
|
updatedData.expiresAt = newExpiresAt.toString() |
|
|
logger.info( |
|
|
`🔄 Adjusted expiry time to 10 minutes for account ${accountId} with refresh token` |
|
|
) |
|
|
} |
|
|
} |
|
|
|
|
|
updatedData.updatedAt = new Date().toISOString() |
|
|
|
|
|
|
|
|
if (Object.prototype.hasOwnProperty.call(updates, 'schedulable')) { |
|
|
|
|
|
delete updatedData.rateLimitAutoStopped |
|
|
delete updatedData.fiveHourAutoStopped |
|
|
delete updatedData.fiveHourStoppedAt |
|
|
delete updatedData.tempErrorAutoStopped |
|
|
|
|
|
delete updatedData.autoStoppedAt |
|
|
delete updatedData.stoppedReason |
|
|
shouldClearAutoStopFields = true |
|
|
|
|
|
await this._clearFiveHourWarningMetadata(accountId, updatedData) |
|
|
|
|
|
|
|
|
if (updates.schedulable === true || updates.schedulable === 'true') { |
|
|
logger.info(`✅ Manually enabled scheduling for account ${accountId}`) |
|
|
} else { |
|
|
logger.info(`⛔ Manually disabled scheduling for account ${accountId}`) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
if (updates.isActive === 'false' && accountData.isActive === 'true') { |
|
|
try { |
|
|
const webhookNotifier = require('../utils/webhookNotifier') |
|
|
await webhookNotifier.sendAccountAnomalyNotification({ |
|
|
accountId, |
|
|
accountName: updatedData.name || 'Unknown Account', |
|
|
platform: 'claude-oauth', |
|
|
status: 'disabled', |
|
|
errorCode: 'CLAUDE_OAUTH_MANUALLY_DISABLED', |
|
|
reason: 'Account manually disabled by administrator' |
|
|
}) |
|
|
} catch (webhookError) { |
|
|
logger.error( |
|
|
'Failed to send webhook notification for manual account disable:', |
|
|
webhookError |
|
|
) |
|
|
} |
|
|
} |
|
|
|
|
|
await redis.setClaudeAccount(accountId, updatedData) |
|
|
|
|
|
if (shouldClearAutoStopFields) { |
|
|
const fieldsToRemove = [ |
|
|
'rateLimitAutoStopped', |
|
|
'fiveHourAutoStopped', |
|
|
'fiveHourStoppedAt', |
|
|
'tempErrorAutoStopped', |
|
|
'autoStoppedAt', |
|
|
'stoppedReason' |
|
|
] |
|
|
await this._removeAccountFields(accountId, fieldsToRemove, 'manual_schedule_update') |
|
|
} |
|
|
|
|
|
logger.success(`📝 Updated Claude account: ${accountId}`) |
|
|
|
|
|
return { success: true } |
|
|
} catch (error) { |
|
|
logger.error('❌ Failed to update Claude account:', error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async deleteAccount(accountId) { |
|
|
try { |
|
|
|
|
|
const accountGroupService = require('./accountGroupService') |
|
|
await accountGroupService.removeAccountFromAllGroups(accountId) |
|
|
|
|
|
const result = await redis.deleteClaudeAccount(accountId) |
|
|
|
|
|
if (result === 0) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
logger.success(`🗑️ Deleted Claude account: ${accountId}`) |
|
|
|
|
|
return { success: true } |
|
|
} catch (error) { |
|
|
logger.error('❌ Failed to delete Claude account:', error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
isSubscriptionExpired(account) { |
|
|
if (!account.subscriptionExpiresAt) { |
|
|
return false |
|
|
} |
|
|
|
|
|
const expiryDate = new Date(account.subscriptionExpiresAt) |
|
|
const now = new Date() |
|
|
|
|
|
if (expiryDate <= now) { |
|
|
logger.debug( |
|
|
`⏰ Account ${account.name} (${account.id}) expired at ${account.subscriptionExpiresAt}` |
|
|
) |
|
|
return true |
|
|
} |
|
|
|
|
|
return false |
|
|
} |
|
|
|
|
|
|
|
|
async selectAvailableAccount(sessionHash = null, modelName = null) { |
|
|
try { |
|
|
const accounts = await redis.getAllClaudeAccounts() |
|
|
|
|
|
let activeAccounts = accounts.filter( |
|
|
(account) => |
|
|
account.isActive === 'true' && |
|
|
account.status !== 'error' && |
|
|
account.schedulable !== 'false' && |
|
|
!this.isSubscriptionExpired(account) |
|
|
) |
|
|
|
|
|
|
|
|
if (modelName && modelName.toLowerCase().includes('opus')) { |
|
|
activeAccounts = activeAccounts.filter((account) => { |
|
|
|
|
|
if (account.subscriptionInfo) { |
|
|
try { |
|
|
const info = JSON.parse(account.subscriptionInfo) |
|
|
|
|
|
if (info.hasClaudePro === true && info.hasClaudeMax !== true) { |
|
|
return false |
|
|
} |
|
|
if (info.accountType === 'claude_pro' || info.accountType === 'claude_free') { |
|
|
return false |
|
|
} |
|
|
} catch (e) { |
|
|
|
|
|
return true |
|
|
} |
|
|
} |
|
|
|
|
|
return true |
|
|
}) |
|
|
|
|
|
if (activeAccounts.length === 0) { |
|
|
throw new Error('No Claude accounts available that support Opus model') |
|
|
} |
|
|
} |
|
|
|
|
|
if (activeAccounts.length === 0) { |
|
|
throw new Error('No active Claude accounts available') |
|
|
} |
|
|
|
|
|
|
|
|
if (sessionHash) { |
|
|
const mappedAccountId = await redis.getSessionAccountMapping(sessionHash) |
|
|
if (mappedAccountId) { |
|
|
|
|
|
const mappedAccount = activeAccounts.find((acc) => acc.id === mappedAccountId) |
|
|
if (mappedAccount) { |
|
|
|
|
|
await redis.extendSessionAccountMappingTTL(sessionHash) |
|
|
logger.info( |
|
|
`🎯 Using sticky session account: ${mappedAccount.name} (${mappedAccountId}) for session ${sessionHash}` |
|
|
) |
|
|
return mappedAccountId |
|
|
} else { |
|
|
logger.warn( |
|
|
`⚠️ Mapped account ${mappedAccountId} is no longer available, selecting new account` |
|
|
) |
|
|
|
|
|
await redis.deleteSessionAccountMapping(sessionHash) |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
const sortedAccounts = activeAccounts.sort((a, b) => { |
|
|
const aLastUsed = new Date(a.lastUsedAt || 0).getTime() |
|
|
const bLastUsed = new Date(b.lastUsedAt || 0).getTime() |
|
|
return aLastUsed - bLastUsed |
|
|
}) |
|
|
|
|
|
const selectedAccountId = sortedAccounts[0].id |
|
|
|
|
|
|
|
|
if (sessionHash) { |
|
|
|
|
|
const ttlSeconds = (config.session?.stickyTtlHours || 1) * 60 * 60 |
|
|
await redis.setSessionAccountMapping(sessionHash, selectedAccountId, ttlSeconds) |
|
|
logger.info( |
|
|
`🎯 Created new sticky session mapping: ${sortedAccounts[0].name} (${selectedAccountId}) for session ${sessionHash}` |
|
|
) |
|
|
} |
|
|
|
|
|
return selectedAccountId |
|
|
} catch (error) { |
|
|
logger.error('❌ Failed to select available account:', error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async selectAccountForApiKey(apiKeyData, sessionHash = null, modelName = null) { |
|
|
try { |
|
|
|
|
|
if (apiKeyData.claudeAccountId) { |
|
|
const boundAccount = await redis.getClaudeAccount(apiKeyData.claudeAccountId) |
|
|
if ( |
|
|
boundAccount && |
|
|
boundAccount.isActive === 'true' && |
|
|
boundAccount.status !== 'error' && |
|
|
boundAccount.schedulable !== 'false' && |
|
|
!this.isSubscriptionExpired(boundAccount) |
|
|
) { |
|
|
logger.info( |
|
|
`🎯 Using bound dedicated account: ${boundAccount.name} (${apiKeyData.claudeAccountId}) for API key ${apiKeyData.name}` |
|
|
) |
|
|
return apiKeyData.claudeAccountId |
|
|
} else { |
|
|
logger.warn( |
|
|
`⚠️ Bound account ${apiKeyData.claudeAccountId} is not available, falling back to shared pool` |
|
|
) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
const accounts = await redis.getAllClaudeAccounts() |
|
|
|
|
|
let sharedAccounts = accounts.filter( |
|
|
(account) => |
|
|
account.isActive === 'true' && |
|
|
account.status !== 'error' && |
|
|
account.schedulable !== 'false' && |
|
|
(account.accountType === 'shared' || !account.accountType) && |
|
|
!this.isSubscriptionExpired(account) |
|
|
) |
|
|
|
|
|
|
|
|
if (modelName && modelName.toLowerCase().includes('opus')) { |
|
|
sharedAccounts = sharedAccounts.filter((account) => { |
|
|
|
|
|
if (account.subscriptionInfo) { |
|
|
try { |
|
|
const info = JSON.parse(account.subscriptionInfo) |
|
|
|
|
|
if (info.hasClaudePro === true && info.hasClaudeMax !== true) { |
|
|
return false |
|
|
} |
|
|
if (info.accountType === 'claude_pro' || info.accountType === 'claude_free') { |
|
|
return false |
|
|
} |
|
|
} catch (e) { |
|
|
|
|
|
return true |
|
|
} |
|
|
} |
|
|
|
|
|
return true |
|
|
}) |
|
|
|
|
|
if (sharedAccounts.length === 0) { |
|
|
throw new Error('No shared Claude accounts available that support Opus model') |
|
|
} |
|
|
} |
|
|
|
|
|
if (sharedAccounts.length === 0) { |
|
|
throw new Error('No active shared Claude accounts available') |
|
|
} |
|
|
|
|
|
|
|
|
if (sessionHash) { |
|
|
const mappedAccountId = await redis.getSessionAccountMapping(sessionHash) |
|
|
if (mappedAccountId) { |
|
|
|
|
|
const mappedAccount = sharedAccounts.find((acc) => acc.id === mappedAccountId) |
|
|
if (mappedAccount) { |
|
|
|
|
|
const isRateLimited = await this.isAccountRateLimited(mappedAccountId) |
|
|
if (isRateLimited) { |
|
|
logger.warn( |
|
|
`⚠️ Mapped account ${mappedAccountId} is rate limited, selecting new account` |
|
|
) |
|
|
await redis.deleteSessionAccountMapping(sessionHash) |
|
|
} else { |
|
|
|
|
|
await redis.extendSessionAccountMappingTTL(sessionHash) |
|
|
logger.info( |
|
|
`🎯 Using sticky session shared account: ${mappedAccount.name} (${mappedAccountId}) for session ${sessionHash}` |
|
|
) |
|
|
return mappedAccountId |
|
|
} |
|
|
} else { |
|
|
logger.warn( |
|
|
`⚠️ Mapped shared account ${mappedAccountId} is no longer available, selecting new account` |
|
|
) |
|
|
|
|
|
await redis.deleteSessionAccountMapping(sessionHash) |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
const nonRateLimitedAccounts = [] |
|
|
const rateLimitedAccounts = [] |
|
|
|
|
|
for (const account of sharedAccounts) { |
|
|
const isRateLimited = await this.isAccountRateLimited(account.id) |
|
|
if (isRateLimited) { |
|
|
const rateLimitInfo = await this.getAccountRateLimitInfo(account.id) |
|
|
account._rateLimitInfo = rateLimitInfo |
|
|
rateLimitedAccounts.push(account) |
|
|
} else { |
|
|
nonRateLimitedAccounts.push(account) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
let candidateAccounts = nonRateLimitedAccounts |
|
|
|
|
|
|
|
|
if (candidateAccounts.length === 0) { |
|
|
logger.warn('⚠️ All shared accounts are rate limited, selecting from rate limited pool') |
|
|
candidateAccounts = rateLimitedAccounts.sort((a, b) => { |
|
|
const aRateLimitedAt = new Date(a._rateLimitInfo.rateLimitedAt).getTime() |
|
|
const bRateLimitedAt = new Date(b._rateLimitInfo.rateLimitedAt).getTime() |
|
|
return aRateLimitedAt - bRateLimitedAt |
|
|
}) |
|
|
} else { |
|
|
|
|
|
candidateAccounts = candidateAccounts.sort((a, b) => { |
|
|
const aLastUsed = new Date(a.lastUsedAt || 0).getTime() |
|
|
const bLastUsed = new Date(b.lastUsedAt || 0).getTime() |
|
|
return aLastUsed - bLastUsed |
|
|
}) |
|
|
} |
|
|
|
|
|
if (candidateAccounts.length === 0) { |
|
|
throw new Error('No available shared Claude accounts') |
|
|
} |
|
|
|
|
|
const selectedAccountId = candidateAccounts[0].id |
|
|
|
|
|
|
|
|
if (sessionHash) { |
|
|
|
|
|
const ttlSeconds = (config.session?.stickyTtlHours || 1) * 60 * 60 |
|
|
await redis.setSessionAccountMapping(sessionHash, selectedAccountId, ttlSeconds) |
|
|
logger.info( |
|
|
`🎯 Created new sticky session mapping for shared account: ${candidateAccounts[0].name} (${selectedAccountId}) for session ${sessionHash}` |
|
|
) |
|
|
} |
|
|
|
|
|
logger.info( |
|
|
`🎯 Selected shared account: ${candidateAccounts[0].name} (${selectedAccountId}) for API key ${apiKeyData.name}` |
|
|
) |
|
|
return selectedAccountId |
|
|
} catch (error) { |
|
|
logger.error('❌ Failed to select account for API key:', error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
_createProxyAgent(proxyConfig) { |
|
|
const proxyAgent = ProxyHelper.createProxyAgent(proxyConfig) |
|
|
if (proxyAgent) { |
|
|
logger.info( |
|
|
`🌐 Using proxy for Claude request: ${ProxyHelper.getProxyDescription(proxyConfig)}` |
|
|
) |
|
|
} else if (proxyConfig) { |
|
|
logger.debug('🌐 Failed to create proxy agent for Claude') |
|
|
} else { |
|
|
logger.debug('🌐 No proxy configured for Claude request') |
|
|
} |
|
|
return proxyAgent |
|
|
} |
|
|
|
|
|
|
|
|
_encryptSensitiveData(data) { |
|
|
if (!data) { |
|
|
return '' |
|
|
} |
|
|
|
|
|
try { |
|
|
const key = this._generateEncryptionKey() |
|
|
const iv = crypto.randomBytes(16) |
|
|
|
|
|
const cipher = crypto.createCipheriv(this.ENCRYPTION_ALGORITHM, key, iv) |
|
|
let encrypted = cipher.update(data, 'utf8', 'hex') |
|
|
encrypted += cipher.final('hex') |
|
|
|
|
|
|
|
|
return `${iv.toString('hex')}:${encrypted}` |
|
|
} catch (error) { |
|
|
logger.error('❌ Encryption error:', error) |
|
|
return data |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
_decryptSensitiveData(encryptedData) { |
|
|
if (!encryptedData) { |
|
|
return '' |
|
|
} |
|
|
|
|
|
|
|
|
const cacheKey = crypto.createHash('sha256').update(encryptedData).digest('hex') |
|
|
const cached = this._decryptCache.get(cacheKey) |
|
|
if (cached !== undefined) { |
|
|
return cached |
|
|
} |
|
|
|
|
|
try { |
|
|
let decrypted = '' |
|
|
|
|
|
|
|
|
if (encryptedData.includes(':')) { |
|
|
|
|
|
const parts = encryptedData.split(':') |
|
|
if (parts.length === 2) { |
|
|
const key = this._generateEncryptionKey() |
|
|
const iv = Buffer.from(parts[0], 'hex') |
|
|
const encrypted = parts[1] |
|
|
|
|
|
const decipher = crypto.createDecipheriv(this.ENCRYPTION_ALGORITHM, key, iv) |
|
|
decrypted = decipher.update(encrypted, 'hex', 'utf8') |
|
|
decrypted += decipher.final('utf8') |
|
|
|
|
|
|
|
|
this._decryptCache.set(cacheKey, decrypted, 5 * 60 * 1000) |
|
|
|
|
|
|
|
|
if ((this._decryptCache.hits + this._decryptCache.misses) % 1000 === 0) { |
|
|
this._decryptCache.printStats() |
|
|
} |
|
|
|
|
|
return decrypted |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
try { |
|
|
const decipher = crypto.createDecipher('aes-256-cbc', config.security.encryptionKey) |
|
|
decrypted = decipher.update(encryptedData, 'hex', 'utf8') |
|
|
decrypted += decipher.final('utf8') |
|
|
|
|
|
|
|
|
this._decryptCache.set(cacheKey, decrypted, 5 * 60 * 1000) |
|
|
|
|
|
return decrypted |
|
|
} catch (oldError) { |
|
|
|
|
|
logger.warn('⚠️ Could not decrypt data, returning as-is:', oldError.message) |
|
|
return encryptedData |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error('❌ Decryption error:', error) |
|
|
return encryptedData |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
_generateEncryptionKey() { |
|
|
|
|
|
|
|
|
|
|
|
if (!this._encryptionKeyCache) { |
|
|
|
|
|
|
|
|
this._encryptionKeyCache = crypto.scryptSync( |
|
|
config.security.encryptionKey, |
|
|
this.ENCRYPTION_SALT, |
|
|
32 |
|
|
) |
|
|
logger.info('🔑 Encryption key derived and cached for performance optimization') |
|
|
} |
|
|
return this._encryptionKeyCache |
|
|
} |
|
|
|
|
|
|
|
|
_maskEmail(email) { |
|
|
if (!email || !email.includes('@')) { |
|
|
return email |
|
|
} |
|
|
|
|
|
const [username, domain] = email.split('@') |
|
|
const maskedUsername = |
|
|
username.length > 2 |
|
|
? `${username.slice(0, 2)}***${username.slice(-1)}` |
|
|
: `${username.slice(0, 1)}***` |
|
|
|
|
|
return `${maskedUsername}@${domain}` |
|
|
} |
|
|
|
|
|
|
|
|
_toNumberOrNull(value) { |
|
|
if (value === undefined || value === null || value === '') { |
|
|
return null |
|
|
} |
|
|
|
|
|
const num = Number(value) |
|
|
return Number.isFinite(num) ? num : null |
|
|
} |
|
|
|
|
|
|
|
|
async cleanupErrorAccounts() { |
|
|
try { |
|
|
const accounts = await redis.getAllClaudeAccounts() |
|
|
let cleanedCount = 0 |
|
|
|
|
|
for (const account of accounts) { |
|
|
if (account.status === 'error' && account.lastRefreshAt) { |
|
|
const lastRefresh = new Date(account.lastRefreshAt) |
|
|
const now = new Date() |
|
|
const hoursSinceLastRefresh = (now - lastRefresh) / (1000 * 60 * 60) |
|
|
|
|
|
|
|
|
if (hoursSinceLastRefresh > 24) { |
|
|
account.status = 'created' |
|
|
account.errorMessage = '' |
|
|
await redis.setClaudeAccount(account.id, account) |
|
|
cleanedCount++ |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
if (cleanedCount > 0) { |
|
|
logger.success(`🧹 Reset ${cleanedCount} error accounts`) |
|
|
} |
|
|
|
|
|
return cleanedCount |
|
|
} catch (error) { |
|
|
logger.error('❌ Failed to cleanup error accounts:', error) |
|
|
return 0 |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async markAccountRateLimited(accountId, sessionHash = null, rateLimitResetTimestamp = null) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
|
|
|
const updatedAccountData = { ...accountData } |
|
|
updatedAccountData.rateLimitedAt = new Date().toISOString() |
|
|
updatedAccountData.rateLimitStatus = 'limited' |
|
|
|
|
|
updatedAccountData.schedulable = 'false' |
|
|
|
|
|
updatedAccountData.rateLimitAutoStopped = 'true' |
|
|
|
|
|
|
|
|
if (rateLimitResetTimestamp) { |
|
|
|
|
|
const resetTime = new Date(rateLimitResetTimestamp * 1000) |
|
|
updatedAccountData.rateLimitEndAt = resetTime.toISOString() |
|
|
|
|
|
|
|
|
const windowStartTime = new Date(resetTime.getTime() - 5 * 60 * 60 * 1000) |
|
|
updatedAccountData.sessionWindowStart = windowStartTime.toISOString() |
|
|
updatedAccountData.sessionWindowEnd = resetTime.toISOString() |
|
|
|
|
|
const now = new Date() |
|
|
const minutesUntilEnd = Math.ceil((resetTime - now) / (1000 * 60)) |
|
|
logger.warn( |
|
|
`🚫 Account marked as rate limited with accurate reset time: ${accountData.name} (${accountId}) - ${minutesUntilEnd} minutes remaining until ${resetTime.toISOString()}` |
|
|
) |
|
|
} else { |
|
|
|
|
|
const windowData = await this.updateSessionWindow(accountId, updatedAccountData) |
|
|
Object.assign(updatedAccountData, windowData) |
|
|
|
|
|
|
|
|
if (updatedAccountData.sessionWindowEnd) { |
|
|
updatedAccountData.rateLimitEndAt = updatedAccountData.sessionWindowEnd |
|
|
const windowEnd = new Date(updatedAccountData.sessionWindowEnd) |
|
|
const now = new Date() |
|
|
const minutesUntilEnd = Math.ceil((windowEnd - now) / (1000 * 60)) |
|
|
logger.warn( |
|
|
`🚫 Account marked as rate limited until estimated session window ends: ${accountData.name} (${accountId}) - ${minutesUntilEnd} minutes remaining` |
|
|
) |
|
|
} else { |
|
|
|
|
|
const oneHourLater = new Date(Date.now() + 60 * 60 * 1000) |
|
|
updatedAccountData.rateLimitEndAt = oneHourLater.toISOString() |
|
|
logger.warn( |
|
|
`🚫 Account marked as rate limited (1 hour default): ${accountData.name} (${accountId})` |
|
|
) |
|
|
} |
|
|
} |
|
|
|
|
|
await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
|
|
|
|
|
|
if (sessionHash) { |
|
|
await redis.deleteSessionAccountMapping(sessionHash) |
|
|
logger.info(`🗑️ Deleted sticky session mapping for rate limited account: ${accountId}`) |
|
|
} |
|
|
|
|
|
|
|
|
try { |
|
|
const webhookNotifier = require('../utils/webhookNotifier') |
|
|
await webhookNotifier.sendAccountAnomalyNotification({ |
|
|
accountId, |
|
|
accountName: accountData.name || 'Claude Account', |
|
|
platform: 'claude-oauth', |
|
|
status: 'error', |
|
|
errorCode: 'CLAUDE_OAUTH_RATE_LIMITED', |
|
|
reason: `Account rate limited (429 error). ${rateLimitResetTimestamp ? `Reset at: ${formatDateWithTimezone(rateLimitResetTimestamp)}` : 'Estimated reset in 1-5 hours'}`, |
|
|
timestamp: getISOStringWithTimezone(new Date()) |
|
|
}) |
|
|
} catch (webhookError) { |
|
|
logger.error('Failed to send rate limit webhook notification:', webhookError) |
|
|
} |
|
|
|
|
|
return { success: true } |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to mark account as rate limited: ${accountId}`, error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async markAccountOpusRateLimited(accountId, rateLimitResetTimestamp = null) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
const updatedAccountData = { ...accountData } |
|
|
const now = new Date() |
|
|
updatedAccountData.opusRateLimitedAt = now.toISOString() |
|
|
|
|
|
if (rateLimitResetTimestamp) { |
|
|
const resetTime = new Date(rateLimitResetTimestamp * 1000) |
|
|
updatedAccountData.opusRateLimitEndAt = resetTime.toISOString() |
|
|
logger.warn( |
|
|
`🚫 Account ${accountData.name} (${accountId}) reached Opus weekly cap, resets at ${resetTime.toISOString()}` |
|
|
) |
|
|
} else { |
|
|
|
|
|
logger.warn( |
|
|
`⚠️ Account ${accountData.name} (${accountId}) reported Opus limit without reset timestamp` |
|
|
) |
|
|
} |
|
|
|
|
|
await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
return { success: true } |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to mark Opus rate limit for account: ${accountId}`, error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async clearAccountOpusRateLimit(accountId) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
return { success: true } |
|
|
} |
|
|
|
|
|
const updatedAccountData = { ...accountData } |
|
|
delete updatedAccountData.opusRateLimitedAt |
|
|
delete updatedAccountData.opusRateLimitEndAt |
|
|
|
|
|
await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
|
|
|
const redisKey = `claude:account:${accountId}` |
|
|
if (redis.client && typeof redis.client.hdel === 'function') { |
|
|
await redis.client.hdel(redisKey, 'opusRateLimitedAt', 'opusRateLimitEndAt') |
|
|
} |
|
|
|
|
|
logger.info(`✅ Cleared Opus rate limit state for account ${accountId}`) |
|
|
return { success: true } |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to clear Opus rate limit for account: ${accountId}`, error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async isAccountOpusRateLimited(accountId) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
return false |
|
|
} |
|
|
|
|
|
if (!accountData.opusRateLimitEndAt) { |
|
|
return false |
|
|
} |
|
|
|
|
|
const resetTime = new Date(accountData.opusRateLimitEndAt) |
|
|
if (Number.isNaN(resetTime.getTime())) { |
|
|
await this.clearAccountOpusRateLimit(accountId) |
|
|
return false |
|
|
} |
|
|
|
|
|
const now = new Date() |
|
|
if (now >= resetTime) { |
|
|
await this.clearAccountOpusRateLimit(accountId) |
|
|
return false |
|
|
} |
|
|
|
|
|
return true |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to check Opus rate limit status for account: ${accountId}`, error) |
|
|
return false |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async clearExpiredOpusRateLimit(accountId) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
return { success: true } |
|
|
} |
|
|
|
|
|
if (!accountData.opusRateLimitEndAt) { |
|
|
return { success: true } |
|
|
} |
|
|
|
|
|
const resetTime = new Date(accountData.opusRateLimitEndAt) |
|
|
if (Number.isNaN(resetTime.getTime()) || new Date() >= resetTime) { |
|
|
await this.clearAccountOpusRateLimit(accountId) |
|
|
} |
|
|
|
|
|
return { success: true } |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to clear expired Opus rate limit for account: ${accountId}`, error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async removeAccountRateLimit(accountId) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
const accountKey = `claude:account:${accountId}` |
|
|
|
|
|
|
|
|
const redisKey = `claude:account:${accountId}` |
|
|
await redis.client.hdel(redisKey, 'rateLimitedAt', 'rateLimitStatus', 'rateLimitEndAt') |
|
|
delete accountData.rateLimitedAt |
|
|
delete accountData.rateLimitStatus |
|
|
delete accountData.rateLimitEndAt |
|
|
|
|
|
const hadAutoStop = accountData.rateLimitAutoStopped === 'true' |
|
|
|
|
|
|
|
|
if (hadAutoStop && accountData.schedulable === 'false') { |
|
|
accountData.schedulable = 'true' |
|
|
logger.info(`✅ Auto-resuming scheduling for account ${accountId} after rate limit cleared`) |
|
|
logger.info( |
|
|
`📊 Account ${accountId} state after recovery: schedulable=${accountData.schedulable}` |
|
|
) |
|
|
} else { |
|
|
logger.info( |
|
|
`ℹ️ Account ${accountId} did not need auto-resume: autoStopped=${accountData.rateLimitAutoStopped}, schedulable=${accountData.schedulable}` |
|
|
) |
|
|
} |
|
|
|
|
|
if (hadAutoStop) { |
|
|
await redis.client.hdel(redisKey, 'rateLimitAutoStopped') |
|
|
delete accountData.rateLimitAutoStopped |
|
|
} |
|
|
await redis.setClaudeAccount(accountId, accountData) |
|
|
|
|
|
|
|
|
await redis.client.hdel( |
|
|
accountKey, |
|
|
'rateLimitedAt', |
|
|
'rateLimitStatus', |
|
|
'rateLimitEndAt', |
|
|
'rateLimitAutoStopped' |
|
|
) |
|
|
|
|
|
logger.success(`✅ Rate limit removed for account: ${accountData.name} (${accountId})`) |
|
|
|
|
|
return { success: true } |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to remove rate limit for account: ${accountId}`, error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async isAccountRateLimited(accountId) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
return false |
|
|
} |
|
|
|
|
|
const now = new Date() |
|
|
|
|
|
|
|
|
if ( |
|
|
(accountData.rateLimitStatus === 'limited' && accountData.rateLimitedAt) || |
|
|
(accountData.rateLimitAutoStopped === 'true' && accountData.rateLimitEndAt) |
|
|
) { |
|
|
|
|
|
if (accountData.rateLimitEndAt) { |
|
|
const rateLimitEndAt = new Date(accountData.rateLimitEndAt) |
|
|
|
|
|
|
|
|
if (now >= rateLimitEndAt) { |
|
|
await this.removeAccountRateLimit(accountId) |
|
|
return false |
|
|
} |
|
|
|
|
|
return true |
|
|
} else if (accountData.rateLimitedAt) { |
|
|
|
|
|
const rateLimitedAt = new Date(accountData.rateLimitedAt) |
|
|
const hoursSinceRateLimit = (now - rateLimitedAt) / (1000 * 60 * 60) |
|
|
|
|
|
|
|
|
if (hoursSinceRateLimit >= 1) { |
|
|
await this.removeAccountRateLimit(accountId) |
|
|
return false |
|
|
} |
|
|
|
|
|
return true |
|
|
} |
|
|
} |
|
|
|
|
|
return false |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to check rate limit status for account: ${accountId}`, error) |
|
|
return false |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async getAccountRateLimitInfo(accountId) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
return null |
|
|
} |
|
|
|
|
|
if (accountData.rateLimitStatus === 'limited' && accountData.rateLimitedAt) { |
|
|
const rateLimitedAt = new Date(accountData.rateLimitedAt) |
|
|
const now = new Date() |
|
|
const minutesSinceRateLimit = Math.floor((now - rateLimitedAt) / (1000 * 60)) |
|
|
|
|
|
let minutesRemaining |
|
|
let rateLimitEndAt |
|
|
|
|
|
|
|
|
if (accountData.rateLimitEndAt) { |
|
|
;({ rateLimitEndAt } = accountData) |
|
|
const endTime = new Date(accountData.rateLimitEndAt) |
|
|
minutesRemaining = Math.max(0, Math.ceil((endTime - now) / (1000 * 60))) |
|
|
} else { |
|
|
|
|
|
minutesRemaining = Math.max(0, 60 - minutesSinceRateLimit) |
|
|
|
|
|
const endTime = new Date(rateLimitedAt.getTime() + 60 * 60 * 1000) |
|
|
rateLimitEndAt = endTime.toISOString() |
|
|
} |
|
|
|
|
|
return { |
|
|
isRateLimited: minutesRemaining > 0, |
|
|
rateLimitedAt: accountData.rateLimitedAt, |
|
|
minutesSinceRateLimit, |
|
|
minutesRemaining, |
|
|
rateLimitEndAt |
|
|
} |
|
|
} |
|
|
|
|
|
return { |
|
|
isRateLimited: false, |
|
|
rateLimitedAt: null, |
|
|
minutesSinceRateLimit: 0, |
|
|
minutesRemaining: 0, |
|
|
rateLimitEndAt: null |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to get rate limit info for account: ${accountId}`, error) |
|
|
return null |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async updateSessionWindow(accountId, accountData = null) { |
|
|
try { |
|
|
|
|
|
if (!accountData) { |
|
|
accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
} |
|
|
|
|
|
const now = new Date() |
|
|
const currentTime = now.getTime() |
|
|
|
|
|
let shouldClearSessionStatus = false |
|
|
let shouldClearFiveHourFlags = false |
|
|
|
|
|
|
|
|
if (accountData.sessionWindowStart && accountData.sessionWindowEnd) { |
|
|
const windowEnd = new Date(accountData.sessionWindowEnd).getTime() |
|
|
|
|
|
|
|
|
if (currentTime < windowEnd) { |
|
|
accountData.lastRequestTime = now.toISOString() |
|
|
return accountData |
|
|
} |
|
|
|
|
|
|
|
|
const windowStart = new Date(accountData.sessionWindowStart) |
|
|
logger.info( |
|
|
`⏰ Session window expired for account ${accountData.name} (${accountId}): ${windowStart.toISOString()} - ${new Date(windowEnd).toISOString()}` |
|
|
) |
|
|
} |
|
|
|
|
|
|
|
|
const windowStart = this._calculateSessionWindowStart(now) |
|
|
const windowEnd = this._calculateSessionWindowEnd(windowStart) |
|
|
|
|
|
|
|
|
accountData.sessionWindowStart = windowStart.toISOString() |
|
|
accountData.sessionWindowEnd = windowEnd.toISOString() |
|
|
accountData.lastRequestTime = now.toISOString() |
|
|
|
|
|
|
|
|
if (accountData.sessionWindowStatus) { |
|
|
delete accountData.sessionWindowStatus |
|
|
delete accountData.sessionWindowStatusUpdatedAt |
|
|
await this._clearFiveHourWarningMetadata(accountId, accountData) |
|
|
shouldClearSessionStatus = true |
|
|
} |
|
|
|
|
|
|
|
|
if (accountData.fiveHourAutoStopped === 'true' && accountData.schedulable === 'false') { |
|
|
logger.info( |
|
|
`✅ Auto-resuming scheduling for account ${accountData.name} (${accountId}) - new session window started` |
|
|
) |
|
|
accountData.schedulable = 'true' |
|
|
delete accountData.fiveHourAutoStopped |
|
|
delete accountData.fiveHourStoppedAt |
|
|
await this._clearFiveHourWarningMetadata(accountId, accountData) |
|
|
shouldClearFiveHourFlags = true |
|
|
|
|
|
|
|
|
try { |
|
|
const webhookNotifier = require('../utils/webhookNotifier') |
|
|
await webhookNotifier.sendAccountAnomalyNotification({ |
|
|
accountId, |
|
|
accountName: accountData.name || 'Claude Account', |
|
|
platform: 'claude', |
|
|
status: 'resumed', |
|
|
errorCode: 'CLAUDE_5H_LIMIT_RESUMED', |
|
|
reason: '进入新的5小时窗口,已自动恢复调度', |
|
|
timestamp: getISOStringWithTimezone(new Date()) |
|
|
}) |
|
|
} catch (webhookError) { |
|
|
logger.error('Failed to send webhook notification:', webhookError) |
|
|
} |
|
|
} |
|
|
|
|
|
if (shouldClearSessionStatus || shouldClearFiveHourFlags) { |
|
|
const fieldsToRemove = [] |
|
|
if (shouldClearFiveHourFlags) { |
|
|
fieldsToRemove.push('fiveHourAutoStopped', 'fiveHourStoppedAt') |
|
|
} |
|
|
if (shouldClearSessionStatus) { |
|
|
fieldsToRemove.push('sessionWindowStatus', 'sessionWindowStatusUpdatedAt') |
|
|
} |
|
|
await this._removeAccountFields(accountId, fieldsToRemove, 'session_window_refresh') |
|
|
} |
|
|
|
|
|
logger.info( |
|
|
`🕐 Created new session window for account ${accountData.name} (${accountId}): ${windowStart.toISOString()} - ${windowEnd.toISOString()} (from current time)` |
|
|
) |
|
|
|
|
|
return accountData |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to update session window for account ${accountId}:`, error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
_calculateSessionWindowStart(requestTime) { |
|
|
|
|
|
const windowStart = new Date(requestTime) |
|
|
windowStart.setMinutes(0) |
|
|
windowStart.setSeconds(0) |
|
|
windowStart.setMilliseconds(0) |
|
|
|
|
|
return windowStart |
|
|
} |
|
|
|
|
|
|
|
|
_calculateSessionWindowEnd(startTime) { |
|
|
const endTime = new Date(startTime) |
|
|
endTime.setHours(endTime.getHours() + 5) |
|
|
return endTime |
|
|
} |
|
|
|
|
|
async _clearFiveHourWarningMetadata(accountId, accountData = null) { |
|
|
if (accountData) { |
|
|
delete accountData.fiveHourWarningWindow |
|
|
delete accountData.fiveHourWarningCount |
|
|
delete accountData.fiveHourWarningLastSentAt |
|
|
} |
|
|
|
|
|
try { |
|
|
if (redis.client && typeof redis.client.hdel === 'function') { |
|
|
await redis.client.hdel( |
|
|
`claude:account:${accountId}`, |
|
|
'fiveHourWarningWindow', |
|
|
'fiveHourWarningCount', |
|
|
'fiveHourWarningLastSentAt' |
|
|
) |
|
|
} |
|
|
} catch (error) { |
|
|
logger.warn( |
|
|
`⚠️ Failed to clear five-hour warning metadata for account ${accountId}: ${error.message}` |
|
|
) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async getSessionWindowInfo(accountId) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
return null |
|
|
} |
|
|
|
|
|
|
|
|
if (!accountData.sessionWindowStart || !accountData.sessionWindowEnd) { |
|
|
return { |
|
|
hasActiveWindow: false, |
|
|
windowStart: null, |
|
|
windowEnd: null, |
|
|
progress: 0, |
|
|
remainingTime: null, |
|
|
lastRequestTime: accountData.lastRequestTime || null, |
|
|
sessionWindowStatus: accountData.sessionWindowStatus || null |
|
|
} |
|
|
} |
|
|
|
|
|
const now = new Date() |
|
|
const windowStart = new Date(accountData.sessionWindowStart) |
|
|
const windowEnd = new Date(accountData.sessionWindowEnd) |
|
|
const currentTime = now.getTime() |
|
|
|
|
|
|
|
|
if (currentTime >= windowEnd.getTime()) { |
|
|
return { |
|
|
hasActiveWindow: false, |
|
|
windowStart: accountData.sessionWindowStart, |
|
|
windowEnd: accountData.sessionWindowEnd, |
|
|
progress: 100, |
|
|
remainingTime: 0, |
|
|
lastRequestTime: accountData.lastRequestTime || null, |
|
|
sessionWindowStatus: accountData.sessionWindowStatus || null |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
const totalDuration = windowEnd.getTime() - windowStart.getTime() |
|
|
const elapsedTime = currentTime - windowStart.getTime() |
|
|
const progress = Math.round((elapsedTime / totalDuration) * 100) |
|
|
|
|
|
|
|
|
const remainingTime = Math.round((windowEnd.getTime() - currentTime) / (1000 * 60)) |
|
|
|
|
|
return { |
|
|
hasActiveWindow: true, |
|
|
windowStart: accountData.sessionWindowStart, |
|
|
windowEnd: accountData.sessionWindowEnd, |
|
|
progress, |
|
|
remainingTime, |
|
|
lastRequestTime: accountData.lastRequestTime || null, |
|
|
sessionWindowStatus: accountData.sessionWindowStatus || null |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to get session window info for account ${accountId}:`, error) |
|
|
return null |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async fetchOAuthUsage(accountId, accessToken = null, agent = null) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
|
|
|
if (!accessToken) { |
|
|
accessToken = await this.getValidAccessToken(accountId) |
|
|
} |
|
|
|
|
|
|
|
|
if (!agent) { |
|
|
agent = this._createProxyAgent(accountData.proxy) |
|
|
} |
|
|
|
|
|
logger.debug(`📊 Fetching OAuth usage for account: ${accountData.name} (${accountId})`) |
|
|
|
|
|
|
|
|
const axiosConfig = { |
|
|
headers: { |
|
|
Authorization: `Bearer ${accessToken}`, |
|
|
'Content-Type': 'application/json', |
|
|
Accept: 'application/json', |
|
|
'anthropic-beta': 'oauth-2025-04-20', |
|
|
'User-Agent': 'claude-cli/1.0.56 (external, cli)', |
|
|
'Accept-Language': 'en-US,en;q=0.9' |
|
|
}, |
|
|
timeout: 15000 |
|
|
} |
|
|
|
|
|
if (agent) { |
|
|
axiosConfig.httpAgent = agent |
|
|
axiosConfig.httpsAgent = agent |
|
|
axiosConfig.proxy = false |
|
|
} |
|
|
|
|
|
const response = await axios.get('https://api.anthropic.com/api/oauth/usage', axiosConfig) |
|
|
|
|
|
if (response.status === 200 && response.data) { |
|
|
logger.debug('✅ Successfully fetched OAuth usage data:', { |
|
|
accountId, |
|
|
fiveHour: response.data.five_hour?.utilization, |
|
|
sevenDay: response.data.seven_day?.utilization, |
|
|
sevenDayOpus: response.data.seven_day_opus?.utilization |
|
|
}) |
|
|
|
|
|
return response.data |
|
|
} |
|
|
|
|
|
logger.warn(`⚠️ Failed to fetch OAuth usage for account ${accountId}: ${response.status}`) |
|
|
return null |
|
|
} catch (error) { |
|
|
|
|
|
if (error.response?.status === 403) { |
|
|
logger.debug( |
|
|
`⚠️ OAuth usage API returned 403 for account ${accountId}. This account likely uses Setup Token instead of OAuth.` |
|
|
) |
|
|
return null |
|
|
} |
|
|
|
|
|
|
|
|
logger.error( |
|
|
`❌ Failed to fetch OAuth usage for account ${accountId}:`, |
|
|
error.response?.data || error.message |
|
|
) |
|
|
return null |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
buildClaudeUsageSnapshot(accountData) { |
|
|
const updatedAt = accountData.claudeUsageUpdatedAt |
|
|
|
|
|
const fiveHourUtilization = this._toNumberOrNull(accountData.claudeFiveHourUtilization) |
|
|
const fiveHourResetsAt = accountData.claudeFiveHourResetsAt |
|
|
const sevenDayUtilization = this._toNumberOrNull(accountData.claudeSevenDayUtilization) |
|
|
const sevenDayResetsAt = accountData.claudeSevenDayResetsAt |
|
|
const sevenDayOpusUtilization = this._toNumberOrNull(accountData.claudeSevenDayOpusUtilization) |
|
|
const sevenDayOpusResetsAt = accountData.claudeSevenDayOpusResetsAt |
|
|
|
|
|
const hasFiveHourData = fiveHourUtilization !== null || fiveHourResetsAt |
|
|
const hasSevenDayData = sevenDayUtilization !== null || sevenDayResetsAt |
|
|
const hasSevenDayOpusData = sevenDayOpusUtilization !== null || sevenDayOpusResetsAt |
|
|
|
|
|
if (!updatedAt && !hasFiveHourData && !hasSevenDayData && !hasSevenDayOpusData) { |
|
|
return null |
|
|
} |
|
|
|
|
|
const now = Date.now() |
|
|
|
|
|
return { |
|
|
updatedAt, |
|
|
fiveHour: { |
|
|
utilization: fiveHourUtilization, |
|
|
resetsAt: fiveHourResetsAt, |
|
|
remainingSeconds: fiveHourResetsAt |
|
|
? Math.max(0, Math.floor((new Date(fiveHourResetsAt).getTime() - now) / 1000)) |
|
|
: null |
|
|
}, |
|
|
sevenDay: { |
|
|
utilization: sevenDayUtilization, |
|
|
resetsAt: sevenDayResetsAt, |
|
|
remainingSeconds: sevenDayResetsAt |
|
|
? Math.max(0, Math.floor((new Date(sevenDayResetsAt).getTime() - now) / 1000)) |
|
|
: null |
|
|
}, |
|
|
sevenDayOpus: { |
|
|
utilization: sevenDayOpusUtilization, |
|
|
resetsAt: sevenDayOpusResetsAt, |
|
|
remainingSeconds: sevenDayOpusResetsAt |
|
|
? Math.max(0, Math.floor((new Date(sevenDayOpusResetsAt).getTime() - now) / 1000)) |
|
|
: null |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async updateClaudeUsageSnapshot(accountId, usageData) { |
|
|
if (!usageData || typeof usageData !== 'object') { |
|
|
return |
|
|
} |
|
|
|
|
|
const updates = {} |
|
|
|
|
|
|
|
|
if (usageData.five_hour) { |
|
|
if (usageData.five_hour.utilization !== undefined) { |
|
|
updates.claudeFiveHourUtilization = String(usageData.five_hour.utilization) |
|
|
} |
|
|
if (usageData.five_hour.resets_at) { |
|
|
updates.claudeFiveHourResetsAt = usageData.five_hour.resets_at |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
if (usageData.seven_day) { |
|
|
if (usageData.seven_day.utilization !== undefined) { |
|
|
updates.claudeSevenDayUtilization = String(usageData.seven_day.utilization) |
|
|
} |
|
|
if (usageData.seven_day.resets_at) { |
|
|
updates.claudeSevenDayResetsAt = usageData.seven_day.resets_at |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
if (usageData.seven_day_opus) { |
|
|
if (usageData.seven_day_opus.utilization !== undefined) { |
|
|
updates.claudeSevenDayOpusUtilization = String(usageData.seven_day_opus.utilization) |
|
|
} |
|
|
if (usageData.seven_day_opus.resets_at) { |
|
|
updates.claudeSevenDayOpusResetsAt = usageData.seven_day_opus.resets_at |
|
|
} |
|
|
} |
|
|
|
|
|
if (Object.keys(updates).length === 0) { |
|
|
return |
|
|
} |
|
|
|
|
|
updates.claudeUsageUpdatedAt = new Date().toISOString() |
|
|
|
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (accountData && Object.keys(accountData).length > 0) { |
|
|
Object.assign(accountData, updates) |
|
|
await redis.setClaudeAccount(accountId, accountData) |
|
|
logger.debug( |
|
|
`📊 Updated Claude usage snapshot for account ${accountId}:`, |
|
|
Object.keys(updates) |
|
|
) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async fetchAndUpdateAccountProfile(accountId, accessToken = null, agent = null) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
|
|
|
const hasProfileScope = accountData.scopes && accountData.scopes.includes('user:profile') |
|
|
if (!hasProfileScope) { |
|
|
logger.warn( |
|
|
`⚠️ Account ${accountId} does not have user:profile scope, cannot fetch profile` |
|
|
) |
|
|
throw new Error('Account does not have user:profile permission') |
|
|
} |
|
|
|
|
|
|
|
|
if (!accessToken) { |
|
|
accessToken = this._decryptSensitiveData(accountData.accessToken) |
|
|
if (!accessToken) { |
|
|
throw new Error('No access token available') |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
if (!agent) { |
|
|
agent = this._createProxyAgent(accountData.proxy) |
|
|
} |
|
|
|
|
|
logger.info(`📊 Fetching profile info for account: ${accountData.name} (${accountId})`) |
|
|
|
|
|
|
|
|
const axiosConfig = { |
|
|
headers: { |
|
|
Authorization: `Bearer ${accessToken}`, |
|
|
'Content-Type': 'application/json', |
|
|
Accept: 'application/json', |
|
|
'User-Agent': 'claude-cli/1.0.56 (external, cli)', |
|
|
'Accept-Language': 'en-US,en;q=0.9' |
|
|
}, |
|
|
timeout: 15000 |
|
|
} |
|
|
|
|
|
if (agent) { |
|
|
axiosConfig.httpAgent = agent |
|
|
axiosConfig.httpsAgent = agent |
|
|
axiosConfig.proxy = false |
|
|
} |
|
|
|
|
|
const response = await axios.get('https://api.anthropic.com/api/oauth/profile', axiosConfig) |
|
|
|
|
|
if (response.status === 200 && response.data) { |
|
|
const profileData = response.data |
|
|
|
|
|
logger.info('✅ Successfully fetched profile data:', { |
|
|
email: profileData.account?.email, |
|
|
hasClaudeMax: profileData.account?.has_claude_max, |
|
|
hasClaudePro: profileData.account?.has_claude_pro, |
|
|
organizationType: profileData.organization?.organization_type |
|
|
}) |
|
|
|
|
|
|
|
|
const subscriptionInfo = { |
|
|
|
|
|
email: profileData.account?.email, |
|
|
fullName: profileData.account?.full_name, |
|
|
displayName: profileData.account?.display_name, |
|
|
hasClaudeMax: profileData.account?.has_claude_max || false, |
|
|
hasClaudePro: profileData.account?.has_claude_pro || false, |
|
|
accountUuid: profileData.account?.uuid, |
|
|
|
|
|
|
|
|
organizationName: profileData.organization?.name, |
|
|
organizationUuid: profileData.organization?.uuid, |
|
|
billingType: profileData.organization?.billing_type, |
|
|
rateLimitTier: profileData.organization?.rate_limit_tier, |
|
|
organizationType: profileData.organization?.organization_type, |
|
|
|
|
|
|
|
|
accountType: |
|
|
profileData.account?.has_claude_max === true |
|
|
? 'claude_max' |
|
|
: profileData.account?.has_claude_pro === true |
|
|
? 'claude_pro' |
|
|
: 'free', |
|
|
|
|
|
|
|
|
profileFetchedAt: new Date().toISOString() |
|
|
} |
|
|
|
|
|
|
|
|
accountData.subscriptionInfo = JSON.stringify(subscriptionInfo) |
|
|
accountData.profileUpdatedAt = new Date().toISOString() |
|
|
|
|
|
|
|
|
if (profileData.account?.email) { |
|
|
accountData.email = this._encryptSensitiveData(profileData.account.email) |
|
|
} |
|
|
|
|
|
await redis.setClaudeAccount(accountId, accountData) |
|
|
|
|
|
logger.success( |
|
|
`✅ Updated account profile for ${accountData.name} (${accountId}) - Type: ${subscriptionInfo.accountType}` |
|
|
) |
|
|
|
|
|
return subscriptionInfo |
|
|
} else { |
|
|
throw new Error(`Failed to fetch profile with status: ${response.status}`) |
|
|
} |
|
|
} catch (error) { |
|
|
if (error.response?.status === 401) { |
|
|
logger.warn(`⚠️ Profile API returned 401 for account ${accountId} - token may be invalid`) |
|
|
} else if (error.response?.status === 403) { |
|
|
logger.warn( |
|
|
`⚠️ Profile API returned 403 for account ${accountId} - insufficient permissions` |
|
|
) |
|
|
} else { |
|
|
logger.error(`❌ Failed to fetch profile for account ${accountId}:`, error.message) |
|
|
} |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async updateAllAccountProfiles() { |
|
|
try { |
|
|
logger.info('🔄 Starting batch profile update for all accounts...') |
|
|
|
|
|
const accounts = await redis.getAllClaudeAccounts() |
|
|
let successCount = 0 |
|
|
let failureCount = 0 |
|
|
const results = [] |
|
|
|
|
|
for (const account of accounts) { |
|
|
|
|
|
if (account.isActive !== 'true' || account.status === 'error') { |
|
|
logger.info(`⏩ Skipping inactive/error account: ${account.name} (${account.id})`) |
|
|
continue |
|
|
} |
|
|
|
|
|
|
|
|
const hasProfileScope = account.scopes && account.scopes.includes('user:profile') |
|
|
if (!hasProfileScope) { |
|
|
logger.info( |
|
|
`⏩ Skipping account without user:profile scope: ${account.name} (${account.id})` |
|
|
) |
|
|
results.push({ |
|
|
accountId: account.id, |
|
|
accountName: account.name, |
|
|
success: false, |
|
|
error: 'No user:profile permission (Setup Token account)' |
|
|
}) |
|
|
continue |
|
|
} |
|
|
|
|
|
try { |
|
|
|
|
|
const accessToken = await this.getValidAccessToken(account.id) |
|
|
if (accessToken) { |
|
|
const profileInfo = await this.fetchAndUpdateAccountProfile(account.id, accessToken) |
|
|
successCount++ |
|
|
results.push({ |
|
|
accountId: account.id, |
|
|
accountName: account.name, |
|
|
success: true, |
|
|
accountType: profileInfo.accountType |
|
|
}) |
|
|
} |
|
|
} catch (error) { |
|
|
failureCount++ |
|
|
results.push({ |
|
|
accountId: account.id, |
|
|
accountName: account.name, |
|
|
success: false, |
|
|
error: error.message |
|
|
}) |
|
|
logger.warn( |
|
|
`⚠️ Failed to update profile for account ${account.name} (${account.id}): ${error.message}` |
|
|
) |
|
|
} |
|
|
|
|
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, 1000)) |
|
|
} |
|
|
|
|
|
logger.success(`✅ Profile update completed: ${successCount} success, ${failureCount} failed`) |
|
|
|
|
|
return { |
|
|
totalAccounts: accounts.length, |
|
|
successCount, |
|
|
failureCount, |
|
|
results |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error('❌ Failed to update account profiles:', error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async initializeSessionWindows(forceRecalculate = false) { |
|
|
try { |
|
|
logger.info('🔄 Initializing session windows for all Claude accounts...') |
|
|
|
|
|
const accounts = await redis.getAllClaudeAccounts() |
|
|
let validWindowCount = 0 |
|
|
let expiredWindowCount = 0 |
|
|
let noWindowCount = 0 |
|
|
const now = new Date() |
|
|
|
|
|
for (const account of accounts) { |
|
|
|
|
|
if (forceRecalculate && (account.sessionWindowStart || account.sessionWindowEnd)) { |
|
|
logger.info(`🔄 Force recalculating window for account ${account.name} (${account.id})`) |
|
|
delete account.sessionWindowStart |
|
|
delete account.sessionWindowEnd |
|
|
delete account.lastRequestTime |
|
|
await redis.setClaudeAccount(account.id, account) |
|
|
} |
|
|
|
|
|
|
|
|
if (account.sessionWindowStart && account.sessionWindowEnd) { |
|
|
const windowEnd = new Date(account.sessionWindowEnd) |
|
|
const windowStart = new Date(account.sessionWindowStart) |
|
|
const timeUntilExpires = Math.round((windowEnd.getTime() - now.getTime()) / (1000 * 60)) |
|
|
|
|
|
if (now.getTime() < windowEnd.getTime()) { |
|
|
|
|
|
validWindowCount++ |
|
|
logger.info( |
|
|
`✅ Account ${account.name} (${account.id}) has valid window: ${windowStart.toISOString()} - ${windowEnd.toISOString()} (${timeUntilExpires} minutes remaining)` |
|
|
) |
|
|
} else { |
|
|
|
|
|
expiredWindowCount++ |
|
|
logger.warn( |
|
|
`⏰ Account ${account.name} (${account.id}) window expired: ${windowStart.toISOString()} - ${windowEnd.toISOString()}` |
|
|
) |
|
|
|
|
|
|
|
|
delete account.sessionWindowStart |
|
|
delete account.sessionWindowEnd |
|
|
delete account.lastRequestTime |
|
|
await redis.setClaudeAccount(account.id, account) |
|
|
} |
|
|
} else { |
|
|
noWindowCount++ |
|
|
logger.info( |
|
|
`📭 Account ${account.name} (${account.id}) has no session window - will create on next request` |
|
|
) |
|
|
} |
|
|
} |
|
|
|
|
|
logger.success('✅ Session window initialization completed:') |
|
|
logger.success(` 📊 Total accounts: ${accounts.length}`) |
|
|
logger.success(` ✅ Valid windows: ${validWindowCount}`) |
|
|
logger.success(` ⏰ Expired windows: ${expiredWindowCount}`) |
|
|
logger.success(` 📭 No windows: ${noWindowCount}`) |
|
|
|
|
|
return { |
|
|
total: accounts.length, |
|
|
validWindows: validWindowCount, |
|
|
expiredWindows: expiredWindowCount, |
|
|
noWindows: noWindowCount |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error('❌ Failed to initialize session windows:', error) |
|
|
return { |
|
|
total: 0, |
|
|
validWindows: 0, |
|
|
expiredWindows: 0, |
|
|
noWindows: 0, |
|
|
error: error.message |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async markAccountError(accountId, errorType, sessionHash = null) { |
|
|
const ERROR_CONFIG = { |
|
|
unauthorized: { |
|
|
status: 'unauthorized', |
|
|
errorMessage: 'Account unauthorized (401 errors detected)', |
|
|
timestampField: 'unauthorizedAt', |
|
|
errorCode: 'CLAUDE_OAUTH_UNAUTHORIZED', |
|
|
logMessage: 'unauthorized' |
|
|
}, |
|
|
blocked: { |
|
|
status: 'blocked', |
|
|
errorMessage: 'Account blocked (403 error detected - account may be suspended by Claude)', |
|
|
timestampField: 'blockedAt', |
|
|
errorCode: 'CLAUDE_OAUTH_BLOCKED', |
|
|
logMessage: 'blocked' |
|
|
} |
|
|
} |
|
|
|
|
|
try { |
|
|
const errorConfig = ERROR_CONFIG[errorType] |
|
|
if (!errorConfig) { |
|
|
throw new Error(`Unsupported error type: ${errorType}`) |
|
|
} |
|
|
|
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
|
|
|
const updatedAccountData = { ...accountData } |
|
|
updatedAccountData.status = errorConfig.status |
|
|
updatedAccountData.schedulable = 'false' |
|
|
updatedAccountData.errorMessage = errorConfig.errorMessage |
|
|
updatedAccountData[errorConfig.timestampField] = new Date().toISOString() |
|
|
|
|
|
|
|
|
await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
|
|
|
|
|
|
if (sessionHash) { |
|
|
await redis.client.del(`sticky_session:${sessionHash}`) |
|
|
logger.info(`🗑️ Deleted sticky session mapping for hash: ${sessionHash}`) |
|
|
} |
|
|
|
|
|
logger.warn( |
|
|
`⚠️ Account ${accountData.name} (${accountId}) marked as ${errorConfig.logMessage} and disabled for scheduling` |
|
|
) |
|
|
|
|
|
|
|
|
try { |
|
|
const webhookNotifier = require('../utils/webhookNotifier') |
|
|
await webhookNotifier.sendAccountAnomalyNotification({ |
|
|
accountId, |
|
|
accountName: accountData.name, |
|
|
platform: 'claude-oauth', |
|
|
status: errorConfig.status, |
|
|
errorCode: errorConfig.errorCode, |
|
|
reason: errorConfig.errorMessage, |
|
|
timestamp: getISOStringWithTimezone(new Date()) |
|
|
}) |
|
|
} catch (webhookError) { |
|
|
logger.error('Failed to send webhook notification:', webhookError) |
|
|
} |
|
|
|
|
|
return { success: true } |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to mark account ${accountId} as ${errorType}:`, error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async markAccountUnauthorized(accountId, sessionHash = null) { |
|
|
return this.markAccountError(accountId, 'unauthorized', sessionHash) |
|
|
} |
|
|
|
|
|
|
|
|
async markAccountBlocked(accountId, sessionHash = null) { |
|
|
return this.markAccountError(accountId, 'blocked', sessionHash) |
|
|
} |
|
|
|
|
|
|
|
|
async resetAccountStatus(accountId) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
|
|
|
const updatedAccountData = { ...accountData } |
|
|
|
|
|
|
|
|
if (updatedAccountData.accessToken) { |
|
|
updatedAccountData.status = 'active' |
|
|
} else { |
|
|
updatedAccountData.status = 'created' |
|
|
} |
|
|
|
|
|
|
|
|
updatedAccountData.schedulable = 'true' |
|
|
|
|
|
delete updatedAccountData.rateLimitAutoStopped |
|
|
delete updatedAccountData.fiveHourAutoStopped |
|
|
delete updatedAccountData.fiveHourStoppedAt |
|
|
delete updatedAccountData.tempErrorAutoStopped |
|
|
delete updatedAccountData.fiveHourWarningWindow |
|
|
delete updatedAccountData.fiveHourWarningCount |
|
|
delete updatedAccountData.fiveHourWarningLastSentAt |
|
|
|
|
|
delete updatedAccountData.autoStoppedAt |
|
|
delete updatedAccountData.stoppedReason |
|
|
|
|
|
|
|
|
delete updatedAccountData.errorMessage |
|
|
delete updatedAccountData.unauthorizedAt |
|
|
delete updatedAccountData.blockedAt |
|
|
delete updatedAccountData.rateLimitedAt |
|
|
delete updatedAccountData.rateLimitStatus |
|
|
delete updatedAccountData.rateLimitEndAt |
|
|
delete updatedAccountData.tempErrorAt |
|
|
delete updatedAccountData.sessionWindowStart |
|
|
delete updatedAccountData.sessionWindowEnd |
|
|
|
|
|
|
|
|
await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
|
|
|
|
|
|
const fieldsToDelete = [ |
|
|
'errorMessage', |
|
|
'unauthorizedAt', |
|
|
'blockedAt', |
|
|
'rateLimitedAt', |
|
|
'rateLimitStatus', |
|
|
'rateLimitEndAt', |
|
|
'tempErrorAt', |
|
|
'sessionWindowStart', |
|
|
'sessionWindowEnd', |
|
|
|
|
|
'rateLimitAutoStopped', |
|
|
'fiveHourAutoStopped', |
|
|
'fiveHourStoppedAt', |
|
|
'fiveHourWarningWindow', |
|
|
'fiveHourWarningCount', |
|
|
'fiveHourWarningLastSentAt', |
|
|
'tempErrorAutoStopped', |
|
|
|
|
|
'autoStoppedAt', |
|
|
'stoppedReason' |
|
|
] |
|
|
await redis.client.hdel(`claude:account:${accountId}`, ...fieldsToDelete) |
|
|
|
|
|
|
|
|
const errorKey = `claude_account:${accountId}:401_errors` |
|
|
await redis.client.del(errorKey) |
|
|
|
|
|
|
|
|
const rateLimitKey = `ratelimit:${accountId}` |
|
|
await redis.client.del(rateLimitKey) |
|
|
|
|
|
|
|
|
const serverErrorKey = `claude_account:${accountId}:5xx_errors` |
|
|
await redis.client.del(serverErrorKey) |
|
|
|
|
|
logger.info( |
|
|
`✅ Successfully reset all error states for account ${accountData.name} (${accountId})` |
|
|
) |
|
|
|
|
|
return { |
|
|
success: true, |
|
|
account: { |
|
|
id: accountId, |
|
|
name: accountData.name, |
|
|
status: updatedAccountData.status, |
|
|
schedulable: updatedAccountData.schedulable === 'true' |
|
|
} |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to reset account status for ${accountId}:`, error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async cleanupTempErrorAccounts() { |
|
|
try { |
|
|
const accounts = await redis.getAllClaudeAccounts() |
|
|
let cleanedCount = 0 |
|
|
const TEMP_ERROR_RECOVERY_MINUTES = 5 |
|
|
|
|
|
for (const account of accounts) { |
|
|
if (account.status === 'temp_error' && account.tempErrorAt) { |
|
|
const tempErrorAt = new Date(account.tempErrorAt) |
|
|
const now = new Date() |
|
|
const minutesSinceTempError = (now - tempErrorAt) / (1000 * 60) |
|
|
|
|
|
|
|
|
if (minutesSinceTempError > TEMP_ERROR_RECOVERY_MINUTES) { |
|
|
account.status = 'active' |
|
|
|
|
|
if (account.tempErrorAutoStopped === 'true') { |
|
|
account.schedulable = 'true' |
|
|
delete account.tempErrorAutoStopped |
|
|
} |
|
|
delete account.errorMessage |
|
|
delete account.tempErrorAt |
|
|
await redis.setClaudeAccount(account.id, account) |
|
|
|
|
|
|
|
|
await redis.client.hdel( |
|
|
`claude:account:${account.id}`, |
|
|
'errorMessage', |
|
|
'tempErrorAt', |
|
|
'tempErrorAutoStopped' |
|
|
) |
|
|
|
|
|
|
|
|
await this.clearInternalErrors(account.id) |
|
|
cleanedCount++ |
|
|
logger.success(`🧹 Reset temp_error status for account ${account.name} (${account.id})`) |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
if (cleanedCount > 0) { |
|
|
logger.success(`🧹 Reset ${cleanedCount} temp_error accounts`) |
|
|
} |
|
|
|
|
|
return cleanedCount |
|
|
} catch (error) { |
|
|
logger.error('❌ Failed to cleanup temp_error accounts:', error) |
|
|
return 0 |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async recordServerError(accountId, statusCode) { |
|
|
try { |
|
|
const key = `claude_account:${accountId}:5xx_errors` |
|
|
|
|
|
|
|
|
await redis.client.incr(key) |
|
|
await redis.client.expire(key, 300) |
|
|
|
|
|
logger.info(`📝 Recorded ${statusCode} error for account ${accountId}`) |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to record ${statusCode} error for account ${accountId}:`, error) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async recordInternalError(accountId) { |
|
|
return this.recordServerError(accountId, 500) |
|
|
} |
|
|
|
|
|
|
|
|
async getServerErrorCount(accountId) { |
|
|
try { |
|
|
const key = `claude_account:${accountId}:5xx_errors` |
|
|
|
|
|
const count = await redis.client.get(key) |
|
|
return parseInt(count) || 0 |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to get 5xx error count for account ${accountId}:`, error) |
|
|
return 0 |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async getInternalErrorCount(accountId) { |
|
|
return this.getServerErrorCount(accountId) |
|
|
} |
|
|
|
|
|
|
|
|
async clearInternalErrors(accountId) { |
|
|
try { |
|
|
const key = `claude_account:${accountId}:5xx_errors` |
|
|
|
|
|
await redis.client.del(key) |
|
|
logger.info(`✅ Cleared 5xx error count for account ${accountId}`) |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to clear 5xx errors for account ${accountId}:`, error) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async markAccountTempError(accountId, sessionHash = null) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
|
|
|
const updatedAccountData = { ...accountData } |
|
|
updatedAccountData.status = 'temp_error' |
|
|
updatedAccountData.schedulable = 'false' |
|
|
updatedAccountData.errorMessage = 'Account temporarily disabled due to consecutive 500 errors' |
|
|
updatedAccountData.tempErrorAt = new Date().toISOString() |
|
|
|
|
|
updatedAccountData.tempErrorAutoStopped = 'true' |
|
|
|
|
|
|
|
|
await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
|
|
|
|
|
|
setTimeout( |
|
|
async () => { |
|
|
try { |
|
|
const account = await redis.getClaudeAccount(accountId) |
|
|
if (account && account.status === 'temp_error' && account.tempErrorAt) { |
|
|
|
|
|
const tempErrorAt = new Date(account.tempErrorAt) |
|
|
const now = new Date() |
|
|
const minutesSince = (now - tempErrorAt) / (1000 * 60) |
|
|
|
|
|
if (minutesSince >= 5) { |
|
|
|
|
|
account.status = 'active' |
|
|
|
|
|
if (account.tempErrorAutoStopped === 'true') { |
|
|
account.schedulable = 'true' |
|
|
delete account.tempErrorAutoStopped |
|
|
} |
|
|
delete account.errorMessage |
|
|
delete account.tempErrorAt |
|
|
|
|
|
await redis.setClaudeAccount(accountId, account) |
|
|
|
|
|
|
|
|
await redis.client.hdel( |
|
|
`claude:account:${accountId}`, |
|
|
'errorMessage', |
|
|
'tempErrorAt', |
|
|
'tempErrorAutoStopped' |
|
|
) |
|
|
|
|
|
|
|
|
await this.clearInternalErrors(accountId) |
|
|
|
|
|
logger.success( |
|
|
`✅ Auto-recovered temp_error after 5 minutes: ${account.name} (${accountId})` |
|
|
) |
|
|
} else { |
|
|
logger.debug( |
|
|
`⏰ Temp error timer triggered but only ${minutesSince.toFixed(1)} minutes passed for ${account.name} (${accountId})` |
|
|
) |
|
|
} |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to auto-recover temp_error account ${accountId}:`, error) |
|
|
} |
|
|
}, |
|
|
6 * 60 * 1000 |
|
|
) |
|
|
|
|
|
|
|
|
if (sessionHash) { |
|
|
await redis.client.del(`sticky_session:${sessionHash}`) |
|
|
logger.info(`🗑️ Deleted sticky session mapping for hash: ${sessionHash}`) |
|
|
} |
|
|
|
|
|
logger.warn( |
|
|
`⚠️ Account ${accountData.name} (${accountId}) marked as temp_error and disabled for scheduling` |
|
|
) |
|
|
|
|
|
|
|
|
try { |
|
|
const webhookNotifier = require('../utils/webhookNotifier') |
|
|
await webhookNotifier.sendAccountAnomalyNotification({ |
|
|
accountId, |
|
|
accountName: accountData.name, |
|
|
platform: 'claude-oauth', |
|
|
status: 'temp_error', |
|
|
errorCode: 'CLAUDE_OAUTH_TEMP_ERROR', |
|
|
reason: 'Account temporarily disabled due to consecutive 500 errors' |
|
|
}) |
|
|
} catch (webhookError) { |
|
|
logger.error('Failed to send webhook notification:', webhookError) |
|
|
} |
|
|
|
|
|
return { success: true } |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to mark account ${accountId} as temp_error:`, error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async updateSessionWindowStatus(accountId, status) { |
|
|
try { |
|
|
|
|
|
if (!accountId || !status) { |
|
|
logger.warn( |
|
|
`Invalid parameters for updateSessionWindowStatus: accountId=${accountId}, status=${status}` |
|
|
) |
|
|
return |
|
|
} |
|
|
|
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData || Object.keys(accountData).length === 0) { |
|
|
logger.warn(`Account not found: ${accountId}`) |
|
|
return |
|
|
} |
|
|
|
|
|
|
|
|
const validStatuses = ['allowed', 'allowed_warning', 'rejected'] |
|
|
if (!validStatuses.includes(status)) { |
|
|
logger.warn(`Invalid session window status: ${status} for account ${accountId}`) |
|
|
return |
|
|
} |
|
|
|
|
|
const now = new Date() |
|
|
const nowIso = now.toISOString() |
|
|
|
|
|
|
|
|
accountData.sessionWindowStatus = status |
|
|
accountData.sessionWindowStatusUpdatedAt = nowIso |
|
|
|
|
|
|
|
|
if (status === 'allowed_warning' && accountData.autoStopOnWarning === 'true') { |
|
|
const alreadyAutoStopped = |
|
|
accountData.schedulable === 'false' && accountData.fiveHourAutoStopped === 'true' |
|
|
|
|
|
if (!alreadyAutoStopped) { |
|
|
const windowIdentifier = |
|
|
accountData.sessionWindowEnd || accountData.sessionWindowStart || 'unknown' |
|
|
|
|
|
let warningCount = 0 |
|
|
if (accountData.fiveHourWarningWindow === windowIdentifier) { |
|
|
const parsedCount = parseInt(accountData.fiveHourWarningCount || '0', 10) |
|
|
warningCount = Number.isNaN(parsedCount) ? 0 : parsedCount |
|
|
} |
|
|
|
|
|
const maxWarningsPerWindow = this.maxFiveHourWarningsPerWindow |
|
|
|
|
|
logger.warn( |
|
|
`⚠️ Account ${accountData.name} (${accountId}) approaching 5h limit, auto-stopping scheduling` |
|
|
) |
|
|
accountData.schedulable = 'false' |
|
|
|
|
|
accountData.fiveHourAutoStopped = 'true' |
|
|
accountData.fiveHourStoppedAt = nowIso |
|
|
|
|
|
accountData.stoppedReason = '5小时使用量接近限制,已自动停止调度' |
|
|
|
|
|
const canSendWarning = warningCount < maxWarningsPerWindow |
|
|
let updatedWarningCount = warningCount |
|
|
|
|
|
accountData.fiveHourWarningWindow = windowIdentifier |
|
|
if (canSendWarning) { |
|
|
updatedWarningCount += 1 |
|
|
accountData.fiveHourWarningLastSentAt = nowIso |
|
|
} |
|
|
accountData.fiveHourWarningCount = updatedWarningCount.toString() |
|
|
|
|
|
if (canSendWarning) { |
|
|
|
|
|
try { |
|
|
const webhookNotifier = require('../utils/webhookNotifier') |
|
|
await webhookNotifier.sendAccountAnomalyNotification({ |
|
|
accountId, |
|
|
accountName: accountData.name || 'Claude Account', |
|
|
platform: 'claude', |
|
|
status: 'warning', |
|
|
errorCode: 'CLAUDE_5H_LIMIT_WARNING', |
|
|
reason: '5小时使用量接近限制,已自动停止调度', |
|
|
timestamp: getISOStringWithTimezone(now) |
|
|
}) |
|
|
} catch (webhookError) { |
|
|
logger.error('Failed to send webhook notification:', webhookError) |
|
|
} |
|
|
} else { |
|
|
logger.debug( |
|
|
`⚠️ Account ${accountData.name} (${accountId}) reached max ${maxWarningsPerWindow} warning notifications for current 5h window, skipping webhook` |
|
|
) |
|
|
} |
|
|
} else { |
|
|
logger.debug( |
|
|
`⚠️ Account ${accountData.name} (${accountId}) already auto-stopped for 5h limit, skipping duplicate warning` |
|
|
) |
|
|
} |
|
|
} |
|
|
|
|
|
await redis.setClaudeAccount(accountId, accountData) |
|
|
|
|
|
logger.info( |
|
|
`📊 Updated session window status for account ${accountData.name} (${accountId}): ${status}` |
|
|
) |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to update session window status for account ${accountId}:`, error) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async markAccountOverloaded(accountId) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
|
|
|
const overloadMinutes = config.overloadHandling?.enabled || 0 |
|
|
|
|
|
if (overloadMinutes === 0) { |
|
|
logger.info('⏭️ 529 error handling is disabled') |
|
|
return { success: false, error: '529 error handling is disabled' } |
|
|
} |
|
|
|
|
|
const overloadKey = `account:overload:${accountId}` |
|
|
const ttl = overloadMinutes * 60 |
|
|
|
|
|
await redis.setex( |
|
|
overloadKey, |
|
|
ttl, |
|
|
JSON.stringify({ |
|
|
accountId, |
|
|
accountName: accountData.name, |
|
|
markedAt: new Date().toISOString(), |
|
|
expiresAt: new Date(Date.now() + ttl * 1000).toISOString() |
|
|
}) |
|
|
) |
|
|
|
|
|
logger.warn( |
|
|
`🚫 Account ${accountData.name} (${accountId}) marked as overloaded for ${overloadMinutes} minutes` |
|
|
) |
|
|
|
|
|
|
|
|
const updates = { |
|
|
lastOverloadAt: new Date().toISOString(), |
|
|
errorMessage: `529错误 - 过载${overloadMinutes}分钟` |
|
|
} |
|
|
|
|
|
const updatedAccountData = { ...accountData, ...updates } |
|
|
await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
|
|
|
return { success: true, accountName: accountData.name, duration: overloadMinutes } |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to mark account as overloaded: ${accountId}`, error) |
|
|
|
|
|
return { success: false, error: error.message } |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async isAccountOverloaded(accountId) { |
|
|
try { |
|
|
|
|
|
const overloadMinutes = config.overloadHandling?.enabled || 0 |
|
|
if (overloadMinutes === 0) { |
|
|
return false |
|
|
} |
|
|
|
|
|
const overloadKey = `account:overload:${accountId}` |
|
|
const overloadData = await redis.get(overloadKey) |
|
|
|
|
|
if (overloadData) { |
|
|
|
|
|
return true |
|
|
} |
|
|
|
|
|
|
|
|
return false |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to check if account is overloaded: ${accountId}`, error) |
|
|
return false |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async removeAccountOverload(accountId) { |
|
|
try { |
|
|
const accountData = await redis.getClaudeAccount(accountId) |
|
|
if (!accountData) { |
|
|
throw new Error('Account not found') |
|
|
} |
|
|
|
|
|
const overloadKey = `account:overload:${accountId}` |
|
|
await redis.del(overloadKey) |
|
|
|
|
|
logger.info(`✅ Account ${accountData.name} (${accountId}) overload status removed`) |
|
|
|
|
|
|
|
|
if (accountData.errorMessage && accountData.errorMessage.includes('529错误')) { |
|
|
const updatedAccountData = { ...accountData } |
|
|
delete updatedAccountData.errorMessage |
|
|
delete updatedAccountData.lastOverloadAt |
|
|
await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to remove overload status for account: ${accountId}`, error) |
|
|
|
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async checkAndRecoverFiveHourStoppedAccounts() { |
|
|
const result = { |
|
|
checked: 0, |
|
|
recovered: 0, |
|
|
accounts: [] |
|
|
} |
|
|
|
|
|
try { |
|
|
const accounts = await this.getAllAccounts() |
|
|
const now = new Date() |
|
|
|
|
|
for (const account of accounts) { |
|
|
|
|
|
|
|
|
if (account.fiveHourAutoStopped === true && account.schedulable === false) { |
|
|
result.checked++ |
|
|
|
|
|
|
|
|
const lockKey = `lock:account:${account.id}:recovery` |
|
|
const lockValue = `${Date.now()}_${Math.random()}` |
|
|
const lockTTL = 5000 |
|
|
|
|
|
try { |
|
|
|
|
|
const lockAcquired = await redis.setAccountLock(lockKey, lockValue, lockTTL) |
|
|
if (!lockAcquired) { |
|
|
logger.debug( |
|
|
`⏭️ Account ${account.name} (${account.id}) is being processed by another instance` |
|
|
) |
|
|
continue |
|
|
} |
|
|
|
|
|
|
|
|
const latestAccount = await redis.getClaudeAccount(account.id) |
|
|
if ( |
|
|
!latestAccount || |
|
|
latestAccount.fiveHourAutoStopped !== 'true' || |
|
|
latestAccount.schedulable !== 'false' |
|
|
) { |
|
|
|
|
|
await redis.releaseAccountLock(lockKey, lockValue) |
|
|
continue |
|
|
} |
|
|
|
|
|
|
|
|
let shouldRecover = false |
|
|
let newWindowStart = null |
|
|
let newWindowEnd = null |
|
|
|
|
|
if (latestAccount.sessionWindowEnd) { |
|
|
const windowEnd = new Date(latestAccount.sessionWindowEnd) |
|
|
|
|
|
|
|
|
if (now.getTime() > windowEnd.getTime() + 60000) { |
|
|
shouldRecover = true |
|
|
|
|
|
|
|
|
|
|
|
newWindowStart = new Date(windowEnd) |
|
|
newWindowStart.setMilliseconds(newWindowStart.getMilliseconds() + 1) |
|
|
newWindowEnd = new Date(newWindowStart) |
|
|
newWindowEnd.setHours(newWindowEnd.getHours() + 5) |
|
|
|
|
|
logger.info( |
|
|
`🔄 Account ${latestAccount.name} (${latestAccount.id}) has entered new session window. ` + |
|
|
`Old window: ${latestAccount.sessionWindowStart} - ${latestAccount.sessionWindowEnd}, ` + |
|
|
`New window: ${newWindowStart.toISOString()} - ${newWindowEnd.toISOString()}` |
|
|
) |
|
|
} |
|
|
} else { |
|
|
|
|
|
if (latestAccount.fiveHourStoppedAt) { |
|
|
const stoppedAt = new Date(latestAccount.fiveHourStoppedAt) |
|
|
const hoursSinceStopped = (now.getTime() - stoppedAt.getTime()) / (1000 * 60 * 60) |
|
|
|
|
|
|
|
|
if (hoursSinceStopped > 5.017) { |
|
|
|
|
|
shouldRecover = true |
|
|
newWindowStart = this._calculateSessionWindowStart(now) |
|
|
newWindowEnd = this._calculateSessionWindowEnd(newWindowStart) |
|
|
|
|
|
logger.info( |
|
|
`🔄 Account ${latestAccount.name} (${latestAccount.id}) stopped ${hoursSinceStopped.toFixed(2)} hours ago, recovering` |
|
|
) |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
if (shouldRecover) { |
|
|
|
|
|
const updatedAccountData = { ...latestAccount } |
|
|
|
|
|
|
|
|
updatedAccountData.schedulable = 'true' |
|
|
delete updatedAccountData.fiveHourAutoStopped |
|
|
delete updatedAccountData.fiveHourStoppedAt |
|
|
await this._clearFiveHourWarningMetadata(account.id, updatedAccountData) |
|
|
delete updatedAccountData.stoppedReason |
|
|
|
|
|
|
|
|
if (newWindowStart && newWindowEnd) { |
|
|
updatedAccountData.sessionWindowStart = newWindowStart.toISOString() |
|
|
updatedAccountData.sessionWindowEnd = newWindowEnd.toISOString() |
|
|
|
|
|
|
|
|
delete updatedAccountData.sessionWindowStatus |
|
|
delete updatedAccountData.sessionWindowStatusUpdatedAt |
|
|
} |
|
|
|
|
|
|
|
|
await redis.setClaudeAccount(account.id, updatedAccountData) |
|
|
|
|
|
const fieldsToRemove = ['fiveHourAutoStopped', 'fiveHourStoppedAt'] |
|
|
if (newWindowStart && newWindowEnd) { |
|
|
fieldsToRemove.push('sessionWindowStatus', 'sessionWindowStatusUpdatedAt') |
|
|
} |
|
|
await this._removeAccountFields(account.id, fieldsToRemove, 'five_hour_recovery_task') |
|
|
|
|
|
result.recovered++ |
|
|
result.accounts.push({ |
|
|
id: latestAccount.id, |
|
|
name: latestAccount.name, |
|
|
oldWindow: latestAccount.sessionWindowEnd |
|
|
? { |
|
|
start: latestAccount.sessionWindowStart, |
|
|
end: latestAccount.sessionWindowEnd |
|
|
} |
|
|
: null, |
|
|
newWindow: |
|
|
newWindowStart && newWindowEnd |
|
|
? { |
|
|
start: newWindowStart.toISOString(), |
|
|
end: newWindowEnd.toISOString() |
|
|
} |
|
|
: null |
|
|
}) |
|
|
|
|
|
logger.info( |
|
|
`✅ Auto-resumed scheduling for account ${latestAccount.name} (${latestAccount.id}) - 5-hour limit expired` |
|
|
) |
|
|
} |
|
|
|
|
|
|
|
|
await redis.releaseAccountLock(lockKey, lockValue) |
|
|
} catch (error) { |
|
|
|
|
|
if (lockKey && lockValue) { |
|
|
try { |
|
|
await redis.releaseAccountLock(lockKey, lockValue) |
|
|
} catch (unlockError) { |
|
|
logger.error(`Failed to release lock for account ${account.id}:`, unlockError) |
|
|
} |
|
|
} |
|
|
logger.error( |
|
|
`❌ Failed to check/recover 5-hour stopped account ${account.name} (${account.id}):`, |
|
|
error |
|
|
) |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
if (result.recovered > 0) { |
|
|
logger.info( |
|
|
`🔄 5-hour limit recovery completed: ${result.recovered}/${result.checked} accounts recovered` |
|
|
) |
|
|
} |
|
|
|
|
|
return result |
|
|
} catch (error) { |
|
|
logger.error('❌ Failed to check and recover 5-hour stopped accounts:', error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
async _removeAccountFields(accountId, fields = [], context = 'general_cleanup') { |
|
|
if (!Array.isArray(fields) || fields.length === 0) { |
|
|
return |
|
|
} |
|
|
|
|
|
const filteredFields = fields.filter((field) => typeof field === 'string' && field.trim()) |
|
|
if (filteredFields.length === 0) { |
|
|
return |
|
|
} |
|
|
|
|
|
const accountKey = `claude:account:${accountId}` |
|
|
|
|
|
try { |
|
|
await redis.client.hdel(accountKey, ...filteredFields) |
|
|
logger.debug( |
|
|
`🧹 已在 ${context} 阶段为账号 ${accountId} 删除字段 [${filteredFields.join(', ')}]` |
|
|
) |
|
|
} catch (error) { |
|
|
logger.error( |
|
|
`❌ 无法在 ${context} 阶段为账号 ${accountId} 删除字段 [${filteredFields.join(', ')}]:`, |
|
|
error |
|
|
) |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
module.exports = new ClaudeAccountService() |
|
|
|