|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
const logger = require('../utils/logger') |
|
|
const openaiAccountService = require('./openaiAccountService') |
|
|
const claudeAccountService = require('./claudeAccountService') |
|
|
const claudeConsoleAccountService = require('./claudeConsoleAccountService') |
|
|
const unifiedOpenAIScheduler = require('./unifiedOpenAIScheduler') |
|
|
const webhookService = require('./webhookService') |
|
|
|
|
|
class RateLimitCleanupService { |
|
|
constructor() { |
|
|
this.cleanupInterval = null |
|
|
this.isRunning = false |
|
|
|
|
|
this.intervalMs = 5 * 60 * 1000 |
|
|
|
|
|
this.clearedAccounts = [] |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
start(intervalMinutes = 5) { |
|
|
if (this.cleanupInterval) { |
|
|
logger.warn('⚠️ Rate limit cleanup service is already running') |
|
|
return |
|
|
} |
|
|
|
|
|
this.intervalMs = intervalMinutes * 60 * 1000 |
|
|
|
|
|
logger.info(`🧹 Starting rate limit cleanup service (interval: ${intervalMinutes} minutes)`) |
|
|
|
|
|
|
|
|
this.performCleanup() |
|
|
|
|
|
|
|
|
this.cleanupInterval = setInterval(() => { |
|
|
this.performCleanup() |
|
|
}, this.intervalMs) |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
stop() { |
|
|
if (this.cleanupInterval) { |
|
|
clearInterval(this.cleanupInterval) |
|
|
this.cleanupInterval = null |
|
|
logger.info('🛑 Rate limit cleanup service stopped') |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async performCleanup() { |
|
|
if (this.isRunning) { |
|
|
logger.debug('⏭️ Cleanup already in progress, skipping this cycle') |
|
|
return |
|
|
} |
|
|
|
|
|
this.isRunning = true |
|
|
const startTime = Date.now() |
|
|
|
|
|
try { |
|
|
logger.debug('🔍 Starting rate limit cleanup check...') |
|
|
|
|
|
const results = { |
|
|
openai: { checked: 0, cleared: 0, errors: [] }, |
|
|
claude: { checked: 0, cleared: 0, errors: [] }, |
|
|
claudeConsole: { checked: 0, cleared: 0, errors: [] } |
|
|
} |
|
|
|
|
|
|
|
|
await this.cleanupOpenAIAccounts(results.openai) |
|
|
|
|
|
|
|
|
await this.cleanupClaudeAccounts(results.claude) |
|
|
|
|
|
|
|
|
await this.cleanupClaudeConsoleAccounts(results.claudeConsole) |
|
|
|
|
|
const totalChecked = |
|
|
results.openai.checked + results.claude.checked + results.claudeConsole.checked |
|
|
const totalCleared = |
|
|
results.openai.cleared + results.claude.cleared + results.claudeConsole.cleared |
|
|
const duration = Date.now() - startTime |
|
|
|
|
|
if (totalCleared > 0) { |
|
|
logger.info( |
|
|
`✅ Rate limit cleanup completed: ${totalCleared} accounts cleared out of ${totalChecked} checked (${duration}ms)` |
|
|
) |
|
|
logger.info(` OpenAI: ${results.openai.cleared}/${results.openai.checked}`) |
|
|
logger.info(` Claude: ${results.claude.cleared}/${results.claude.checked}`) |
|
|
logger.info( |
|
|
` Claude Console: ${results.claudeConsole.cleared}/${results.claudeConsole.checked}` |
|
|
) |
|
|
|
|
|
|
|
|
if (this.clearedAccounts.length > 0) { |
|
|
await this.sendRecoveryNotifications() |
|
|
} |
|
|
} else { |
|
|
logger.debug( |
|
|
`🔍 Rate limit cleanup check completed: no expired limits found (${duration}ms)` |
|
|
) |
|
|
} |
|
|
|
|
|
|
|
|
const allErrors = [ |
|
|
...results.openai.errors, |
|
|
...results.claude.errors, |
|
|
...results.claudeConsole.errors |
|
|
] |
|
|
if (allErrors.length > 0) { |
|
|
logger.warn(`⚠️ Encountered ${allErrors.length} errors during cleanup:`, allErrors) |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error('❌ Rate limit cleanup failed:', error) |
|
|
} finally { |
|
|
|
|
|
this.clearedAccounts = [] |
|
|
this.isRunning = false |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async cleanupOpenAIAccounts(result) { |
|
|
try { |
|
|
|
|
|
const accounts = await openaiAccountService.getAllAccounts() |
|
|
|
|
|
for (const account of accounts) { |
|
|
const { rateLimitStatus } = account |
|
|
const isRateLimited = |
|
|
rateLimitStatus === 'limited' || |
|
|
(rateLimitStatus && |
|
|
typeof rateLimitStatus === 'object' && |
|
|
(rateLimitStatus.status === 'limited' || rateLimitStatus.isRateLimited === true)) |
|
|
|
|
|
if (isRateLimited) { |
|
|
result.checked++ |
|
|
|
|
|
try { |
|
|
|
|
|
const isStillLimited = await unifiedOpenAIScheduler.isAccountRateLimited(account.id) |
|
|
|
|
|
if (!isStillLimited) { |
|
|
result.cleared++ |
|
|
logger.info( |
|
|
`🧹 Auto-cleared expired rate limit for OpenAI account: ${account.name} (${account.id})` |
|
|
) |
|
|
|
|
|
|
|
|
this.clearedAccounts.push({ |
|
|
platform: 'OpenAI', |
|
|
accountId: account.id, |
|
|
accountName: account.name, |
|
|
previousStatus: 'rate_limited', |
|
|
currentStatus: 'active' |
|
|
}) |
|
|
} |
|
|
} catch (error) { |
|
|
result.errors.push({ |
|
|
accountId: account.id, |
|
|
accountName: account.name, |
|
|
error: error.message |
|
|
}) |
|
|
} |
|
|
} |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error('Failed to cleanup OpenAI accounts:', error) |
|
|
result.errors.push({ error: error.message }) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async cleanupClaudeAccounts(result) { |
|
|
try { |
|
|
|
|
|
const redis = require('../models/redis') |
|
|
const accounts = await redis.getAllClaudeAccounts() |
|
|
|
|
|
for (const account of accounts) { |
|
|
|
|
|
const isRateLimited = |
|
|
account.rateLimitStatus === 'limited' || |
|
|
(account.rateLimitStatus && |
|
|
typeof account.rateLimitStatus === 'object' && |
|
|
account.rateLimitStatus.status === 'limited') |
|
|
|
|
|
const autoStopped = account.rateLimitAutoStopped === 'true' |
|
|
const needsAutoStopRecovery = |
|
|
autoStopped && (account.rateLimitEndAt || account.schedulable === 'false') |
|
|
|
|
|
|
|
|
if (isRateLimited || account.rateLimitedAt || needsAutoStopRecovery) { |
|
|
result.checked++ |
|
|
|
|
|
try { |
|
|
|
|
|
const isStillLimited = await claudeAccountService.isAccountRateLimited(account.id) |
|
|
|
|
|
if (!isStillLimited) { |
|
|
if (!isRateLimited && autoStopped) { |
|
|
await claudeAccountService.removeAccountRateLimit(account.id) |
|
|
} |
|
|
result.cleared++ |
|
|
logger.info( |
|
|
`🧹 Auto-cleared expired rate limit for Claude account: ${account.name} (${account.id})` |
|
|
) |
|
|
|
|
|
|
|
|
this.clearedAccounts.push({ |
|
|
platform: 'Claude', |
|
|
accountId: account.id, |
|
|
accountName: account.name, |
|
|
previousStatus: 'rate_limited', |
|
|
currentStatus: 'active' |
|
|
}) |
|
|
} |
|
|
} catch (error) { |
|
|
result.errors.push({ |
|
|
accountId: account.id, |
|
|
accountName: account.name, |
|
|
error: error.message |
|
|
}) |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
try { |
|
|
const fiveHourResult = await claudeAccountService.checkAndRecoverFiveHourStoppedAccounts() |
|
|
|
|
|
if (fiveHourResult.recovered > 0) { |
|
|
|
|
|
for (const account of fiveHourResult.accounts) { |
|
|
this.clearedAccounts.push({ |
|
|
platform: 'Claude', |
|
|
accountId: account.id, |
|
|
accountName: account.name, |
|
|
previousStatus: '5hour_limited', |
|
|
currentStatus: 'active', |
|
|
windowInfo: account.newWindow |
|
|
}) |
|
|
} |
|
|
|
|
|
|
|
|
result.checked += fiveHourResult.checked |
|
|
result.cleared += fiveHourResult.recovered |
|
|
|
|
|
logger.info( |
|
|
`🕐 Claude 5-hour limit recovery: ${fiveHourResult.recovered}/${fiveHourResult.checked} accounts recovered` |
|
|
) |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error('Failed to check and recover 5-hour stopped Claude accounts:', error) |
|
|
result.errors.push({ |
|
|
type: '5hour_recovery', |
|
|
error: error.message |
|
|
}) |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error('Failed to cleanup Claude accounts:', error) |
|
|
result.errors.push({ error: error.message }) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async cleanupClaudeConsoleAccounts(result) { |
|
|
try { |
|
|
|
|
|
const accounts = await claudeConsoleAccountService.getAllAccounts() |
|
|
|
|
|
for (const account of accounts) { |
|
|
|
|
|
const isRateLimited = |
|
|
account.rateLimitStatus === 'limited' || |
|
|
(account.rateLimitStatus && |
|
|
typeof account.rateLimitStatus === 'object' && |
|
|
account.rateLimitStatus.status === 'limited') |
|
|
|
|
|
const autoStopped = account.rateLimitAutoStopped === 'true' |
|
|
const needsAutoStopRecovery = autoStopped && account.schedulable === 'false' |
|
|
|
|
|
|
|
|
const hasStatusRateLimited = account.status === 'rate_limited' |
|
|
|
|
|
if (isRateLimited || hasStatusRateLimited || needsAutoStopRecovery) { |
|
|
result.checked++ |
|
|
|
|
|
try { |
|
|
|
|
|
const isStillLimited = await claudeConsoleAccountService.isAccountRateLimited( |
|
|
account.id |
|
|
) |
|
|
|
|
|
if (!isStillLimited) { |
|
|
if (!isRateLimited && autoStopped) { |
|
|
await claudeConsoleAccountService.removeAccountRateLimit(account.id) |
|
|
} |
|
|
result.cleared++ |
|
|
|
|
|
|
|
|
if (hasStatusRateLimited && !isRateLimited) { |
|
|
await claudeConsoleAccountService.updateAccount(account.id, { |
|
|
status: 'active' |
|
|
}) |
|
|
} |
|
|
|
|
|
logger.info( |
|
|
`🧹 Auto-cleared expired rate limit for Claude Console account: ${account.name} (${account.id})` |
|
|
) |
|
|
|
|
|
|
|
|
this.clearedAccounts.push({ |
|
|
platform: 'Claude Console', |
|
|
accountId: account.id, |
|
|
accountName: account.name, |
|
|
previousStatus: 'rate_limited', |
|
|
currentStatus: 'active' |
|
|
}) |
|
|
} |
|
|
} catch (error) { |
|
|
result.errors.push({ |
|
|
accountId: account.id, |
|
|
accountName: account.name, |
|
|
error: error.message |
|
|
}) |
|
|
} |
|
|
} |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error('Failed to cleanup Claude Console accounts:', error) |
|
|
result.errors.push({ error: error.message }) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async manualCleanup() { |
|
|
logger.info('🧹 Manual rate limit cleanup triggered') |
|
|
await this.performCleanup() |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async sendRecoveryNotifications() { |
|
|
try { |
|
|
|
|
|
const groupedAccounts = {} |
|
|
for (const account of this.clearedAccounts) { |
|
|
if (!groupedAccounts[account.platform]) { |
|
|
groupedAccounts[account.platform] = [] |
|
|
} |
|
|
groupedAccounts[account.platform].push(account) |
|
|
} |
|
|
|
|
|
|
|
|
const platforms = Object.keys(groupedAccounts) |
|
|
const totalAccounts = this.clearedAccounts.length |
|
|
|
|
|
let message = `🎉 共有 ${totalAccounts} 个账户的限流状态已恢复\n\n` |
|
|
|
|
|
for (const platform of platforms) { |
|
|
const accounts = groupedAccounts[platform] |
|
|
message += `**${platform}** (${accounts.length} 个):\n` |
|
|
for (const account of accounts) { |
|
|
message += `• ${account.accountName} (ID: ${account.accountId})\n` |
|
|
} |
|
|
message += '\n' |
|
|
} |
|
|
|
|
|
|
|
|
await webhookService.sendNotification('rateLimitRecovery', { |
|
|
title: '限流恢复通知', |
|
|
message, |
|
|
totalAccounts, |
|
|
platforms: Object.keys(groupedAccounts), |
|
|
accounts: this.clearedAccounts, |
|
|
timestamp: new Date().toISOString() |
|
|
}) |
|
|
|
|
|
logger.info(`📢 已发送限流恢复通知,涉及 ${totalAccounts} 个账户`) |
|
|
} catch (error) { |
|
|
logger.error('❌ 发送限流恢复通知失败:', error) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
getStatus() { |
|
|
return { |
|
|
running: !!this.cleanupInterval, |
|
|
intervalMinutes: this.intervalMs / (60 * 1000), |
|
|
isProcessing: this.isRunning |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
const rateLimitCleanupService = new RateLimitCleanupService() |
|
|
|
|
|
module.exports = rateLimitCleanupService |
|
|
|