|
|
const openaiAccountService = require('./openaiAccountService') |
|
|
const openaiResponsesAccountService = require('./openaiResponsesAccountService') |
|
|
const accountGroupService = require('./accountGroupService') |
|
|
const redis = require('../models/redis') |
|
|
const logger = require('../utils/logger') |
|
|
|
|
|
class UnifiedOpenAIScheduler { |
|
|
constructor() { |
|
|
this.SESSION_MAPPING_PREFIX = 'unified_openai_session_mapping:' |
|
|
} |
|
|
|
|
|
|
|
|
_isSchedulable(schedulable) { |
|
|
|
|
|
if (schedulable === undefined || schedulable === null) { |
|
|
return true |
|
|
} |
|
|
|
|
|
return schedulable !== false && schedulable !== 'false' |
|
|
} |
|
|
|
|
|
|
|
|
_isRateLimited(rateLimitStatus) { |
|
|
if (!rateLimitStatus) { |
|
|
return false |
|
|
} |
|
|
|
|
|
|
|
|
if (typeof rateLimitStatus === 'string') { |
|
|
return rateLimitStatus === 'limited' |
|
|
} |
|
|
|
|
|
|
|
|
if (typeof rateLimitStatus === 'object') { |
|
|
if (rateLimitStatus.isRateLimited === false) { |
|
|
return false |
|
|
} |
|
|
|
|
|
return rateLimitStatus.status === 'limited' || rateLimitStatus.isRateLimited === true |
|
|
} |
|
|
|
|
|
return false |
|
|
} |
|
|
|
|
|
|
|
|
_hasRateLimitFlag(rateLimitStatus) { |
|
|
if (!rateLimitStatus) { |
|
|
return false |
|
|
} |
|
|
|
|
|
if (typeof rateLimitStatus === 'string') { |
|
|
return rateLimitStatus === 'limited' |
|
|
} |
|
|
|
|
|
if (typeof rateLimitStatus === 'object') { |
|
|
return rateLimitStatus.status === 'limited' || rateLimitStatus.isRateLimited === true |
|
|
} |
|
|
|
|
|
return false |
|
|
} |
|
|
|
|
|
|
|
|
async _ensureAccountReadyForScheduling(account, accountId, { sanitized = true } = {}) { |
|
|
const hasRateLimitFlag = this._hasRateLimitFlag(account.rateLimitStatus) |
|
|
let rateLimitChecked = false |
|
|
let stillLimited = false |
|
|
|
|
|
let isSchedulable = this._isSchedulable(account.schedulable) |
|
|
|
|
|
if (!isSchedulable) { |
|
|
if (!hasRateLimitFlag) { |
|
|
return { canUse: false, reason: 'not_schedulable' } |
|
|
} |
|
|
|
|
|
stillLimited = await this.isAccountRateLimited(accountId) |
|
|
rateLimitChecked = true |
|
|
if (stillLimited) { |
|
|
return { canUse: false, reason: 'rate_limited' } |
|
|
} |
|
|
|
|
|
|
|
|
if (sanitized) { |
|
|
account.schedulable = true |
|
|
} else { |
|
|
account.schedulable = 'true' |
|
|
} |
|
|
isSchedulable = true |
|
|
logger.info(`✅ OpenAI账号 ${account.name || accountId} 已解除限流,恢复调度权限`) |
|
|
} |
|
|
|
|
|
if (hasRateLimitFlag) { |
|
|
if (!rateLimitChecked) { |
|
|
stillLimited = await this.isAccountRateLimited(accountId) |
|
|
rateLimitChecked = true |
|
|
} |
|
|
if (stillLimited) { |
|
|
return { canUse: false, reason: 'rate_limited' } |
|
|
} |
|
|
|
|
|
|
|
|
if (sanitized) { |
|
|
account.rateLimitStatus = { |
|
|
status: 'normal', |
|
|
isRateLimited: false, |
|
|
rateLimitedAt: null, |
|
|
rateLimitResetAt: null, |
|
|
minutesRemaining: 0 |
|
|
} |
|
|
} else { |
|
|
account.rateLimitStatus = 'normal' |
|
|
account.rateLimitedAt = null |
|
|
account.rateLimitResetAt = null |
|
|
} |
|
|
|
|
|
if (account.status === 'rateLimited') { |
|
|
account.status = 'active' |
|
|
} |
|
|
} |
|
|
|
|
|
if (!rateLimitChecked) { |
|
|
stillLimited = await this.isAccountRateLimited(accountId) |
|
|
if (stillLimited) { |
|
|
return { canUse: false, reason: 'rate_limited' } |
|
|
} |
|
|
} |
|
|
|
|
|
return { canUse: true } |
|
|
} |
|
|
|
|
|
|
|
|
async selectAccountForApiKey(apiKeyData, sessionHash = null, requestedModel = null) { |
|
|
try { |
|
|
|
|
|
if (apiKeyData.openaiAccountId) { |
|
|
|
|
|
if (apiKeyData.openaiAccountId.startsWith('group:')) { |
|
|
const groupId = apiKeyData.openaiAccountId.replace('group:', '') |
|
|
logger.info( |
|
|
`🎯 API key ${apiKeyData.name} is bound to group ${groupId}, selecting from group` |
|
|
) |
|
|
return await this.selectAccountFromGroup(groupId, sessionHash, requestedModel, apiKeyData) |
|
|
} |
|
|
|
|
|
|
|
|
let boundAccount = null |
|
|
let accountType = 'openai' |
|
|
|
|
|
|
|
|
if (apiKeyData.openaiAccountId.startsWith('responses:')) { |
|
|
const accountId = apiKeyData.openaiAccountId.replace('responses:', '') |
|
|
boundAccount = await openaiResponsesAccountService.getAccount(accountId) |
|
|
accountType = 'openai-responses' |
|
|
} else { |
|
|
|
|
|
boundAccount = await openaiAccountService.getAccount(apiKeyData.openaiAccountId) |
|
|
accountType = 'openai' |
|
|
} |
|
|
|
|
|
const isActiveBoundAccount = |
|
|
boundAccount && |
|
|
(boundAccount.isActive === true || boundAccount.isActive === 'true') && |
|
|
boundAccount.status !== 'error' && |
|
|
boundAccount.status !== 'unauthorized' |
|
|
|
|
|
if (isActiveBoundAccount) { |
|
|
if (accountType === 'openai') { |
|
|
const readiness = await this._ensureAccountReadyForScheduling( |
|
|
boundAccount, |
|
|
boundAccount.id, |
|
|
{ sanitized: false } |
|
|
) |
|
|
|
|
|
if (!readiness.canUse) { |
|
|
const isRateLimited = readiness.reason === 'rate_limited' |
|
|
const errorMsg = isRateLimited |
|
|
? `Dedicated account ${boundAccount.name} is currently rate limited` |
|
|
: `Dedicated account ${boundAccount.name} is not schedulable` |
|
|
logger.warn(`⚠️ ${errorMsg}`) |
|
|
const error = new Error(errorMsg) |
|
|
error.statusCode = isRateLimited ? 429 : 403 |
|
|
throw error |
|
|
} |
|
|
} else { |
|
|
const hasRateLimitFlag = this._isRateLimited(boundAccount.rateLimitStatus) |
|
|
if (hasRateLimitFlag) { |
|
|
const isRateLimitCleared = await openaiResponsesAccountService.checkAndClearRateLimit( |
|
|
boundAccount.id |
|
|
) |
|
|
if (!isRateLimitCleared) { |
|
|
const errorMsg = `Dedicated account ${boundAccount.name} is currently rate limited` |
|
|
logger.warn(`⚠️ ${errorMsg}`) |
|
|
const error = new Error(errorMsg) |
|
|
error.statusCode = 429 |
|
|
throw error |
|
|
} |
|
|
|
|
|
boundAccount = await openaiResponsesAccountService.getAccount(boundAccount.id) |
|
|
if (!boundAccount) { |
|
|
const errorMsg = `Dedicated account ${apiKeyData.openaiAccountId} not found after rate limit reset` |
|
|
logger.warn(`⚠️ ${errorMsg}`) |
|
|
const error = new Error(errorMsg) |
|
|
error.statusCode = 404 |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
if (!this._isSchedulable(boundAccount.schedulable)) { |
|
|
const errorMsg = `Dedicated account ${boundAccount.name} is not schedulable` |
|
|
logger.warn(`⚠️ ${errorMsg}`) |
|
|
const error = new Error(errorMsg) |
|
|
error.statusCode = 403 |
|
|
throw error |
|
|
} |
|
|
|
|
|
|
|
|
if (openaiResponsesAccountService.isSubscriptionExpired(boundAccount)) { |
|
|
const errorMsg = `Dedicated account ${boundAccount.name} subscription has expired` |
|
|
logger.warn(`⚠️ ${errorMsg}`) |
|
|
const error = new Error(errorMsg) |
|
|
error.statusCode = 403 |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if ( |
|
|
accountType === 'openai' && |
|
|
requestedModel && |
|
|
boundAccount.supportedModels && |
|
|
boundAccount.supportedModels.length > 0 |
|
|
) { |
|
|
const modelSupported = boundAccount.supportedModels.includes(requestedModel) |
|
|
if (!modelSupported) { |
|
|
const errorMsg = `Dedicated account ${boundAccount.name} does not support model ${requestedModel}` |
|
|
logger.warn(`⚠️ ${errorMsg}`) |
|
|
const error = new Error(errorMsg) |
|
|
error.statusCode = 400 |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
logger.info( |
|
|
`🎯 Using bound dedicated ${accountType} account: ${boundAccount.name} (${boundAccount.id}) for API key ${apiKeyData.name}` |
|
|
) |
|
|
|
|
|
if (accountType === 'openai') { |
|
|
await openaiAccountService.recordUsage(boundAccount.id, 0) |
|
|
} else { |
|
|
await openaiResponsesAccountService.updateAccount(boundAccount.id, { |
|
|
lastUsedAt: new Date().toISOString() |
|
|
}) |
|
|
} |
|
|
return { |
|
|
accountId: boundAccount.id, |
|
|
accountType |
|
|
} |
|
|
} else { |
|
|
|
|
|
let errorMsg |
|
|
if (!boundAccount) { |
|
|
errorMsg = `Dedicated account ${apiKeyData.openaiAccountId} not found` |
|
|
} else if (!(boundAccount.isActive === true || boundAccount.isActive === 'true')) { |
|
|
errorMsg = `Dedicated account ${boundAccount.name} is not active` |
|
|
} else if (boundAccount.status === 'unauthorized') { |
|
|
errorMsg = `Dedicated account ${boundAccount.name} is unauthorized` |
|
|
} else if (boundAccount.status === 'error') { |
|
|
errorMsg = `Dedicated account ${boundAccount.name} is not available (error status)` |
|
|
} else { |
|
|
errorMsg = `Dedicated account ${boundAccount.name} is not available (inactive or forbidden)` |
|
|
} |
|
|
logger.warn(`⚠️ ${errorMsg}`) |
|
|
const error = new Error(errorMsg) |
|
|
error.statusCode = boundAccount ? 403 : 404 |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
if (sessionHash) { |
|
|
const mappedAccount = await this._getSessionMapping(sessionHash) |
|
|
if (mappedAccount) { |
|
|
|
|
|
const isAvailable = await this._isAccountAvailable( |
|
|
mappedAccount.accountId, |
|
|
mappedAccount.accountType |
|
|
) |
|
|
if (isAvailable) { |
|
|
|
|
|
await this._extendSessionMappingTTL(sessionHash) |
|
|
logger.info( |
|
|
`🎯 Using sticky session account: ${mappedAccount.accountId} (${mappedAccount.accountType}) for session ${sessionHash}` |
|
|
) |
|
|
|
|
|
await openaiAccountService.recordUsage(mappedAccount.accountId, 0) |
|
|
return mappedAccount |
|
|
} else { |
|
|
logger.warn( |
|
|
`⚠️ Mapped account ${mappedAccount.accountId} is no longer available, selecting new account` |
|
|
) |
|
|
await this._deleteSessionMapping(sessionHash) |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
const availableAccounts = await this._getAllAvailableAccounts(apiKeyData, requestedModel) |
|
|
|
|
|
if (availableAccounts.length === 0) { |
|
|
|
|
|
if (requestedModel) { |
|
|
const error = new Error( |
|
|
`No available OpenAI accounts support the requested model: ${requestedModel}` |
|
|
) |
|
|
error.statusCode = 400 |
|
|
throw error |
|
|
} else { |
|
|
const error = new Error('No available OpenAI accounts') |
|
|
error.statusCode = 402 |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
const sortedAccounts = availableAccounts.sort((a, b) => { |
|
|
const aLastUsed = new Date(a.lastUsedAt || 0).getTime() |
|
|
const bLastUsed = new Date(b.lastUsedAt || 0).getTime() |
|
|
return aLastUsed - bLastUsed |
|
|
}) |
|
|
|
|
|
|
|
|
const selectedAccount = sortedAccounts[0] |
|
|
|
|
|
|
|
|
if (sessionHash) { |
|
|
await this._setSessionMapping( |
|
|
sessionHash, |
|
|
selectedAccount.accountId, |
|
|
selectedAccount.accountType |
|
|
) |
|
|
logger.info( |
|
|
`🎯 Created new sticky session mapping: ${selectedAccount.name} (${selectedAccount.accountId}, ${selectedAccount.accountType}) for session ${sessionHash}` |
|
|
) |
|
|
} |
|
|
|
|
|
logger.info( |
|
|
`🎯 Selected account: ${selectedAccount.name} (${selectedAccount.accountId}, ${selectedAccount.accountType}) for API key ${apiKeyData.name}` |
|
|
) |
|
|
|
|
|
|
|
|
await openaiAccountService.recordUsage(selectedAccount.accountId, 0) |
|
|
|
|
|
return { |
|
|
accountId: selectedAccount.accountId, |
|
|
accountType: selectedAccount.accountType |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error('❌ Failed to select account for API key:', error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async _getAllAvailableAccounts(apiKeyData, requestedModel = null) { |
|
|
const availableAccounts = [] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
const openaiAccounts = await openaiAccountService.getAllAccounts() |
|
|
for (let account of openaiAccounts) { |
|
|
if ( |
|
|
account.isActive && |
|
|
account.status !== 'error' && |
|
|
(account.accountType === 'shared' || !account.accountType) |
|
|
) { |
|
|
const accountId = account.id || account.accountId |
|
|
|
|
|
const readiness = await this._ensureAccountReadyForScheduling(account, accountId, { |
|
|
sanitized: true |
|
|
}) |
|
|
|
|
|
if (!readiness.canUse) { |
|
|
if (readiness.reason === 'rate_limited') { |
|
|
logger.debug(`⏭️ 跳过 OpenAI 账号 ${account.name} - 仍处于限流状态`) |
|
|
} else { |
|
|
logger.debug(`⏭️ 跳过 OpenAI 账号 ${account.name} - 已被管理员禁用调度`) |
|
|
} |
|
|
continue |
|
|
} |
|
|
|
|
|
|
|
|
const isExpired = openaiAccountService.isTokenExpired(account) |
|
|
if (isExpired) { |
|
|
if (!account.refreshToken) { |
|
|
logger.warn( |
|
|
`⚠️ OpenAI account ${account.name} token expired and no refresh token available` |
|
|
) |
|
|
continue |
|
|
} |
|
|
|
|
|
|
|
|
try { |
|
|
logger.info(`🔄 Auto-refreshing expired token for OpenAI account ${account.name}`) |
|
|
await openaiAccountService.refreshAccountToken(account.id) |
|
|
|
|
|
account = await openaiAccountService.getAccount(account.id) |
|
|
logger.info(`✅ Token refreshed successfully for ${account.name}`) |
|
|
} catch (refreshError) { |
|
|
logger.error(`❌ Failed to refresh token for ${account.name}:`, refreshError.message) |
|
|
continue |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if (requestedModel && account.supportedModels && account.supportedModels.length > 0) { |
|
|
const modelSupported = account.supportedModels.includes(requestedModel) |
|
|
if (!modelSupported) { |
|
|
logger.debug( |
|
|
`⏭️ Skipping OpenAI account ${account.name} - doesn't support model ${requestedModel}` |
|
|
) |
|
|
continue |
|
|
} |
|
|
} |
|
|
|
|
|
availableAccounts.push({ |
|
|
...account, |
|
|
accountId: account.id, |
|
|
accountType: 'openai', |
|
|
priority: parseInt(account.priority) || 50, |
|
|
lastUsedAt: account.lastUsedAt || '0' |
|
|
}) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
const openaiResponsesAccounts = await openaiResponsesAccountService.getAllAccounts() |
|
|
for (const account of openaiResponsesAccounts) { |
|
|
if ( |
|
|
(account.isActive === true || account.isActive === 'true') && |
|
|
account.status !== 'error' && |
|
|
account.status !== 'rateLimited' && |
|
|
(account.accountType === 'shared' || !account.accountType) |
|
|
) { |
|
|
const hasRateLimitFlag = this._hasRateLimitFlag(account.rateLimitStatus) |
|
|
const schedulable = this._isSchedulable(account.schedulable) |
|
|
|
|
|
if (!schedulable && !hasRateLimitFlag) { |
|
|
logger.debug(`⏭️ Skipping OpenAI-Responses account ${account.name} - not schedulable`) |
|
|
continue |
|
|
} |
|
|
|
|
|
let isRateLimitCleared = false |
|
|
if (hasRateLimitFlag) { |
|
|
isRateLimitCleared = await openaiResponsesAccountService.checkAndClearRateLimit( |
|
|
account.id |
|
|
) |
|
|
|
|
|
if (!isRateLimitCleared) { |
|
|
logger.debug(`⏭️ Skipping OpenAI-Responses account ${account.name} - rate limited`) |
|
|
continue |
|
|
} |
|
|
|
|
|
if (!schedulable) { |
|
|
account.schedulable = 'true' |
|
|
account.status = 'active' |
|
|
logger.info(`✅ OpenAI-Responses账号 ${account.name} 已解除限流,恢复调度权限`) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
if (openaiResponsesAccountService.isSubscriptionExpired(account)) { |
|
|
logger.debug( |
|
|
`⏭️ Skipping OpenAI-Responses account ${account.name} - subscription expired` |
|
|
) |
|
|
continue |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
availableAccounts.push({ |
|
|
...account, |
|
|
accountId: account.id, |
|
|
accountType: 'openai-responses', |
|
|
priority: parseInt(account.priority) || 50, |
|
|
lastUsedAt: account.lastUsedAt || '0' |
|
|
}) |
|
|
} |
|
|
} |
|
|
|
|
|
return availableAccounts |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async _isAccountAvailable(accountId, accountType) { |
|
|
try { |
|
|
if (accountType === 'openai') { |
|
|
const account = await openaiAccountService.getAccount(accountId) |
|
|
if ( |
|
|
!account || |
|
|
!account.isActive || |
|
|
account.status === 'error' || |
|
|
account.status === 'unauthorized' |
|
|
) { |
|
|
return false |
|
|
} |
|
|
const readiness = await this._ensureAccountReadyForScheduling(account, accountId, { |
|
|
sanitized: false |
|
|
}) |
|
|
|
|
|
if (!readiness.canUse) { |
|
|
if (readiness.reason === 'rate_limited') { |
|
|
logger.debug( |
|
|
`🚫 OpenAI account ${accountId} still rate limited when checking availability` |
|
|
) |
|
|
} else { |
|
|
logger.info(`🚫 OpenAI account ${accountId} is not schedulable`) |
|
|
} |
|
|
return false |
|
|
} |
|
|
|
|
|
return true |
|
|
} else if (accountType === 'openai-responses') { |
|
|
const account = await openaiResponsesAccountService.getAccount(accountId) |
|
|
if ( |
|
|
!account || |
|
|
(account.isActive !== true && account.isActive !== 'true') || |
|
|
account.status === 'error' || |
|
|
account.status === 'unauthorized' |
|
|
) { |
|
|
return false |
|
|
} |
|
|
|
|
|
if (!this._isSchedulable(account.schedulable)) { |
|
|
logger.info(`🚫 OpenAI-Responses account ${accountId} is not schedulable`) |
|
|
return false |
|
|
} |
|
|
|
|
|
if (openaiResponsesAccountService.isSubscriptionExpired(account)) { |
|
|
logger.info(`🚫 OpenAI-Responses account ${accountId} subscription expired`) |
|
|
return false |
|
|
} |
|
|
|
|
|
const isRateLimitCleared = |
|
|
await openaiResponsesAccountService.checkAndClearRateLimit(accountId) |
|
|
return !this._isRateLimited(account.rateLimitStatus) || isRateLimitCleared |
|
|
} |
|
|
return false |
|
|
} catch (error) { |
|
|
logger.warn(`⚠️ Failed to check account availability: ${accountId}`, error) |
|
|
return false |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async _getSessionMapping(sessionHash) { |
|
|
const client = redis.getClientSafe() |
|
|
const mappingData = await client.get(`${this.SESSION_MAPPING_PREFIX}${sessionHash}`) |
|
|
|
|
|
if (mappingData) { |
|
|
try { |
|
|
return JSON.parse(mappingData) |
|
|
} catch (error) { |
|
|
logger.warn('⚠️ Failed to parse session mapping:', error) |
|
|
return null |
|
|
} |
|
|
} |
|
|
|
|
|
return null |
|
|
} |
|
|
|
|
|
|
|
|
async _setSessionMapping(sessionHash, accountId, accountType) { |
|
|
const client = redis.getClientSafe() |
|
|
const mappingData = JSON.stringify({ accountId, accountType }) |
|
|
|
|
|
const appConfig = require('../../config/config') |
|
|
const ttlHours = appConfig.session?.stickyTtlHours || 1 |
|
|
const ttlSeconds = Math.max(1, Math.floor(ttlHours * 60 * 60)) |
|
|
await client.setex(`${this.SESSION_MAPPING_PREFIX}${sessionHash}`, ttlSeconds, mappingData) |
|
|
} |
|
|
|
|
|
|
|
|
async _deleteSessionMapping(sessionHash) { |
|
|
const client = redis.getClientSafe() |
|
|
await client.del(`${this.SESSION_MAPPING_PREFIX}${sessionHash}`) |
|
|
} |
|
|
|
|
|
|
|
|
async _extendSessionMappingTTL(sessionHash) { |
|
|
try { |
|
|
const client = redis.getClientSafe() |
|
|
const key = `${this.SESSION_MAPPING_PREFIX}${sessionHash}` |
|
|
const remainingTTL = await client.ttl(key) |
|
|
|
|
|
if (remainingTTL === -2) { |
|
|
return false |
|
|
} |
|
|
if (remainingTTL === -1) { |
|
|
return true |
|
|
} |
|
|
|
|
|
const appConfig = require('../../config/config') |
|
|
const ttlHours = appConfig.session?.stickyTtlHours || 1 |
|
|
const renewalThresholdMinutes = appConfig.session?.renewalThresholdMinutes || 0 |
|
|
if (!renewalThresholdMinutes) { |
|
|
return true |
|
|
} |
|
|
|
|
|
const fullTTL = Math.max(1, Math.floor(ttlHours * 60 * 60)) |
|
|
const threshold = Math.max(0, Math.floor(renewalThresholdMinutes * 60)) |
|
|
|
|
|
if (remainingTTL < threshold) { |
|
|
await client.expire(key, fullTTL) |
|
|
logger.debug( |
|
|
`🔄 Renewed unified OpenAI session TTL: ${sessionHash} (was ${Math.round(remainingTTL / 60)}m, renewed to ${ttlHours}h)` |
|
|
) |
|
|
} else { |
|
|
logger.debug( |
|
|
`✅ Unified OpenAI session TTL sufficient: ${sessionHash} (remaining ${Math.round(remainingTTL / 60)}m)` |
|
|
) |
|
|
} |
|
|
return true |
|
|
} catch (error) { |
|
|
logger.error('❌ Failed to extend unified OpenAI session TTL:', error) |
|
|
return false |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async markAccountRateLimited(accountId, accountType, sessionHash = null, resetsInSeconds = null) { |
|
|
try { |
|
|
if (accountType === 'openai') { |
|
|
await openaiAccountService.setAccountRateLimited(accountId, true, resetsInSeconds) |
|
|
} else if (accountType === 'openai-responses') { |
|
|
|
|
|
const duration = resetsInSeconds ? Math.ceil(resetsInSeconds / 60) : null |
|
|
await openaiResponsesAccountService.markAccountRateLimited(accountId, duration) |
|
|
|
|
|
|
|
|
await openaiResponsesAccountService.updateAccount(accountId, { |
|
|
schedulable: 'false', |
|
|
rateLimitResetAt: resetsInSeconds |
|
|
? new Date(Date.now() + resetsInSeconds * 1000).toISOString() |
|
|
: new Date(Date.now() + 3600000).toISOString() |
|
|
}) |
|
|
} |
|
|
|
|
|
|
|
|
if (sessionHash) { |
|
|
await this._deleteSessionMapping(sessionHash) |
|
|
} |
|
|
|
|
|
return { success: true } |
|
|
} catch (error) { |
|
|
logger.error( |
|
|
`❌ Failed to mark account as rate limited: ${accountId} (${accountType})`, |
|
|
error |
|
|
) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async markAccountUnauthorized( |
|
|
accountId, |
|
|
accountType, |
|
|
sessionHash = null, |
|
|
reason = 'OpenAI账号认证失败(401错误)' |
|
|
) { |
|
|
try { |
|
|
if (accountType === 'openai') { |
|
|
await openaiAccountService.markAccountUnauthorized(accountId, reason) |
|
|
} else if (accountType === 'openai-responses') { |
|
|
await openaiResponsesAccountService.markAccountUnauthorized(accountId, reason) |
|
|
} else { |
|
|
logger.warn( |
|
|
`⚠️ Unsupported account type ${accountType} when marking unauthorized for account ${accountId}` |
|
|
) |
|
|
return { success: false } |
|
|
} |
|
|
|
|
|
if (sessionHash) { |
|
|
await this._deleteSessionMapping(sessionHash) |
|
|
} |
|
|
|
|
|
return { success: true } |
|
|
} catch (error) { |
|
|
logger.error( |
|
|
`❌ Failed to mark account as unauthorized: ${accountId} (${accountType})`, |
|
|
error |
|
|
) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async removeAccountRateLimit(accountId, accountType) { |
|
|
try { |
|
|
if (accountType === 'openai') { |
|
|
await openaiAccountService.setAccountRateLimited(accountId, false) |
|
|
} else if (accountType === 'openai-responses') { |
|
|
|
|
|
await openaiResponsesAccountService.updateAccount(accountId, { |
|
|
rateLimitedAt: '', |
|
|
rateLimitStatus: '', |
|
|
rateLimitResetAt: '', |
|
|
status: 'active', |
|
|
errorMessage: '', |
|
|
schedulable: 'true' |
|
|
}) |
|
|
logger.info(`✅ Rate limit cleared for OpenAI-Responses account ${accountId}`) |
|
|
} |
|
|
|
|
|
return { success: true } |
|
|
} catch (error) { |
|
|
logger.error( |
|
|
`❌ Failed to remove rate limit for account: ${accountId} (${accountType})`, |
|
|
error |
|
|
) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async isAccountRateLimited(accountId) { |
|
|
try { |
|
|
const account = await openaiAccountService.getAccount(accountId) |
|
|
if (!account) { |
|
|
return false |
|
|
} |
|
|
|
|
|
if (this._isRateLimited(account.rateLimitStatus)) { |
|
|
|
|
|
if (account.rateLimitResetAt) { |
|
|
const resetTime = new Date(account.rateLimitResetAt).getTime() |
|
|
const now = Date.now() |
|
|
const isStillLimited = now < resetTime |
|
|
|
|
|
|
|
|
if (!isStillLimited) { |
|
|
logger.info(`✅ Auto-clearing rate limit for account ${accountId} (reset time reached)`) |
|
|
await openaiAccountService.setAccountRateLimited(accountId, false) |
|
|
return false |
|
|
} |
|
|
|
|
|
return isStillLimited |
|
|
} |
|
|
|
|
|
|
|
|
if (account.rateLimitedAt) { |
|
|
const limitedAt = new Date(account.rateLimitedAt).getTime() |
|
|
const now = Date.now() |
|
|
const limitDuration = 60 * 60 * 1000 |
|
|
return now < limitedAt + limitDuration |
|
|
} |
|
|
} |
|
|
return false |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to check rate limit status: ${accountId}`, error) |
|
|
return false |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async selectAccountFromGroup(groupId, sessionHash = null, requestedModel = null) { |
|
|
try { |
|
|
|
|
|
const group = await accountGroupService.getGroup(groupId) |
|
|
if (!group) { |
|
|
const error = new Error(`Group ${groupId} not found`) |
|
|
error.statusCode = 404 |
|
|
throw error |
|
|
} |
|
|
|
|
|
if (group.platform !== 'openai') { |
|
|
const error = new Error(`Group ${group.name} is not an OpenAI group`) |
|
|
error.statusCode = 400 |
|
|
throw error |
|
|
} |
|
|
|
|
|
logger.info(`👥 Selecting account from OpenAI group: ${group.name}`) |
|
|
|
|
|
|
|
|
if (sessionHash) { |
|
|
const mappedAccount = await this._getSessionMapping(sessionHash) |
|
|
if (mappedAccount) { |
|
|
|
|
|
const isInGroup = await this._isAccountInGroup(mappedAccount.accountId, groupId) |
|
|
if (isInGroup) { |
|
|
const isAvailable = await this._isAccountAvailable( |
|
|
mappedAccount.accountId, |
|
|
mappedAccount.accountType |
|
|
) |
|
|
if (isAvailable) { |
|
|
|
|
|
await this._extendSessionMappingTTL(sessionHash) |
|
|
logger.info( |
|
|
`🎯 Using sticky session account from group: ${mappedAccount.accountId} (${mappedAccount.accountType})` |
|
|
) |
|
|
|
|
|
await openaiAccountService.recordUsage(mappedAccount.accountId, 0) |
|
|
return mappedAccount |
|
|
} |
|
|
} |
|
|
|
|
|
await this._deleteSessionMapping(sessionHash) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
const memberIds = await accountGroupService.getGroupMembers(groupId) |
|
|
if (memberIds.length === 0) { |
|
|
const error = new Error(`Group ${group.name} has no members`) |
|
|
error.statusCode = 402 |
|
|
throw error |
|
|
} |
|
|
|
|
|
|
|
|
const availableAccounts = [] |
|
|
for (const memberId of memberIds) { |
|
|
const account = await openaiAccountService.getAccount(memberId) |
|
|
if (account && account.isActive && account.status !== 'error') { |
|
|
const readiness = await this._ensureAccountReadyForScheduling(account, account.id, { |
|
|
sanitized: false |
|
|
}) |
|
|
|
|
|
if (!readiness.canUse) { |
|
|
if (readiness.reason === 'rate_limited') { |
|
|
logger.debug( |
|
|
`⏭️ Skipping group member OpenAI account ${account.name} - still rate limited` |
|
|
) |
|
|
} else { |
|
|
logger.debug( |
|
|
`⏭️ Skipping group member OpenAI account ${account.name} - not schedulable` |
|
|
) |
|
|
} |
|
|
continue |
|
|
} |
|
|
|
|
|
|
|
|
const isExpired = openaiAccountService.isTokenExpired(account) |
|
|
if (isExpired && !account.refreshToken) { |
|
|
logger.warn( |
|
|
`⚠️ Group member OpenAI account ${account.name} token expired and no refresh token available` |
|
|
) |
|
|
continue |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if (requestedModel && account.supportedModels && account.supportedModels.length > 0) { |
|
|
const modelSupported = account.supportedModels.includes(requestedModel) |
|
|
if (!modelSupported) { |
|
|
logger.debug( |
|
|
`⏭️ Skipping group member OpenAI account ${account.name} - doesn't support model ${requestedModel}` |
|
|
) |
|
|
continue |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
availableAccounts.push({ |
|
|
...account, |
|
|
accountId: account.id, |
|
|
accountType: 'openai', |
|
|
priority: parseInt(account.priority) || 50, |
|
|
lastUsedAt: account.lastUsedAt || '0' |
|
|
}) |
|
|
} |
|
|
} |
|
|
|
|
|
if (availableAccounts.length === 0) { |
|
|
const error = new Error(`No available accounts in group ${group.name}`) |
|
|
error.statusCode = 402 |
|
|
throw error |
|
|
} |
|
|
|
|
|
|
|
|
const sortedAccounts = availableAccounts.sort((a, b) => { |
|
|
const aLastUsed = new Date(a.lastUsedAt || 0).getTime() |
|
|
const bLastUsed = new Date(b.lastUsedAt || 0).getTime() |
|
|
return aLastUsed - bLastUsed |
|
|
}) |
|
|
|
|
|
|
|
|
const selectedAccount = sortedAccounts[0] |
|
|
|
|
|
|
|
|
if (sessionHash) { |
|
|
await this._setSessionMapping( |
|
|
sessionHash, |
|
|
selectedAccount.accountId, |
|
|
selectedAccount.accountType |
|
|
) |
|
|
logger.info( |
|
|
`🎯 Created new sticky session mapping from group: ${selectedAccount.name} (${selectedAccount.accountId})` |
|
|
) |
|
|
} |
|
|
|
|
|
logger.info( |
|
|
`🎯 Selected account from group: ${selectedAccount.name} (${selectedAccount.accountId})` |
|
|
) |
|
|
|
|
|
|
|
|
await openaiAccountService.recordUsage(selectedAccount.accountId, 0) |
|
|
|
|
|
return { |
|
|
accountId: selectedAccount.accountId, |
|
|
accountType: selectedAccount.accountType |
|
|
} |
|
|
} catch (error) { |
|
|
logger.error(`❌ Failed to select account from group ${groupId}:`, error) |
|
|
throw error |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async _isAccountInGroup(accountId, groupId) { |
|
|
const members = await accountGroupService.getGroupMembers(groupId) |
|
|
return members.includes(accountId) |
|
|
} |
|
|
|
|
|
|
|
|
async updateAccountLastUsed(accountId, accountType) { |
|
|
try { |
|
|
if (accountType === 'openai') { |
|
|
await openaiAccountService.updateAccount(accountId, { |
|
|
lastUsedAt: new Date().toISOString() |
|
|
}) |
|
|
} |
|
|
} catch (error) { |
|
|
logger.warn(`⚠️ Failed to update last used time for account ${accountId}:`, error) |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
module.exports = new UnifiedOpenAIScheduler() |
|
|
|