Spaces:
Paused
Paused
File size: 7,224 Bytes
4f2665c ba262d0 4f2665c df56b50 4f2665c ba262d0 4f2665c df56b50 4f2665c ba262d0 4f2665c ba262d0 4f2665c ba262d0 4f2665c ba262d0 4f2665c ba262d0 4f2665c ba262d0 4f2665c df56b50 4f2665c df56b50 4f2665c df56b50 4f2665c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 | /**
* Usage Refresher β background quota refresh for all accounts.
*
* - Periodically fetches official quota from Codex backend
* - Caches quota on AccountEntry for fast dashboard reads
* - Marks quota-exhausted accounts as rate_limited
* - Evaluates warning thresholds and updates warning store
* - Non-fatal: all errors log warnings but never crash the server
*/
import { CodexApi } from "../proxy/codex-api.js";
import { CodexApiError } from "../proxy/codex-types.js";
import { getConfig } from "../config.js";
import { jitter } from "../utils/jitter.js";
import { toQuota } from "./quota-utils.js";
import {
evaluateThresholds,
updateWarnings,
type QuotaWarning,
} from "./quota-warnings.js";
import type { AccountPool } from "./account-pool.js";
import type { CookieJar } from "../proxy/cookie-jar.js";
import type { ProxyPool } from "../proxy/proxy-pool.js";
import type { UsageStatsStore } from "./usage-stats.js";
/** Check if a CodexApiError indicates the account is banned/suspended (non-CF 403). */
function isBanError(err: unknown): boolean {
if (!(err instanceof CodexApiError)) return false;
if (err.status !== 403) return false;
// Cloudflare challenge pages contain cf_chl or are HTML β not a ban
const body = err.body.toLowerCase();
if (body.includes("cf_chl") || body.includes("<!doctype") || body.includes("<html")) return false;
return true;
}
/** Check if a CodexApiError is a 401 token invalidation. */
function isTokenInvalidError(err: unknown): boolean {
if (!(err instanceof CodexApiError)) return false;
return err.status === 401;
}
const INITIAL_DELAY_MS = 3_000; // 3s after startup
let _refreshTimer: ReturnType<typeof setTimeout> | null = null;
let _accountPool: AccountPool | null = null;
let _cookieJar: CookieJar | null = null;
let _proxyPool: ProxyPool | null = null;
let _usageStats: UsageStatsStore | null = null;
async function fetchQuotaForAllAccounts(
pool: AccountPool,
cookieJar: CookieJar,
proxyPool: ProxyPool | null,
): Promise<void> {
if (!pool.isAuthenticated()) return;
const entries = pool.getAllEntries().filter((e) => e.status === "active" || e.status === "rate_limited" || e.status === "banned");
if (entries.length === 0) return;
const config = getConfig();
const thresholds = config.quota.warning_thresholds;
console.log(`[QuotaRefresh] Refreshing quota for ${entries.length} active/rate-limited/banned account(s)`);
const results = await Promise.allSettled(
entries.map(async (entry) => {
const proxyUrl = proxyPool?.resolveProxyUrl(entry.id);
const api = new CodexApi(entry.token, entry.accountId, cookieJar, entry.id, proxyUrl);
const usage = await api.getUsage();
const quota = toQuota(usage);
// Auto-recover banned accounts that respond successfully
if (entry.status === "banned") {
pool.markStatus(entry.id, "active");
console.log(`[QuotaRefresh] Account ${entry.id} (${entry.email ?? "?"}) unbanned β quota fetch succeeded`);
}
// Cache quota on the account
pool.updateCachedQuota(entry.id, quota);
// Sync rate limit window
const resetAt = usage.rate_limit.primary_window?.reset_at ?? null;
const windowSec = usage.rate_limit.primary_window?.limit_window_seconds ?? null;
pool.syncRateLimitWindow(entry.id, resetAt, windowSec);
// Mark exhausted if limit reached (primary or secondary)
if (config.quota.skip_exhausted) {
const primaryExhausted = quota.rate_limit.limit_reached;
const secondaryExhausted = quota.secondary_rate_limit?.limit_reached ?? false;
if (primaryExhausted || secondaryExhausted) {
const exhaustResetAt = primaryExhausted
? quota.rate_limit.reset_at
: quota.secondary_rate_limit?.reset_at ?? null;
pool.markQuotaExhausted(entry.id, exhaustResetAt);
console.log(`[QuotaRefresh] Account ${entry.id} (${entry.email ?? "?"}) quota exhausted β marked rate_limited`);
}
}
// Evaluate warning thresholds
const warnings: QuotaWarning[] = [];
const pw = evaluateThresholds(
entry.id,
entry.email,
quota.rate_limit.used_percent,
quota.rate_limit.reset_at,
"primary",
thresholds.primary,
);
if (pw) warnings.push(pw);
const sw = evaluateThresholds(
entry.id,
entry.email,
quota.secondary_rate_limit?.used_percent ?? null,
quota.secondary_rate_limit?.reset_at ?? null,
"secondary",
thresholds.secondary,
);
if (sw) warnings.push(sw);
updateWarnings(entry.id, warnings);
}),
);
let succeeded = 0;
for (let i = 0; i < results.length; i++) {
const r = results[i];
if (r.status === "fulfilled") {
succeeded++;
} else {
const entry = entries[i];
const msg = r.reason instanceof Error ? r.reason.message : String(r.reason);
// Detect banned accounts (non-CF 403)
if (isBanError(r.reason)) {
pool.markStatus(entry.id, "banned");
console.warn(`[QuotaRefresh] Account ${entry.id} (${entry.email ?? "?"}) banned β 403 from upstream`);
} else if (isTokenInvalidError(r.reason)) {
pool.markStatus(entry.id, "expired");
console.warn(`[QuotaRefresh] Account ${entry.id} (${entry.email ?? "?"}) token invalidated β 401 from upstream`);
} else {
console.warn(`[QuotaRefresh] Account ${entry.id} quota fetch failed: ${msg}`);
}
}
}
console.log(`[QuotaRefresh] Done: ${succeeded}/${entries.length} succeeded`);
// Record usage snapshot for time-series history
if (_usageStats) {
try {
_usageStats.recordSnapshot(pool);
} catch (err) {
console.warn("[QuotaRefresh] Failed to record usage snapshot:", err instanceof Error ? err.message : err);
}
}
}
function scheduleNext(
pool: AccountPool,
cookieJar: CookieJar,
): void {
const config = getConfig();
const intervalMs = jitter(config.quota.refresh_interval_minutes * 60 * 1000, 0.15);
_refreshTimer = setTimeout(async () => {
try {
await fetchQuotaForAllAccounts(pool, cookieJar, _proxyPool);
} finally {
scheduleNext(pool, cookieJar);
}
}, intervalMs);
}
/**
* Start the background quota refresh loop.
*/
export function startQuotaRefresh(
accountPool: AccountPool,
cookieJar: CookieJar,
proxyPool?: ProxyPool,
usageStats?: UsageStatsStore,
): void {
_accountPool = accountPool;
_cookieJar = cookieJar;
_proxyPool = proxyPool ?? null;
_usageStats = usageStats ?? null;
_refreshTimer = setTimeout(async () => {
try {
await fetchQuotaForAllAccounts(accountPool, cookieJar, _proxyPool);
} finally {
scheduleNext(accountPool, cookieJar);
}
}, INITIAL_DELAY_MS);
const config = getConfig();
console.log(`[QuotaRefresh] Scheduled initial quota refresh in 3s (interval: ${config.quota.refresh_interval_minutes}min)`);
}
/**
* Stop the background refresh timer.
*/
export function stopQuotaRefresh(): void {
if (_refreshTimer) {
clearTimeout(_refreshTimer);
_refreshTimer = null;
console.log("[QuotaRefresh] Stopped quota refresh");
}
}
|