Spaces:
Paused
Paused
File size: 6,864 Bytes
5d0a52f 5dd5107 347f81b 5d0a52f 5dd5107 0d2f54c 5d0a52f 347f81b 5d0a52f 347f81b 5d0a52f 347f81b 5d0a52f 0d2f54c 5d0a52f 5dd5107 347f81b 5dd5107 5d0a52f 347f81b 85aec43 5dd5107 85aec43 5dd5107 85aec43 347f81b 0d2f54c 85aec43 347f81b 85aec43 5d0a52f 347f81b 5d0a52f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 | /**
* RefreshScheduler β per-account JWT auto-refresh.
* Schedules a refresh at `exp - margin` for each account.
* Uses OAuth refresh_token instead of Codex CLI.
*
* Features:
* - Exponential backoff (5 attempts: 5s β 15s β 45s β 135s β 300s)
* - Permanent failure detection (invalid_grant / invalid_token)
* - Recovery scheduling (10 min) for temporary failures
* - Crash recovery: "refreshing" β immediate retry, "expired" + refreshToken β delayed retry
*/
import { getConfig } from "../config.js";
import { decodeJwtPayload } from "./jwt-utils.js";
import { refreshAccessToken } from "./oauth-pkce.js";
import { jitter, jitterInt } from "../utils/jitter.js";
import type { AccountPool } from "./account-pool.js";
/** Errors that indicate the refresh token itself is invalid (permanent failure). */
const PERMANENT_ERRORS = ["invalid_grant", "invalid_token", "access_denied"];
const MAX_ATTEMPTS = 5;
const BASE_DELAY_MS = 5_000;
const RECOVERY_DELAY_MS = 10 * 60 * 1000; // 10 minutes
export class RefreshScheduler {
private timers: Map<string, ReturnType<typeof setTimeout>> = new Map();
private pool: AccountPool;
constructor(pool: AccountPool) {
this.pool = pool;
this.scheduleAll();
}
/** Schedule refresh for all accounts in the pool. */
scheduleAll(): void {
for (const entry of this.pool.getAllEntries()) {
if (entry.status === "active") {
this.scheduleOne(entry.id, entry.token);
} else if (entry.status === "refreshing") {
// Crash recovery: was mid-refresh when process died
console.log(`[RefreshScheduler] Account ${entry.id}: recovering from 'refreshing' state`);
this.doRefresh(entry.id);
} else if (entry.status === "expired" && entry.refreshToken) {
// Attempt recovery for expired accounts that still have a refresh token
const delay = jitterInt(30_000, 0.3);
console.log(`[RefreshScheduler] Account ${entry.id}: expired with refresh_token, recovery attempt in ${Math.round(delay / 1000)}s`);
const timer = setTimeout(() => {
this.timers.delete(entry.id);
this.doRefresh(entry.id);
}, delay);
if (timer.unref) timer.unref();
this.timers.set(entry.id, timer);
} else if (entry.status === "expired" && !entry.refreshToken) {
console.warn(`[RefreshScheduler] Account ${entry.id}: expired with no refresh_token. Re-login required at /`);
}
}
}
/** Schedule refresh for a single account. */
scheduleOne(entryId: string, token: string): void {
// Clear existing timer
this.clearOne(entryId);
const payload = decodeJwtPayload(token);
if (!payload || typeof payload.exp !== "number") return;
const config = getConfig();
const refreshAt = payload.exp - jitter(config.auth.refresh_margin_seconds, 0.15);
const delayMs = (refreshAt - Math.floor(Date.now() / 1000)) * 1000;
if (delayMs <= 0) {
// Already past refresh time β attempt refresh immediately
this.doRefresh(entryId);
return;
}
const timer = setTimeout(() => {
this.timers.delete(entryId);
this.doRefresh(entryId);
}, delayMs);
// Prevent the timer from keeping the process alive
if (timer.unref) timer.unref();
this.timers.set(entryId, timer);
const expiresIn = Math.round(delayMs / 1000);
console.log(
`[RefreshScheduler] Account ${entryId}: refresh scheduled in ${expiresIn}s`,
);
}
/** Cancel timer for one account. */
clearOne(entryId: string): void {
const timer = this.timers.get(entryId);
if (timer) {
clearTimeout(timer);
this.timers.delete(entryId);
}
}
/** Cancel all timers. */
destroy(): void {
for (const timer of this.timers.values()) {
clearTimeout(timer);
}
this.timers.clear();
}
// ββ Internal ββββββββββββββββββββββββββββββββββββββββββββββββββββ
private async doRefresh(entryId: string): Promise<void> {
const entry = this.pool.getEntry(entryId);
if (!entry) return;
if (!entry.refreshToken) {
console.warn(
`[RefreshScheduler] Account ${entryId} has no refresh_token, cannot auto-refresh. Re-login required at /`,
);
this.pool.markStatus(entryId, "expired");
return;
}
console.log(`[RefreshScheduler] Refreshing account ${entryId} (${entry.email ?? "?"})`);
this.pool.markStatus(entryId, "refreshing");
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
try {
const tokens = await refreshAccessToken(entry.refreshToken);
// Update token and refresh_token (if a new one was issued)
this.pool.updateToken(
entryId,
tokens.access_token,
tokens.refresh_token,
);
console.log(`[RefreshScheduler] Account ${entryId} refreshed successfully`);
this.scheduleOne(entryId, tokens.access_token);
return;
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
// Check for permanent failures
if (PERMANENT_ERRORS.some((e) => msg.toLowerCase().includes(e))) {
console.error(`[RefreshScheduler] Permanent failure for ${entryId}: ${msg}`);
this.pool.markStatus(entryId, "expired");
return;
}
if (attempt < MAX_ATTEMPTS) {
// Exponential backoff: 5s, 15s, 45s, 135s, 300s (capped)
const backoff = Math.min(BASE_DELAY_MS * Math.pow(3, attempt - 1), 300_000);
const retryDelay = jitterInt(backoff, 0.3);
console.warn(
`[RefreshScheduler] Attempt ${attempt}/${MAX_ATTEMPTS} failed for ${entryId}: ${msg}, retrying in ${Math.round(retryDelay / 1000)}s...`,
);
await new Promise((r) => setTimeout(r, retryDelay));
} else {
console.error(
`[RefreshScheduler] All ${MAX_ATTEMPTS} attempts failed for ${entryId}: ${msg}`,
);
// Don't mark expired β schedule recovery attempt in 10 minutes
this.pool.markStatus(entryId, "active"); // keep active so it can still be used
this.scheduleRecovery(entryId);
}
}
}
}
/**
* Schedule a recovery refresh attempt after all retries are exhausted.
* Gives the server time to recover from temporary issues.
*/
private scheduleRecovery(entryId: string): void {
const delay = jitterInt(RECOVERY_DELAY_MS, 0.2);
console.log(
`[RefreshScheduler] Recovery attempt for ${entryId} in ${Math.round(delay / 60000)}m`,
);
const timer = setTimeout(() => {
this.timers.delete(entryId);
this.doRefresh(entryId);
}, delay);
if (timer.unref) timer.unref();
this.timers.set(entryId, timer);
}
}
|