File size: 6,864 Bytes
5d0a52f
 
 
5dd5107
347f81b
 
 
 
 
 
5d0a52f
 
 
 
5dd5107
0d2f54c
5d0a52f
 
347f81b
 
 
 
 
 
 
5d0a52f
 
 
 
 
 
 
 
 
 
 
 
347f81b
5d0a52f
347f81b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5d0a52f
 
 
 
 
 
 
 
 
 
 
 
 
0d2f54c
5d0a52f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5dd5107
 
347f81b
5dd5107
 
 
 
 
5d0a52f
 
 
347f81b
85aec43
5dd5107
 
 
 
 
 
 
85aec43
5dd5107
85aec43
 
 
347f81b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0d2f54c
85aec43
347f81b
 
 
 
 
 
85aec43
 
5d0a52f
 
347f81b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5d0a52f
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
/**
 * RefreshScheduler β€” per-account JWT auto-refresh.
 * Schedules a refresh at `exp - margin` for each account.
 * Uses OAuth refresh_token instead of Codex CLI.
 *
 * Features:
 * - Exponential backoff (5 attempts: 5s β†’ 15s β†’ 45s β†’ 135s β†’ 300s)
 * - Permanent failure detection (invalid_grant / invalid_token)
 * - Recovery scheduling (10 min) for temporary failures
 * - Crash recovery: "refreshing" β†’ immediate retry, "expired" + refreshToken β†’ delayed retry
 */

import { getConfig } from "../config.js";
import { decodeJwtPayload } from "./jwt-utils.js";
import { refreshAccessToken } from "./oauth-pkce.js";
import { jitter, jitterInt } from "../utils/jitter.js";
import type { AccountPool } from "./account-pool.js";

/** Errors that indicate the refresh token itself is invalid (permanent failure). */
const PERMANENT_ERRORS = ["invalid_grant", "invalid_token", "access_denied"];

const MAX_ATTEMPTS = 5;
const BASE_DELAY_MS = 5_000;
const RECOVERY_DELAY_MS = 10 * 60 * 1000; // 10 minutes

export class RefreshScheduler {
  private timers: Map<string, ReturnType<typeof setTimeout>> = new Map();
  private pool: AccountPool;

  constructor(pool: AccountPool) {
    this.pool = pool;
    this.scheduleAll();
  }

  /** Schedule refresh for all accounts in the pool. */
  scheduleAll(): void {
    for (const entry of this.pool.getAllEntries()) {
      if (entry.status === "active") {
        this.scheduleOne(entry.id, entry.token);
      } else if (entry.status === "refreshing") {
        // Crash recovery: was mid-refresh when process died
        console.log(`[RefreshScheduler] Account ${entry.id}: recovering from 'refreshing' state`);
        this.doRefresh(entry.id);
      } else if (entry.status === "expired" && entry.refreshToken) {
        // Attempt recovery for expired accounts that still have a refresh token
        const delay = jitterInt(30_000, 0.3);
        console.log(`[RefreshScheduler] Account ${entry.id}: expired with refresh_token, recovery attempt in ${Math.round(delay / 1000)}s`);
        const timer = setTimeout(() => {
          this.timers.delete(entry.id);
          this.doRefresh(entry.id);
        }, delay);
        if (timer.unref) timer.unref();
        this.timers.set(entry.id, timer);
      } else if (entry.status === "expired" && !entry.refreshToken) {
        console.warn(`[RefreshScheduler] Account ${entry.id}: expired with no refresh_token. Re-login required at /`);
      }
    }
  }

  /** Schedule refresh for a single account. */
  scheduleOne(entryId: string, token: string): void {
    // Clear existing timer
    this.clearOne(entryId);

    const payload = decodeJwtPayload(token);
    if (!payload || typeof payload.exp !== "number") return;

    const config = getConfig();
    const refreshAt = payload.exp - jitter(config.auth.refresh_margin_seconds, 0.15);
    const delayMs = (refreshAt - Math.floor(Date.now() / 1000)) * 1000;

    if (delayMs <= 0) {
      // Already past refresh time β€” attempt refresh immediately
      this.doRefresh(entryId);
      return;
    }

    const timer = setTimeout(() => {
      this.timers.delete(entryId);
      this.doRefresh(entryId);
    }, delayMs);

    // Prevent the timer from keeping the process alive
    if (timer.unref) timer.unref();

    this.timers.set(entryId, timer);

    const expiresIn = Math.round(delayMs / 1000);
    console.log(
      `[RefreshScheduler] Account ${entryId}: refresh scheduled in ${expiresIn}s`,
    );
  }

  /** Cancel timer for one account. */
  clearOne(entryId: string): void {
    const timer = this.timers.get(entryId);
    if (timer) {
      clearTimeout(timer);
      this.timers.delete(entryId);
    }
  }

  /** Cancel all timers. */
  destroy(): void {
    for (const timer of this.timers.values()) {
      clearTimeout(timer);
    }
    this.timers.clear();
  }

  // ── Internal ────────────────────────────────────────────────────

  private async doRefresh(entryId: string): Promise<void> {
    const entry = this.pool.getEntry(entryId);
    if (!entry) return;

    if (!entry.refreshToken) {
      console.warn(
        `[RefreshScheduler] Account ${entryId} has no refresh_token, cannot auto-refresh. Re-login required at /`,
      );
      this.pool.markStatus(entryId, "expired");
      return;
    }

    console.log(`[RefreshScheduler] Refreshing account ${entryId} (${entry.email ?? "?"})`);
    this.pool.markStatus(entryId, "refreshing");

    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
      try {
        const tokens = await refreshAccessToken(entry.refreshToken);
        // Update token and refresh_token (if a new one was issued)
        this.pool.updateToken(
          entryId,
          tokens.access_token,
          tokens.refresh_token,
        );
        console.log(`[RefreshScheduler] Account ${entryId} refreshed successfully`);
        this.scheduleOne(entryId, tokens.access_token);
        return;
      } catch (err) {
        const msg = err instanceof Error ? err.message : String(err);

        // Check for permanent failures
        if (PERMANENT_ERRORS.some((e) => msg.toLowerCase().includes(e))) {
          console.error(`[RefreshScheduler] Permanent failure for ${entryId}: ${msg}`);
          this.pool.markStatus(entryId, "expired");
          return;
        }

        if (attempt < MAX_ATTEMPTS) {
          // Exponential backoff: 5s, 15s, 45s, 135s, 300s (capped)
          const backoff = Math.min(BASE_DELAY_MS * Math.pow(3, attempt - 1), 300_000);
          const retryDelay = jitterInt(backoff, 0.3);
          console.warn(
            `[RefreshScheduler] Attempt ${attempt}/${MAX_ATTEMPTS} failed for ${entryId}: ${msg}, retrying in ${Math.round(retryDelay / 1000)}s...`,
          );
          await new Promise((r) => setTimeout(r, retryDelay));
        } else {
          console.error(
            `[RefreshScheduler] All ${MAX_ATTEMPTS} attempts failed for ${entryId}: ${msg}`,
          );
          // Don't mark expired β€” schedule recovery attempt in 10 minutes
          this.pool.markStatus(entryId, "active"); // keep active so it can still be used
          this.scheduleRecovery(entryId);
        }
      }
    }
  }

  /**
   * Schedule a recovery refresh attempt after all retries are exhausted.
   * Gives the server time to recover from temporary issues.
   */
  private scheduleRecovery(entryId: string): void {
    const delay = jitterInt(RECOVERY_DELAY_MS, 0.2);
    console.log(
      `[RefreshScheduler] Recovery attempt for ${entryId} in ${Math.round(delay / 60000)}m`,
    );
    const timer = setTimeout(() => {
      this.timers.delete(entryId);
      this.doRefresh(entryId);
    }, delay);
    if (timer.unref) timer.unref();
    this.timers.set(entryId, timer);
  }
}