File size: 7,224 Bytes
4f2665c
 
 
 
 
 
 
 
 
 
 
ba262d0
4f2665c
 
 
 
 
 
 
 
 
 
 
df56b50
4f2665c
ba262d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4f2665c
 
 
 
 
 
df56b50
4f2665c
 
 
 
 
 
 
 
ba262d0
4f2665c
 
 
 
 
ba262d0
4f2665c
 
 
 
 
 
 
 
ba262d0
 
 
 
 
 
4f2665c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ba262d0
 
4f2665c
 
 
ba262d0
4f2665c
ba262d0
 
 
 
 
 
 
 
 
 
 
4f2665c
 
 
 
df56b50
 
 
 
 
 
 
 
 
4f2665c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
df56b50
4f2665c
 
 
 
df56b50
4f2665c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
/**
 * Usage Refresher β€” background quota refresh for all accounts.
 *
 * - Periodically fetches official quota from Codex backend
 * - Caches quota on AccountEntry for fast dashboard reads
 * - Marks quota-exhausted accounts as rate_limited
 * - Evaluates warning thresholds and updates warning store
 * - Non-fatal: all errors log warnings but never crash the server
 */

import { CodexApi } from "../proxy/codex-api.js";
import { CodexApiError } from "../proxy/codex-types.js";
import { getConfig } from "../config.js";
import { jitter } from "../utils/jitter.js";
import { toQuota } from "./quota-utils.js";
import {
  evaluateThresholds,
  updateWarnings,
  type QuotaWarning,
} from "./quota-warnings.js";
import type { AccountPool } from "./account-pool.js";
import type { CookieJar } from "../proxy/cookie-jar.js";
import type { ProxyPool } from "../proxy/proxy-pool.js";
import type { UsageStatsStore } from "./usage-stats.js";

/** Check if a CodexApiError indicates the account is banned/suspended (non-CF 403). */
function isBanError(err: unknown): boolean {
  if (!(err instanceof CodexApiError)) return false;
  if (err.status !== 403) return false;
  // Cloudflare challenge pages contain cf_chl or are HTML β€” not a ban
  const body = err.body.toLowerCase();
  if (body.includes("cf_chl") || body.includes("<!doctype") || body.includes("<html")) return false;
  return true;
}

/** Check if a CodexApiError is a 401 token invalidation. */
function isTokenInvalidError(err: unknown): boolean {
  if (!(err instanceof CodexApiError)) return false;
  return err.status === 401;
}

const INITIAL_DELAY_MS = 3_000; // 3s after startup

let _refreshTimer: ReturnType<typeof setTimeout> | null = null;
let _accountPool: AccountPool | null = null;
let _cookieJar: CookieJar | null = null;
let _proxyPool: ProxyPool | null = null;
let _usageStats: UsageStatsStore | null = null;

async function fetchQuotaForAllAccounts(
  pool: AccountPool,
  cookieJar: CookieJar,
  proxyPool: ProxyPool | null,
): Promise<void> {
  if (!pool.isAuthenticated()) return;

  const entries = pool.getAllEntries().filter((e) => e.status === "active" || e.status === "rate_limited" || e.status === "banned");
  if (entries.length === 0) return;

  const config = getConfig();
  const thresholds = config.quota.warning_thresholds;

  console.log(`[QuotaRefresh] Refreshing quota for ${entries.length} active/rate-limited/banned account(s)`);

  const results = await Promise.allSettled(
    entries.map(async (entry) => {
      const proxyUrl = proxyPool?.resolveProxyUrl(entry.id);
      const api = new CodexApi(entry.token, entry.accountId, cookieJar, entry.id, proxyUrl);
      const usage = await api.getUsage();
      const quota = toQuota(usage);

      // Auto-recover banned accounts that respond successfully
      if (entry.status === "banned") {
        pool.markStatus(entry.id, "active");
        console.log(`[QuotaRefresh] Account ${entry.id} (${entry.email ?? "?"}) unbanned β€” quota fetch succeeded`);
      }

      // Cache quota on the account
      pool.updateCachedQuota(entry.id, quota);

      // Sync rate limit window
      const resetAt = usage.rate_limit.primary_window?.reset_at ?? null;
      const windowSec = usage.rate_limit.primary_window?.limit_window_seconds ?? null;
      pool.syncRateLimitWindow(entry.id, resetAt, windowSec);

      // Mark exhausted if limit reached (primary or secondary)
      if (config.quota.skip_exhausted) {
        const primaryExhausted = quota.rate_limit.limit_reached;
        const secondaryExhausted = quota.secondary_rate_limit?.limit_reached ?? false;
        if (primaryExhausted || secondaryExhausted) {
          const exhaustResetAt = primaryExhausted
            ? quota.rate_limit.reset_at
            : quota.secondary_rate_limit?.reset_at ?? null;
          pool.markQuotaExhausted(entry.id, exhaustResetAt);
          console.log(`[QuotaRefresh] Account ${entry.id} (${entry.email ?? "?"}) quota exhausted β€” marked rate_limited`);
        }
      }

      // Evaluate warning thresholds
      const warnings: QuotaWarning[] = [];
      const pw = evaluateThresholds(
        entry.id,
        entry.email,
        quota.rate_limit.used_percent,
        quota.rate_limit.reset_at,
        "primary",
        thresholds.primary,
      );
      if (pw) warnings.push(pw);

      const sw = evaluateThresholds(
        entry.id,
        entry.email,
        quota.secondary_rate_limit?.used_percent ?? null,
        quota.secondary_rate_limit?.reset_at ?? null,
        "secondary",
        thresholds.secondary,
      );
      if (sw) warnings.push(sw);

      updateWarnings(entry.id, warnings);
    }),
  );

  let succeeded = 0;
  for (let i = 0; i < results.length; i++) {
    const r = results[i];
    if (r.status === "fulfilled") {
      succeeded++;
    } else {
      const entry = entries[i];
      const msg = r.reason instanceof Error ? r.reason.message : String(r.reason);

      // Detect banned accounts (non-CF 403)
      if (isBanError(r.reason)) {
        pool.markStatus(entry.id, "banned");
        console.warn(`[QuotaRefresh] Account ${entry.id} (${entry.email ?? "?"}) banned β€” 403 from upstream`);
      } else if (isTokenInvalidError(r.reason)) {
        pool.markStatus(entry.id, "expired");
        console.warn(`[QuotaRefresh] Account ${entry.id} (${entry.email ?? "?"}) token invalidated β€” 401 from upstream`);
      } else {
        console.warn(`[QuotaRefresh] Account ${entry.id} quota fetch failed: ${msg}`);
      }
    }
  }

  console.log(`[QuotaRefresh] Done: ${succeeded}/${entries.length} succeeded`);

  // Record usage snapshot for time-series history
  if (_usageStats) {
    try {
      _usageStats.recordSnapshot(pool);
    } catch (err) {
      console.warn("[QuotaRefresh] Failed to record usage snapshot:", err instanceof Error ? err.message : err);
    }
  }
}

function scheduleNext(
  pool: AccountPool,
  cookieJar: CookieJar,
): void {
  const config = getConfig();
  const intervalMs = jitter(config.quota.refresh_interval_minutes * 60 * 1000, 0.15);
  _refreshTimer = setTimeout(async () => {
    try {
      await fetchQuotaForAllAccounts(pool, cookieJar, _proxyPool);
    } finally {
      scheduleNext(pool, cookieJar);
    }
  }, intervalMs);
}

/**
 * Start the background quota refresh loop.
 */
export function startQuotaRefresh(
  accountPool: AccountPool,
  cookieJar: CookieJar,
  proxyPool?: ProxyPool,
  usageStats?: UsageStatsStore,
): void {
  _accountPool = accountPool;
  _cookieJar = cookieJar;
  _proxyPool = proxyPool ?? null;
  _usageStats = usageStats ?? null;

  _refreshTimer = setTimeout(async () => {
    try {
      await fetchQuotaForAllAccounts(accountPool, cookieJar, _proxyPool);
    } finally {
      scheduleNext(accountPool, cookieJar);
    }
  }, INITIAL_DELAY_MS);

  const config = getConfig();
  console.log(`[QuotaRefresh] Scheduled initial quota refresh in 3s (interval: ${config.quota.refresh_interval_minutes}min)`);
}

/**
 * Stop the background refresh timer.
 */
export function stopQuotaRefresh(): void {
  if (_refreshTimer) {
    clearTimeout(_refreshTimer);
    _refreshTimer = null;
    console.log("[QuotaRefresh] Stopped quota refresh");
  }
}