Spaces:
Runtime error
Runtime error
File size: 13,368 Bytes
cd8bd0a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 | /**
* Proxy Egress IP visibility.
*
* The proxy logs already capture the INBOUND client IP (x-forwarded-for), but
* NOT the OUTBOUND/egress IP β the address the upstream actually sees. For
* rotating providers (codex/openai) this is critical: when several accounts
* egress through the SAME IP at high volume, the provider flags it as anomaly
* and revokes the tokens ("Your authentication token has been invalidated").
*
* This module resolves the real egress IP (via an echo-IP service through the
* resolved proxy/dispatcher) and detects same-rotation-group accounts sharing
* an egress IP, so the operator can confirm exactly which IP each account is
* entering and leaving by.
*/
import { request as undiciRequest } from "undici";
import { createProxyDispatcher, proxyConfigToUrl } from "@omniroute/open-sse/utils/proxyDispatcher.ts";
import { rotationGroupFor } from "@omniroute/open-sse/services/refreshSerializer.ts";
const EGRESS_ECHO_URL = "https://api64.ipify.org?format=json";
const EGRESS_PROBE_TIMEOUT_MS = 6000;
const EGRESS_CACHE_TTL_MS = 5 * 60 * 1000;
export interface EgressProbeResult {
ip: string | null;
latencyMs: number;
error?: string;
}
export type EgressProbe = (proxyUrl: string | null) => Promise<EgressProbeResult>;
const egressCache = new Map<string, { ip: string | null; at: number }>();
async function defaultEgressProbe(proxyUrl: string | null): Promise<EgressProbeResult> {
const start = Date.now();
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), EGRESS_PROBE_TIMEOUT_MS);
try {
const dispatcher = proxyUrl ? createProxyDispatcher(proxyUrl) : undefined;
const res = await undiciRequest(EGRESS_ECHO_URL, {
method: "GET",
dispatcher,
signal: controller.signal,
headersTimeout: EGRESS_PROBE_TIMEOUT_MS,
bodyTimeout: EGRESS_PROBE_TIMEOUT_MS,
});
const text = await res.body.text();
let ip: string | null = null;
try {
ip = (JSON.parse(text) as { ip?: string }).ip ?? null;
} catch {
// non-JSON body β leave ip null
}
return { ip, latencyMs: Date.now() - start };
} catch (error) {
return {
ip: null,
latencyMs: Date.now() - start,
error: error instanceof Error ? error.message : String(error),
};
} finally {
clearTimeout(timeout);
}
}
let probe: EgressProbe = defaultEgressProbe;
/** Test seam: override the network probe. */
export function _setEgressProbeForTests(fn: EgressProbe | null): void {
probe = fn ?? defaultEgressProbe;
}
export function clearEgressCache(): void {
egressCache.clear();
}
/**
* Synchronous read of the cached egress IP for a proxy URL (null = direct).
* Non-blocking β used by the request hot path to log the egress IP without an
* echo-IP round-trip. Returns null if not yet probed.
*/
export function getCachedEgressIp(proxyUrl: string | null): string | null {
const cached = egressCache.get(proxyUrl ?? "__direct__");
if (!cached) return null;
if (Date.now() - cached.at >= EGRESS_CACHE_TTL_MS) return null;
return cached.ip;
}
const warmingInFlight = new Set<string>();
/**
* Fire-and-forget: populate the egress cache for a proxy URL in the background
* so subsequent proxy log lines carry the real egress IP. Deduped per URL.
*/
export function warmEgressIp(proxyUrl: string | null): void {
const key = proxyUrl ?? "__direct__";
if (warmingInFlight.has(key) || getCachedEgressIp(proxyUrl) !== null) return;
warmingInFlight.add(key);
void resolveEgressIp(proxyUrl)
.catch(() => undefined)
.finally(() => warmingInFlight.delete(key));
}
/**
* Resolve the egress IP for a given proxy URL (null = direct/host IP).
* Cached per proxyUrl to avoid an echo-IP round-trip on every call.
*/
export async function resolveEgressIp(
proxyUrl: string | null,
opts: { cacheTtlMs?: number; force?: boolean } = {}
): Promise<EgressProbeResult & { cached: boolean }> {
const key = proxyUrl ?? "__direct__";
const ttl = opts.cacheTtlMs ?? EGRESS_CACHE_TTL_MS;
const cached = egressCache.get(key);
if (!opts.force && cached && Date.now() - cached.at < ttl) {
return { ip: cached.ip, latencyMs: 0, cached: true };
}
const result = await probe(proxyUrl);
egressCache.set(key, { ip: result.ip, at: Date.now() });
return { ...result, cached: false };
}
export interface ConnectionEgress {
connectionId: string;
provider: string;
account: string | null;
proxyLevel: string;
proxyHost: string | null;
egressIp: string | null;
error?: string;
}
export interface EgressSharingWarning {
egressIp: string;
rotationGroup: string;
connections: string[]; // connectionId/account labels sharing this IP within one rotation group
}
export interface EgressDiagnostic {
connections: ConnectionEgress[];
byEgressIp: Record<string, string[]>;
sharedWithinRotationGroup: EgressSharingWarning[];
}
/**
* PURE: group egress results by IP and flag IPs shared by β₯2 accounts of the
* SAME rotation group (codex+openai share one Auth0 family β the exact
* condition that triggers anomaly revocation). Direct/unknown IPs are reported
* but only same-group sharing is a warning.
*/
export function analyzeEgressSharing(connections: ConnectionEgress[]): {
byEgressIp: Record<string, string[]>;
sharedWithinRotationGroup: EgressSharingWarning[];
} {
const byEgressIp: Record<string, string[]> = {};
// ip -> rotationGroup -> labels
const byIpGroup = new Map<string, Map<string, string[]>>();
for (const c of connections) {
if (!c.egressIp) continue;
const label = c.account || c.connectionId;
(byEgressIp[c.egressIp] ??= []).push(label);
const group = rotationGroupFor(c.provider) || `provider:${c.provider}`;
let groups = byIpGroup.get(c.egressIp);
if (!groups) {
groups = new Map();
byIpGroup.set(c.egressIp, groups);
}
const list = groups.get(group) ?? [];
list.push(label);
groups.set(group, list);
}
const sharedWithinRotationGroup: EgressSharingWarning[] = [];
for (const [egressIp, groups] of byIpGroup) {
for (const [rotationGroup, labels] of groups) {
if (labels.length >= 2) {
sharedWithinRotationGroup.push({ egressIp, rotationGroup, connections: labels });
}
}
}
return { byEgressIp, sharedWithinRotationGroup };
}
/**
* Diagnose egress IPs for every OAuth connection: resolve each connection's
* proxy, probe the real egress IP, and flag same-rotation-group IP sharing.
*/
export async function diagnoseAllEgressIps(deps?: {
getConnections?: () => Promise<
Array<{ id: string; provider: string; name?: string; email?: string; authType?: string }>
>;
resolveProxy?: (
connectionId: string
) => Promise<{ proxy?: unknown; level?: string } | null>;
}): Promise<EgressDiagnostic> {
const getConnections =
deps?.getConnections ??
(async () => {
const { getProviderConnections } = await import("./localDb");
return (await getProviderConnections({ authType: "oauth" })) as Array<{
id: string;
provider: string;
name?: string;
email?: string;
}>;
});
const resolveProxy =
deps?.resolveProxy ??
(async (connectionId: string) => {
const { resolveProxyForConnection } = await import("./db/settings");
return resolveProxyForConnection(connectionId);
});
const conns = await getConnections();
const results: ConnectionEgress[] = [];
for (const c of conns) {
const resolved = await resolveProxy(c.id);
const proxyObj = (resolved?.proxy ?? null) as {
type?: string;
host?: string;
port?: number | string;
} | null;
const proxyUrl = proxyObj ? proxyConfigToUrl(proxyObj) : null;
const egress = await resolveEgressIp(proxyUrl);
results.push({
connectionId: c.id,
provider: c.provider,
account: c.email || c.name || c.id.slice(0, 8),
proxyLevel: resolved?.level || "direct",
proxyHost: proxyObj?.host ?? null,
egressIp: egress.ip,
...(egress.error ? { error: egress.error } : {}),
});
}
const { byEgressIp, sharedWithinRotationGroup } = analyzeEgressSharing(results);
return { connections: results, byEgressIp, sharedWithinRotationGroup };
}
export interface ProxyValidationResult {
proxyId: string;
host: string;
port: number | string;
alive: boolean;
egressIp: string | null;
latencyMs: number;
previousStatus: string | null;
newStatus: "active" | "error";
}
/**
* Validate every proxy in the registry by probing its real egress IP, and
* persist the result to `proxy_registry.status` (active/error). Combined with
* PROXY_ALIVE_PREDICATE in resolution, a dead proxy is automatically taken out
* of rotation β fixing the "all proxies marked active but actually dead" state
* that left codex accounts falling back to the shared host /64 IP.
*
* Deps are injectable for tests.
*/
export async function validateProxyPool(deps?: {
listProxies?: () => Promise<
Array<{ id: string; type: string; host: string; port: number | string; username?: string | null; password?: string | null; status?: string | null }>
>;
markStatus?: (id: string, status: string, meta: { latencyMs: number; egressIp: string | null }) => Promise<void>;
}): Promise<ProxyValidationResult[]> {
const listProxies =
deps?.listProxies ??
(async () => {
const { listProxies: real } = await import("./db/proxies");
return (await real({ includeSecrets: true })) as Array<{
id: string;
type: string;
host: string;
port: number | string;
username?: string | null;
password?: string | null;
status?: string | null;
}>;
});
const markStatus =
deps?.markStatus ??
(async (id: string, status: string) => {
const { updateProxy } = await import("./db/proxies");
await updateProxy(id, { status });
});
const proxies = await listProxies();
const report: ProxyValidationResult[] = [];
for (const p of proxies) {
const url = proxyConfigToUrl({
type: p.type,
host: p.host,
port: p.port,
username: p.username ?? undefined,
password: p.password ?? undefined,
});
const probe = await resolveEgressIp(url, { force: true });
const alive = !!probe.ip && !probe.error;
const newStatus: "active" | "error" = alive ? "active" : "error";
await markStatus(p.id, newStatus, { latencyMs: probe.latencyMs, egressIp: probe.ip });
report.push({
proxyId: p.id,
host: p.host,
port: p.port,
alive,
egressIp: probe.ip,
latencyMs: probe.latencyMs,
previousStatus: p.status ?? null,
newStatus,
});
}
return report;
}
export interface DistributionPlan {
assignments: Array<{ connectionId: string; account: string; proxyId: string }>;
unassigned: Array<{ connectionId: string; account: string }>;
sharingRisk: boolean;
note: string;
}
/**
* PURE: plan a 1-proxy-per-connection assignment so no two accounts of the same
* rotation group share an egress IP (the codex anomaly trigger). Default is
* strict 1:1 β extras are left UNASSIGNED (better unrouted than sharing an IP).
* allowSharing=true round-robins instead, flagging sharingRisk.
*/
export function planProxyDistribution(
connections: Array<{ id: string; account?: string }>,
liveProxyIds: string[],
opts: { allowSharing?: boolean } = {}
): DistributionPlan {
const assignments: DistributionPlan["assignments"] = [];
const unassigned: DistributionPlan["unassigned"] = [];
let sharingRisk = false;
connections.forEach((c, i) => {
const account = c.account || c.id.slice(0, 8);
if (liveProxyIds.length === 0) {
unassigned.push({ connectionId: c.id, account });
return;
}
if (opts.allowSharing) {
assignments.push({ connectionId: c.id, account, proxyId: liveProxyIds[i % liveProxyIds.length] });
} else if (i < liveProxyIds.length) {
assignments.push({ connectionId: c.id, account, proxyId: liveProxyIds[i] });
} else {
unassigned.push({ connectionId: c.id, account });
}
});
if (opts.allowSharing && liveProxyIds.length < connections.length) sharingRisk = true;
const note =
liveProxyIds.length === 0
? "No live proxies available β add working proxies before distributing."
: liveProxyIds.length < connections.length && !opts.allowSharing
? `Only ${liveProxyIds.length} live proxies for ${connections.length} accounts β ${unassigned.length} left unassigned (avoid shared-IP anomaly).`
: "1 distinct proxy per account.";
return { assignments, unassigned, sharingRisk, note };
}
/**
* Apply a distribution plan: assign each proxy to its connection (account scope).
*/
export async function applyProxyDistribution(
plan: DistributionPlan,
deps?: { assign?: (connectionId: string, proxyId: string) => Promise<void> }
): Promise<{ applied: number }> {
const assign =
deps?.assign ??
(async (connectionId: string, proxyId: string) => {
const { assignProxyToScope } = await import("./db/proxies");
await assignProxyToScope("account", connectionId, proxyId);
});
let applied = 0;
for (const a of plan.assignments) {
await assign(a.connectionId, a.proxyId);
applied++;
}
return { applied };
}
|