open-sse/utils/cacheControlPolicy.ts

/**
 * Cache Control Policy
 *
 * Determines when to preserve client-side prompt caching headers (cache_control)
 * vs. applying OmniRoute's own caching strategy.
 *
 * Client-side caching (e.g., Claude Code) should be preserved when:
 * 1. Client is Claude Code or similar caching-aware client
 * 2. Request will hit a deterministic target (single model or deterministic combo strategy)
 * 3. Provider supports prompt caching (Anthropic, Alibaba Qwen, etc.)
 */

import type { RoutingStrategyValue } from "../../src/shared/constants/routingStrategies";

/**
 * Cache control preservation modes
 */
export type CacheControlMode = "auto" | "always" | "never";

/**
 * Cache control settings from the database
 */
export interface CacheControlSettings {
  alwaysPreserveClientCache?: CacheControlMode;
}

/**
 * Cache metrics for tracking effectiveness
 */
export interface CacheControlMetrics {
  // Totals
  totalRequests: number;
  requestsWithCacheControl: number;

  // Token counts
  totalInputTokens: number;
  totalCachedTokens: number;
  totalCacheCreationTokens: number;

  // Savings
  tokensSaved: number;
  estimatedCostSaved: number;

  // Breakdowns
  byProvider: Record<
    string,
    {
      requests: number;
      inputTokens: number;
      cachedTokens: number;
      cacheCreationTokens: number;
    }
  >;
  byStrategy: Record<
    string,
    {
      requests: number;
      inputTokens: number;
      cachedTokens: number;
      cacheCreationTokens: number;
    }
  >;

  lastUpdated: string;
}

/**
 * Routing strategies that are deterministic (same request → same provider)
 */
const DETERMINISTIC_STRATEGIES: Set<RoutingStrategyValue> = new Set(["priority", "cost-optimized"]);

/**
 * Providers that support prompt caching
 */
const CACHING_PROVIDERS = new Set([
  "claude",
  "anthropic",
  "zai",
  "qwen",
  "deepseek",
  // #3088 — Xiaomi MiMo honors OpenAI-format cache_control breakpoints. Without
  // this entry, shouldPreserveCacheControl() returns false for Claude Code
  // clients and filterToOpenAIFormat() strips cache_control, so Xiaomi never
  // sees the cache hints and every request is a cache miss.
  "xiaomi-mimo",
  // #3955 — OpenAI / Codex / Azure-OpenAI use AUTOMATIC prefix caching: the longest
  // matching prefix of a request is cached upstream WITHOUT any explicit cache_control
  // markers. They must count as caching providers so the cache-aware compression guard
  // preserves the cacheable prefix (system prompt / earliest messages) instead of
  // rewriting it and forcing a cache miss. This also activates the intended
  // `prompt_cache_key` cache-routing hint for OpenAI in chatCore.
  "openai",
  "codex",
  "azure",
  // #2069 — Alibaba DashScope's OpenAI-compatible endpoints (alibaba /
  // alibaba-cn, upstream "alicode"/"alicode-intl") natively honor
  // `cache_control: {type:"ephemeral"}` breakpoints. Without these entries
  // shouldPreserveCacheControl() returns false for Claude Code clients and the
  // OpenAI-format translator strips cache_control, so DashScope never sees the
  // hints and every request is a cache miss.
  "alibaba",
  "alibaba-cn",
]);

/**
 * Providers that honor EXPLICIT `cache_control` breakpoints carried inside an
 * OpenAI-format request body (i.e. the markers must be passed THROUGH the
 * Claude→OpenAI translation instead of stripped).
 *
 * This is a strict subset of CACHING_PROVIDERS and deliberately excludes
 * `openai` / `codex` / `azure`: those use AUTOMATIC prefix caching (#3955) and
 * do NOT accept explicit `cache_control` fields in the request — forwarding the
 * markers there is meaningless at best and a 400 "unknown field" at worst, and
 * it broke the chatCore "strips cache markers for non-Claude providers" test.
 * Claude-format providers re-inject markers via prepareClaudeRequest, so they
 * are not listed here either.
 */
const OPENAI_FORMAT_CACHE_CONTROL_PROVIDERS = new Set([
  // #2069 — DashScope OpenAI-compatible endpoints accept ephemeral breakpoints.
  "alibaba",
  "alibaba-cn",
  // #3088 — Xiaomi MiMo honors OpenAI-format cache_control breakpoints.
  "xiaomi-mimo",
]);

/**
 * Whether `cache_control` markers should be PASSED THROUGH the OpenAI-format
 * translation for this provider (vs. stripped). Used to gate the request-side
 * passthrough so generic / implicit-cache OpenAI providers keep getting cleaned.
 */
export function providerHonorsOpenAIFormatCacheControl(
  provider: string | null | undefined
): boolean {
  if (!provider) return false;
  return OPENAI_FORMAT_CACHE_CONTROL_PROVIDERS.has(provider.toLowerCase());
}

/**
 * Detect if the client is Claude Code or another caching-aware client
 */
export function isClaudeCodeClient(userAgent: string | null | undefined): boolean {
  if (!userAgent) return false;
  const ua = userAgent.toLowerCase();

  // Claude Code user agents
  if (ua.includes("claude-code") || ua.includes("claude_code")) return true;
  if (ua.includes("claude-cli/")) return true;
  if (ua.includes("sdk-cli")) return true;
  if (ua.includes("anthropic") && ua.includes("cli")) return true;

  return false;
}

/**
 * Check if a provider supports prompt caching
 * Supports caching if:
 * 1. Provider is in the known caching providers list, OR
 * 2. Provider uses Claude protocol (detected via targetFormat)
 */
export function providerSupportsCaching(
  provider: string | null | undefined,
  targetFormat?: string | null
): boolean {
  if (!provider) return false;
  if (CACHING_PROVIDERS.has(provider.toLowerCase())) return true;
  // All Claude-protocol providers support prompt caching
  if (targetFormat === "claude") return true;
  return false;
}

/**
 * Check if a routing strategy is deterministic
 */
export function isDeterministicStrategy(
  strategy: RoutingStrategyValue | null | undefined
): boolean {
  if (!strategy) return false;
  return DETERMINISTIC_STRATEGIES.has(strategy);
}

/**
 * Determine if client-side cache_control headers should be preserved
 *
 * @param userAgent - User-Agent header from the request
 * @param isCombo - Whether this is a combo model
 * @param comboStrategy - The combo's routing strategy (if applicable)
 * @param targetProvider - The target provider for the request
 * @param settings - Cache control settings from database (optional)
 * @returns true if cache_control should be preserved, false if OmniRoute should manage it
 */
export function shouldPreserveCacheControl({
  userAgent,
  isCombo,
  comboStrategy,
  targetProvider,
  targetFormat,
  settings,
}: {
  userAgent: string | null | undefined;
  isCombo: boolean;
  comboStrategy?: RoutingStrategyValue | null;
  targetProvider: string | null | undefined;
  targetFormat?: string | null;
  settings?: CacheControlSettings;
}): boolean {
  // User override takes precedence
  if (settings?.alwaysPreserveClientCache === "always") {
    return true;
  }
  if (settings?.alwaysPreserveClientCache === "never") {
    return false;
  }

  // Auto mode: use automatic detection (existing logic)
  // Must be a caching-aware client
  if (!isClaudeCodeClient(userAgent)) {
    return false;
  }

  // Target provider must support caching
  if (!providerSupportsCaching(targetProvider, targetFormat)) {
    return false;
  }

  // Single model: always preserve (deterministic)
  if (!isCombo) {
    return true;
  }

  // Combo: only preserve if strategy is deterministic
  return isDeterministicStrategy(comboStrategy);
}

/**
 * Track cache control metrics for a request
 */
export function trackCacheMetrics({
  preserved,
  provider,
  strategy,
  metrics,
  inputTokens,
  cachedTokens,
  cacheCreationTokens,
}: {
  preserved: boolean;
  provider: string;
  strategy: string | null | undefined;
  metrics: CacheControlMetrics;
  inputTokens?: number;
  cachedTokens?: number;
  cacheCreationTokens?: number;
}): CacheControlMetrics {
  const now = new Date().toISOString();

  // Initialize metrics if empty
  if (!metrics) {
    metrics = {
      totalRequests: 0,
      requestsWithCacheControl: 0,
      totalInputTokens: 0,
      totalCachedTokens: 0,
      totalCacheCreationTokens: 0,
      tokensSaved: 0,
      estimatedCostSaved: 0,
      byProvider: {},
      byStrategy: {},
      lastUpdated: now,
    };
  }

  // Increment total requests
  metrics.totalRequests++;

  // Track token counts
  const input = inputTokens || 0;
  const cached = cachedTokens || 0;
  const creation = cacheCreationTokens || 0;

  metrics.totalInputTokens += input;
  metrics.totalCachedTokens += cached;
  metrics.totalCacheCreationTokens += creation;

  // Calculate tokens saved (cached tokens are reused, not charged)
  if (cached > 0) {
    metrics.tokensSaved += cached;
  }

  // Only track requests where cache_control was preserved
  if (preserved) {
    metrics.requestsWithCacheControl++;

    // Initialize provider tracking
    if (!metrics.byProvider[provider]) {
      metrics.byProvider[provider] = {
        requests: 0,
        inputTokens: 0,
        cachedTokens: 0,
        cacheCreationTokens: 0,
      };
    }
    metrics.byProvider[provider].requests++;
    metrics.byProvider[provider].inputTokens += input;
    metrics.byProvider[provider].cachedTokens += cached;
    metrics.byProvider[provider].cacheCreationTokens += creation;

    // Initialize strategy tracking
    if (strategy && !metrics.byStrategy[strategy]) {
      metrics.byStrategy[strategy] = {
        requests: 0,
        inputTokens: 0,
        cachedTokens: 0,
        cacheCreationTokens: 0,
      };
    }
    if (strategy) {
      metrics.byStrategy[strategy].requests++;
      metrics.byStrategy[strategy].inputTokens += input;
      metrics.byStrategy[strategy].cachedTokens += cached;
      metrics.byStrategy[strategy].cacheCreationTokens += creation;
    }
  }

  metrics.lastUpdated = now;
  return metrics;
}

/**
 * Record cache token usage and update metrics
 */
export function updateCacheTokenMetrics({
  metrics,
  provider,
  strategy,
  inputTokens,
  cachedTokens,
  cacheCreationTokens,
  costSaved,
}: {
  metrics: CacheControlMetrics;
  provider: string;
  strategy: string | null | undefined;
  inputTokens: number;
  cachedTokens: number;
  cacheCreationTokens: number;
  costSaved?: number;
}): CacheControlMetrics {
  metrics.totalCachedTokens += cachedTokens;
  metrics.totalCacheCreationTokens += cacheCreationTokens;
  metrics.totalInputTokens += inputTokens;

  // Cached tokens are reused (saved), creation tokens are new cache writes
  metrics.tokensSaved += cachedTokens;
  if (costSaved !== undefined) {
    metrics.estimatedCostSaved += costSaved;
  }

  // Update provider tracking
  if (metrics.byProvider[provider]) {
    metrics.byProvider[provider].cachedTokens += cachedTokens;
    metrics.byProvider[provider].cacheCreationTokens += cacheCreationTokens;
    metrics.byProvider[provider].inputTokens += inputTokens;
  }

  // Update strategy tracking
  if (strategy && metrics.byStrategy[strategy]) {
    metrics.byStrategy[strategy].cachedTokens += cachedTokens;
    metrics.byStrategy[strategy].cacheCreationTokens += cacheCreationTokens;
    metrics.byStrategy[strategy].inputTokens += inputTokens;
  }

  metrics.lastUpdated = new Date().toISOString();
  return metrics;
}