import { useState, useMemo, useCallback, useEffect } from 'react' import providersData from '../data/providers.json' import benchmarksData from '../data/benchmarks.json' import { ManagementPanel } from './components/ManagementPanel' interface Model { name: string category?: string type: string size_b?: number input_price_per_1m?: number output_price_per_1m?: number price_per_image?: number price_per_minute?: number audio_price_per_1m?: number price_per_1m_tokens_30d?: number currency: string capabilities?: string[] display_name?: string; hf_id?: string; canonical_id?: string; ollama_id?: string; hf_private?: boolean; size_source?: 'hf-total' | 'hf-config-estimate' | 'hf-card' | 'ollama' | 'manual' | 'benchmark' | 'openrouter'; provider?: Provider; complianceStatus?: string; } interface Provider { name: string url: string headquarters: string region: string gdpr_compliant: boolean eu_endpoints: boolean models: Model[] } type SortConfig = { key: string; direction: 'asc' | 'desc'; } | null; interface BenchmarkEntry { slug?: string; // LLMStats slug hf_id?: string; // HF leaderboard fullname name: string; // LLMStats benchmarks mmlu?: number; mmlu_pro?: number; gpqa?: number; human_eval?: number; math?: number; gsm8k?: number; mmmu?: number; hellaswag?: number; ifeval?: number; arc?: number; drop?: number; mbpp?: number; mgsm?: number; bbh?: number; // HF-specific hf_math_lvl5?: number; hf_musr?: number; hf_avg?: number; params_b?: number; // LiveBench (livebench.ai — contamination-free, monthly updated) lb_name?: string; lb_global?: number; lb_reasoning?: number; lb_coding?: number; lb_math?: number; lb_language?: number; lb_if?: number; lb_data_analysis?: number; // Chatbot Arena (lmarena.ai — real human preference votes) arena_name?: string; arena_elo?: number; // raw ELO score ~800-1500 (higher = better) arena_rank?: number; arena_votes?: number; // Aider code editing benchmark (aider.chat) aider_pass_rate?: number; // 0-1, first-pass success on 133 coding tasks // Artificial Analysis (artificialanalysis.ai) aa_id?: string; aa_name?: string; aa_slug?: string; aa_intelligence?: number; // 0-100 intelligence index aa_coding?: number; // 0-100 coding index aa_math?: number; // 0-100 math index aa_mmlu_pro?: number; aa_gpqa?: number; aa_livecodebench?: number; aa_hle?: number; aa_scicode?: number; aa_math_500?: number; aa_aime?: number; aa_tokens_per_s?: number; aa_latency_s?: number; // MTEB (Massive Text Embedding Benchmark) mteb_avg?: number; mteb_retrieval?: number; // OCR Benchmark ocr_avg?: number; } const normalizeName = (s?: string) => (s || '').toLowerCase().replace(/[-_.]/g, ' ').replace(/[^a-z0-9 ]/g, '').replace(/\s+/g, ' ').trim(); const EXCHANGE_RATE_EUR_TO_USD = 1.05 const CAP_ICON: Record = { // Input Modalities vision: '👁', // Image Input video: '🎬', // Video Input audio: '🎤', // Audio Input (ASR) files: '📄', // File/PDF Input // Output Modalities 'image-out': '🎨', // Image Generation 'video-out': '🎥', // Video Generation 'audio-out': '🔊', // Audio Generation (TTS) // Functional Capabilities tools: '🔧', reasoning: '💡', embedding: '🧩', 'eu-endpoint': '🇪🇺', } function App() { const [searchTerm, setSearchTerm] = useState('') const [selectedType, setSelectedType] = useState('all') const [selectedRegion, setSelectedRegion] = useState('all') const [sortConfig, setSortConfig] = useState({ key: 'price', direction: 'asc' }); const [groupByModel, setGroupByModel] = useState(false); const [showBenchmarks, setShowBenchmarks] = useState(false); const [showManagement, setShowManagement] = useState(false); const [dataVersion, setDataVersion] = useState(0); // Live data — initialized from bundled JSON, refreshed from /api/* when available const [liveProviders, setLiveProviders] = useState((providersData as any).providers); const [liveBenchmarks, setLiveBenchmarks] = useState(benchmarksData as BenchmarkEntry[]); const getGroupKey = useCallback((m: Model) => { return (m.hf_id || m.canonical_id || m.name || '').toLowerCase(); }, []); useEffect(() => { fetch('/api/data') .then(r => r.ok ? r.json() : null) .then(d => { if (d?.providers) setLiveProviders(d.providers); }) .catch(() => {}); fetch('/api/benchmarks') .then(r => r.ok ? r.json() : null) .then(d => { if (Array.isArray(d)) setLiveBenchmarks(d); }) .catch(() => {}); }, [dataVersion]); const fmtNum = (v?: number, decimals = 0) => (v !== undefined && Number.isFinite(v)) ? v.toFixed(decimals) : '–'; const fmtPct = (v?: number) => (v !== undefined && Number.isFinite(v)) ? `${(v * 100).toFixed(0)}%` : '–'; // Build benchmark lookup maps const { nameMap, hfIdMap } = useMemo(() => { const nameMap = new Map(); const hfIdMap = new Map(); for (const b of liveBenchmarks) { nameMap.set(normalizeName(b.name), b); if (b.slug) { const slugModel = b.slug.split('/').pop() || ''; if (slugModel) nameMap.set(normalizeName(slugModel), b); } if (b.hf_id) { hfIdMap.set(normalizeName(b.hf_id), b); const modelPart = b.hf_id.split('/').pop() || ''; const normModel = normalizeName(modelPart); nameMap.set(normModel, b); const words = normModel.split(' '); if (words.length > 1) nameMap.set(words.slice(1).join(' '), b); } if (b.lb_name) nameMap.set(normalizeName(b.lb_name), b); if (b.arena_name) nameMap.set(normalizeName(b.arena_name), b); if (b.aa_name) nameMap.set(normalizeName(b.aa_name), b); if (b.aa_slug) nameMap.set(normalizeName(b.aa_slug), b); } return { nameMap, hfIdMap }; }, [liveBenchmarks]); const findBenchmark = useCallback((modelName: string): BenchmarkEntry | undefined => { // Strip @region (e.g. @us-east-1) and :effort (e.g. :high) suffixes before normalizing const cleanName = modelName.replace(/@[^/]+$/, '').replace(/:[^/]+$/, ''); const norm = normalizeName(cleanName); // Direct HF ID match (for providers that use "org/model-id" format) let modelPart = ''; if (cleanName.includes('/')) { if (hfIdMap.has(norm)) return hfIdMap.get(norm)!; // Try model part only (after "/") modelPart = normalizeName(cleanName.split('/').pop() || ''); if (nameMap.has(modelPart)) return nameMap.get(modelPart)!; // Strip first word from model part — handles "Meta-Llama-..." vs "Llama-..." const modelWords = modelPart.split(' '); if (modelWords.length > 1) { const stripped = modelWords.slice(1).join(' '); if (nameMap.has(stripped)) return nameMap.get(stripped)!; } } if (nameMap.has(norm)) return nameMap.get(norm); // Longest startsWith match — handles date-suffixed variants like "claude-3-5-sonnet-20241022" let best: BenchmarkEntry | undefined; let bestLen = 0; for (const [key, val] of nameMap) { if (norm.startsWith(key)) { const rest = norm.slice(key.length); if ((rest === '' || /^ \d/.test(rest)) && key.length > bestLen) { best = val; bestLen = key.length; } } } if (best) return best; // Reverse prefix: benchmark key starts with provider name — handles cases where the // benchmark stores a base name longer than the provider's model ID. // Check both norm and modelPart (for versionless names like "anthropic/claude-haiku-4-5"). // Pick the highest lb_global among all matches (best variant of the model). let bestReverse: BenchmarkEntry | undefined; let bestReverseScore = -1; for (const [key, val] of nameMap) { const score = val.lb_global ?? 0; if ( (key.startsWith(norm + ' ') || (modelPart && key.startsWith(modelPart + ' '))) && score > bestReverseScore ) { bestReverse = val; bestReverseScore = score; } } return bestReverse; }, [nameMap, hfIdMap]); const handleDataUpdated = useCallback(() => { // Increment version to signal that a page reload would show fresh data setDataVersion((v) => v + 1); }, []); const getComplianceStatus = (provider: Provider, model?: Model) => { const hq = provider.headquarters.toLowerCase(); const isEU = provider.region === 'EU' || hq === 'germany' || hq === 'france' || hq === 'netherlands'; const isUS = hq === 'usa' || provider.region === 'US'; const isEEA = provider.region === 'EEA Equivalent' || hq === 'switzerland'; // Model-level override for OpenRouter or similar aggregators const hasEuEndpoint = model?.capabilities?.includes('eu-endpoint') || provider.eu_endpoints; if (isEU) return 'EU'; if (isEEA) return 'EEA'; if (isUS && hasEuEndpoint) return 'US/EU'; if (isUS) return 'US'; return 'Other'; }; const allModels = useMemo(() => { const rawModels: any[] = [] liveProviders.forEach((provider: Provider) => { provider.models.forEach((model) => { const status = getComplianceStatus(provider, model); let cleanName = model.name; // Strip provider/ prefix for all models if (cleanName.includes('/')) { cleanName = cleanName.split('/').pop() || cleanName; } rawModels.push({ ...model, name: cleanName, provider, complianceStatus: status }) }) }) // De-duplicate: filter out models with same name, provider, and price const seen = new Set(); const uniqueModels = rawModels.filter(m => { const key = `${m.provider?.name}|${m.name}|${m.input_price_per_1m}|${m.output_price_per_1m}`; if (seen.has(key)) return false; seen.add(key); return true; }); return uniqueModels; }, [liveProviders]) const filteredModels = useMemo(() => { return allModels.filter((model) => { const providerName = model.provider?.name || ''; const matchesSearch = model.name.toLowerCase().includes(searchTerm.toLowerCase()) || providerName.toLowerCase().includes(searchTerm.toLowerCase()) const caps = model.capabilities || []; const matchesType = selectedType === 'all' || model.type === selectedType || (selectedType === 'audio' && (caps.includes('audio') || caps.includes('audio-out'))) || (selectedType === 'vision' && (caps.includes('vision') || caps.includes('video'))) || (selectedType === 'chat' && model.type === 'chat'); const status = model.complianceStatus || 'Other'; let matchesRegion = selectedRegion === 'all' || status === selectedRegion; // US filter includes US/EU if (selectedRegion === 'US' && status === 'US/EU') matchesRegion = true; return matchesSearch && matchesType && matchesRegion }) }, [searchTerm, selectedType, selectedRegion, allModels]) const getNormalizedPriceUSD = (model: Model) => { const price = model.input_price_per_1m || model.price_per_image || model.price_per_minute || 0 return model.currency === 'EUR' ? price * EXCHANGE_RATE_EUR_TO_USD : price } const sortedModels = useMemo(() => { return [...filteredModels].sort((a, b) => { if (!sortConfig) return 0; let aValue: any; let bValue: any; switch (sortConfig.key) { case 'provider': aValue = a.provider.name; bValue = b.provider.name; break; case 'model': aValue = a.name; bValue = b.name; break; case 'size': aValue = a.size_b || 0; bValue = b.size_b || 0; break; case 'price': aValue = getNormalizedPriceUSD(a); bValue = getNormalizedPriceUSD(b); break; case 'output_price': aValue = a.output_price_per_1m ?? 0; bValue = b.output_price_per_1m ?? 0; break; case 'compliance': aValue = a.complianceStatus; bValue = b.complianceStatus; break; case 'mmlu': case 'gpqa': case 'human_eval': case 'math': case 'gsm8k': case 'mmmu': case 'ifeval': case 'bbh': case 'hf_math_lvl5': case 'hf_musr': case 'hf_avg': case 'lb_global': case 'lb_reasoning': case 'lb_coding': case 'lb_math': case 'lb_language': case 'lb_if': case 'lb_data_analysis': case 'arena_elo': case 'aider_pass_rate': case 'aa_intelligence': case 'aa_tokens_per_s': case 'mteb_avg': case 'mteb_retrieval': { try { const bA = findBenchmark(a.name); const bB = findBenchmark(b.name); aValue = bA?.[sortConfig.key as keyof BenchmarkEntry] as number ?? -1; bValue = bB?.[sortConfig.key as keyof BenchmarkEntry] as number ?? -1; } catch (e) { aValue = -1; bValue = -1; } break; } default: return 0; } if (aValue < bValue) return sortConfig.direction === 'asc' ? -1 : 1; if (aValue > bValue) return sortConfig.direction === 'asc' ? 1 : -1; return 0; }); }, [filteredModels, sortConfig]) const displayModels = useMemo(() => { if (!groupByModel) return sortedModels; const groups: Record = {}; const groupOrder: string[] = []; sortedModels.forEach(m => { const key = getGroupKey(m); if (!groups[key]) { groups[key] = []; groupOrder.push(key); } groups[key].push(m); }); const result: typeof sortedModels = []; groupOrder.forEach(key => { result.push(...groups[key]); }); return result; }, [sortedModels, groupByModel, getGroupKey]); const requestSort = (key: string) => { let direction: 'asc' | 'desc' = 'asc'; if (sortConfig && sortConfig.key === key && sortConfig.direction === 'asc') { direction = 'desc'; } setSortConfig({ key, direction }); }; const formatPrice = (price: number | undefined, currency: string) => { if (price === undefined) return '-' if (price === 0) return 'Free' const symbol = currency === 'USD' ? '$' : '€' return `${symbol}${price.toFixed(4)}` } const getSortIcon = (key: string) => { if (sortConfig?.key !== key) return '↕️'; return sortConfig.direction === 'asc' ? '🔼' : '🔽'; } return (

AI Provider Comparison

Analyze costs, data sovereignty, and model efficiency.

{dataVersion > 0 && ( ↻ data updated )} GitHub
{showManagement && ( setShowManagement(false)} onDataUpdated={handleDataUpdated} /> )}
setSearchTerm(e.target.value)} className="search-input" />
{showBenchmarks && <> } {displayModels.map((model, idx) => { const prev = displayModels[idx - 1]; const isGroupStart = groupByModel && ( idx === 0 || getGroupKey(prev) !== getGroupKey(model) ); const bm = findBenchmark(model.name); return ( {showBenchmarks && ( <> )} ) })}
requestSort('provider')} className="sortable">Provider {getSortIcon('provider')} requestSort('compliance')} className="sortable">Jurisdiction {getSortIcon('compliance')} requestSort('model')} className="sortable">Model {getSortIcon('model')} Caps requestSort('size')} className="sortable">Size (B) {getSortIcon('size')} requestSort('price')} className="sortable">Input Price (USD) {getSortIcon('price')} requestSort('output_price')} className="sortable">Output Price {getSortIcon('output_price')} requestSort('arena_elo')} className="sortable" title="Chatbot Arena ELO (human preference votes)">Arena ELO {getSortIcon('arena_elo')} requestSort('aider_pass_rate')} className="sortable" title="Aider code editing benchmark (pass rate, 133 tasks)">Aider {getSortIcon('aider_pass_rate')} requestSort('aa_intelligence')} className="sortable" title="Artificial Analysis Intelligence Index (0-100)">AA Intel {getSortIcon('aa_intelligence')} requestSort('aa_tokens_per_s')} className="sortable" title="Artificial Analysis Median Speed (Tokens per Second)">AA Speed {getSortIcon('aa_tokens_per_s')} requestSort('mteb_avg')} className="sortable" title="MTEB (Massive Text Embedding Benchmark) Average">MTEB {getSortIcon('mteb_avg')} requestSort('mteb_retrieval')} className="sortable" title="MTEB Retrieval Average">MTEB-Ret {getSortIcon('mteb_retrieval')} requestSort('lb_global')} className="sortable" title="LiveBench overall average (contamination-free)">LB {getSortIcon('lb_global')} requestSort('lb_math')} className="sortable" title="LiveBench Mathematics">LB-Math {getSortIcon('lb_math')} requestSort('lb_coding')} className="sortable" title="LiveBench Coding + Agentic Coding">LB-Code {getSortIcon('lb_coding')} requestSort('lb_reasoning')} className="sortable" title="LiveBench Reasoning">LB-Reas {getSortIcon('lb_reasoning')} requestSort('gpqa')} className="sortable" title="Graduate-level reasoning (GPQA)">GPQA {getSortIcon('gpqa')} requestSort('mmlu_pro')} className="sortable" title="MMLU-Pro knowledge">MMLU-PRO {getSortIcon('mmlu_pro')} requestSort('ifeval')} className="sortable" title="Instruction following (IFEval)">IFEval {getSortIcon('ifeval')} requestSort('bbh')} className="sortable" title="Big-Bench Hard reasoning">BBH {getSortIcon('bbh')} requestSort('hf_math_lvl5')} className="sortable" title="MATH Level 5 (HF leaderboard)">MATH L5 {getSortIcon('hf_math_lvl5')} requestSort('hf_musr')} className="sortable" title="Multi-step Soft Reasoning (HF leaderboard)">MUSR {getSortIcon('hf_musr')} requestSort('mmlu')} className="sortable" title="Classic MMLU (LLMStats)">MMLU {getSortIcon('mmlu')} requestSort('human_eval')} className="sortable" title="HumanEval coding (LLMStats)">HumanEval {getSortIcon('human_eval')}
{model.provider.name} {model.complianceStatus}
{model.display_name ?? model.name}
Type: {model.type}
{model.size_b &&
Size: {model.size_b}B
} {model.hf_id && ( )} {!model.hf_id && model.hf_private && (
HF: Proprietary API
)} {model.ollama_id && ( )} {model.capabilities && model.capabilities.length > 0 && (
Caps: {model.capabilities.join(', ')}
)} {bm?.ocr_avg !== undefined && (
OCR: {bm.ocr_avg.toFixed(1)} (olmOCR-bench)
)}
{model.type === 'embedding' && ( {CAP_ICON.embedding} )} {(model.capabilities || []).map((cap: string) => ( {CAP_ICON[cap] ?? cap} ))} {model.size_b ? `${model.size_b}B` : '-'}
{model.price_per_image !== undefined && !model.input_price_per_1m ? `$${model.price_per_image}/MP` : model.price_per_minute !== undefined ? `${formatPrice(model.price_per_minute, model.currency)}/min` : formatPrice(model.input_price_per_1m, model.currency)} {model.audio_price_per_1m !== undefined && (
{CAP_ICON.audio} {formatPrice(model.audio_price_per_1m, model.currency)}/M
)}
{model.price_per_image !== undefined && !model.output_price_per_1m ? '–' : model.price_per_minute !== undefined ? '–' : formatPrice(model.output_price_per_1m, model.currency)} {fmtNum(bm?.arena_elo)} {fmtPct(bm?.aider_pass_rate)} {fmtNum(bm?.aa_intelligence)} {fmtNum(bm?.aa_tokens_per_s)} {fmtNum(bm?.mteb_avg, 1)} {fmtNum(bm?.mteb_retrieval, 1)} {fmtPct(bm?.lb_global)} {fmtPct(bm?.lb_math)} {fmtPct(bm?.lb_coding)} {fmtPct(bm?.lb_reasoning)} {fmtPct(bm?.gpqa)} {fmtPct(bm?.mmlu_pro)} {fmtPct(bm?.ifeval)} {fmtPct(bm?.bbh)} {fmtPct(bm?.hf_math_lvl5)} {fmtPct(bm?.hf_musr)} {fmtPct(bm?.mmlu)} {fmtPct(bm?.human_eval)}
) } export default App