#!/usr/bin/env node const https = require('https'); const fs = require('fs'); const path = require('path'); const SOURCE_URL = 'https://raw.githubusercontent.com/ThinkInAIXYZ/PublicProviderConf/refs/heads/dev/dist/all.json'; const CONSTANTS_PATH = path.join(__dirname, '../../frontend/src/features/models/data/constants.ts'); const OUTPUT_PATH = path.join(__dirname, '../../frontend/src/features/models/data/providers.json'); const MODELS_JSON_PATH = path.join(__dirname, './models.json'); function fetchJSON(url) { return new Promise((resolve, reject) => { https.get(url, (res) => { let data = ''; res.on('data', (chunk) => { data += chunk; }); res.on('end', () => { try { resolve(JSON.parse(data)); } catch (e) { reject(new Error(`Failed to parse JSON: ${e.message}`)); } }); }).on('error', (err) => { reject(err); }); }); } function extractDeveloperIds(constantsPath) { const content = fs.readFileSync(constantsPath, 'utf8'); const match = content.match(/export const DEVELOPER_IDS = \[([\s\S]*?)\]/); if (!match) { throw new Error('Could not find DEVELOPER_IDS in constants.ts'); } const idsString = match[1]; const ids = idsString .split(',') .map(line => line.trim()) .filter(line => line.startsWith("'") || line.startsWith('"')) .map(line => line.replace(/^['"]|['"]$/g, '')); return ids; } function filterProviders(data, allowedIds) { if (!data.providers) { throw new Error('Invalid data structure: missing providers field'); } const filtered = {}; for (const [key, value] of Object.entries(data.providers)) { if (allowedIds.includes(value.id)) { filtered[key] = value; } } return { providers: filtered }; } function sortModelsByDate(data) { for (const provider of Object.values(data.providers)) { if (provider.models && Array.isArray(provider.models)) { provider.models.sort((a, b) => { const dateA = a.release_date ? new Date(a.release_date) : new Date(0); const dateB = b.release_date ? new Date(b.release_date) : new Date(0); return dateB - dateA; }); } } return data; } function mergeWithModelsJson(data, modelsJsonPath) { if (!fs.existsSync(modelsJsonPath)) { console.log('models.json does not exist, skipping merge'); return data; } console.log('Merging with models.json...'); const modelsJson = JSON.parse(fs.readFileSync(modelsJsonPath, 'utf8')); for (const [providerKey, models] of Object.entries(modelsJson)) { if (data.providers[providerKey]) { const existingProvider = data.providers[providerKey]; if (!existingProvider.models) { existingProvider.models = []; } const existingIds = new Set(existingProvider.models.map(m => m.id)); for (const model of models) { if (!existingIds.has(model.id)) { existingProvider.models.push(model); existingIds.add(model.id); } } } else { data.providers[providerKey] = { id: providerKey, models: models }; } } return data; } async function main() { try { console.log('Fetching model developers data from:', SOURCE_URL); const data = await fetchJSON(SOURCE_URL); console.log('Extracting allowed developer IDs from:', CONSTANTS_PATH); const allowedIds = extractDeveloperIds(CONSTANTS_PATH); console.log('Allowed developer IDs:', allowedIds); console.log('Filtering providers...'); const filtered = filterProviders(data, allowedIds); const providerCount = Object.keys(filtered.providers).length; console.log(`Filtered to ${providerCount} providers`); console.log('Merging with models.json...'); mergeWithModelsJson(filtered, MODELS_JSON_PATH); console.log('Sorting models by release date...'); sortModelsByDate(filtered); console.log('Writing to:', OUTPUT_PATH); fs.writeFileSync(OUTPUT_PATH, JSON.stringify(filtered, null, 2) + '\n'); console.log('Sync completed successfully!'); } catch (error) { console.error('Error during sync:', error.message); process.exit(1); } } main();