File size: 2,538 Bytes
5569a86 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | /**
* Probe every enabled model with a minimal request to find broken model IDs.
* Usage: npx tsx src/scripts/test-all-models.ts
*/
import { initDb, getDb } from '../db/index.js';
import { decrypt } from '../lib/crypto.js';
import { getProvider } from '../providers/index.js';
interface Row {
id: number;
platform: string;
model_id: string;
display_name: string;
}
interface Key {
encrypted_key: string;
iv: string;
auth_tag: string;
}
async function main() {
await initDb();
const db = getDb();
const models = await db.all<Row>(`
SELECT m.id, m.platform, m.model_id, m.display_name
FROM models m
WHERE m.enabled = 1
AND EXISTS (SELECT 1 FROM api_keys k WHERE k.platform = m.platform AND k.enabled = 1)
ORDER BY m.intelligence_rank, m.platform
`);
const results: { row: Row; ok: boolean; ms: number; error?: string; reply?: string }[] = [];
for (const row of models) {
const keyRow = await db.get<Key>(`
SELECT encrypted_key, iv, auth_tag FROM api_keys
WHERE platform = ? AND enabled = 1 ORDER BY id LIMIT 1
`, [row.platform]);
if (!keyRow) { results.push({ row, ok: false, ms: 0, error: 'no key' }); continue; }
const apiKey = decrypt(keyRow.encrypted_key, keyRow.iv, keyRow.auth_tag);
const provider = getProvider(row.platform as any);
if (!provider) { results.push({ row, ok: false, ms: 0, error: 'no provider' }); continue; }
const start = Date.now();
try {
const res = await provider.chatCompletion(apiKey, [{ role: 'user', content: 'hi' }], row.model_id, { max_tokens: 5 });
const raw = res.choices?.[0]?.message?.content;
const reply = typeof raw === 'string' ? raw.slice(0, 40) : '';
results.push({ row, ok: true, ms: Date.now() - start, reply });
} catch (err: any) {
results.push({ row, ok: false, ms: Date.now() - start, error: String(err?.message ?? err).slice(0, 200) });
}
}
console.log('\n=== Results ===\n');
const pad = (s: string, n: number) => s.length > n ? s.slice(0, n - 1) + '…' : s.padEnd(n);
for (const r of results) {
const status = r.ok ? '✓' : '✗';
console.log(`${status} ${pad(r.row.platform, 12)} ${pad(r.row.model_id, 52)} ${String(r.ms).padStart(5)}ms ${r.ok ? `"${r.reply}"` : r.error}`);
}
const okCount = results.filter(r => r.ok).length;
console.log(`\n${okCount}/${results.length} models working\n`);
process.exit(0);
}
main().catch(err => {
console.error('Failed to run test-all-models:', err);
process.exit(1);
});
|