| |
| import { resolve } from 'node:path' |
| import type { Question } from '../../src/decisions/adapters/crivo.js' |
| import { Crivo } from '../../src/decisions/adapters/crivo.js' |
| import { MODEL_SIZES } from '../../src/decisions/models.js' |
|
|
| for (const size of process.argv.slice(2).length ? process.argv.slice(2) : MODEL_SIZES) { |
| const directory = resolve('.cache/variants', size) |
| const fixture = await Bun.file(`${directory}/parity.json`).json() |
| for (const dtype of ['fp32', 'q8'] as const) { |
| const model = await Crivo.load({ |
| modelId: directory, |
| dtype, |
| device: 'cpu', |
| localFilesOnly: true |
| }) |
| let maxError = 0 |
| try { |
| for (const sample of fixture.cases) { |
| const answers = await model.answerMany( |
| sample.state, |
| sample.questions.map((question: Question) => ({ question })) |
| ) |
| for (const [index, answer] of answers.entries()) { |
| const expected: number[] = sample[dtype === 'q8' ? 'int8' : 'fp32'][index] |
| const actual = |
| answer.type === 'noul' |
| ? [1 - answer.noul, answer.noul] |
| : Object.values(answer.probabilities) |
| for (const [column, value] of actual.entries()) { |
| const difference = Math.abs(value - (expected[column] ?? Number.NaN)) |
| if (!Number.isFinite(difference) || difference > 1e-4) |
| throw new Error(`${size}/${dtype}: probability mismatch ${difference}`) |
| maxError = Math.max(maxError, difference) |
| } |
| } |
| } |
| } finally { |
| await model.dispose() |
| } |
| console.log(JSON.stringify({ size, dtype, maxProbabilityError: maxError })) |
| } |
| } |
|
|
