import sharp from 'sharp'; import * as ort from 'onnxruntime-node'; import { readFile } from 'node:fs/promises'; import { dirname, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; const MODEL_DIR = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const PADDING_MULTIPLE = 32; export type ModelFormat = 'fp32' | 'fp16'; const MODEL_FILES: Record = { fp32: 'model.onnx', fp16: 'model.fp16.onnx', }; export class Classifier { private constructor( private readonly session: ort.InferenceSession, private readonly preprocess: { resize_longest_side_px: number; mean: [number, number, number]; std: [number, number, number]; }, private readonly labels: { screen: { labels: string[] }; safety: { labels: string[] }; }, ) {} /** * Load the exported ONNX classifier and sidecar metadata. * * @param modelDir Directory containing the exported model artifacts. * @returns Ready-to-use classifier instance. */ static async create(modelDir = MODEL_DIR, modelFormat: ModelFormat = 'fp32'): Promise { const session = await ort.InferenceSession.create(resolve(modelDir, 'onnx', MODEL_FILES[modelFormat]), { executionProviders: ['cpu'], }); const preprocess = JSON.parse(await readFile(resolve(modelDir, 'preprocess.json'), 'utf8')) as { resize_longest_side_px: number; mean: [number, number, number]; std: [number, number, number]; }; const labels = JSON.parse(await readFile(resolve(modelDir, 'inference/labels.json'), 'utf8')) as { screen: { labels: string[] }; safety: { labels: string[] }; }; return new Classifier(session, preprocess, labels); } /** * Classify one image. * * @param imagePath Path to an image file. * @returns Predicted public screen and safety labels. */ async classify(imagePath: string): Promise<{ screen: string; safety: string }> { const [prediction] = await this.classifyBatch([imagePath]); if (prediction === undefined) { throw new Error('No prediction was produced.'); } return prediction; } /** * Classify a batch of images. * * @param imagePaths Paths to image files. * @returns Predicted public screen and safety labels for each image. */ async classifyBatch(imagePaths: readonly string[]): Promise> { if (imagePaths.length === 0) { return []; } const images = await Promise.all(imagePaths.map((imagePath) => preprocessImage(imagePath, this.preprocess))); const output = await this.session.run({ image: collateImages(images) }); return decodePredictions(outputTensor(output, 'screen'), outputTensor(output, 'safety'), this.labels); } } /** * Classify one image with a classifier loaded from disk. * * @param imagePath Path to an image file. * @param modelDir Directory containing the exported model artifacts. * @returns Predicted public screen and safety labels. */ export async function classify( imagePath: string, modelDir = MODEL_DIR, modelFormat: ModelFormat = 'fp32', ): Promise<{ screen: string; safety: string }> { const classifier = await Classifier.create(modelDir, modelFormat); if (imagePath.length === 0) { throw new Error('Image path must not be empty.'); } return classifier.classify(imagePath); } /** * Classify a batch of images with a classifier loaded from disk. * * @param imagePaths Paths to image files. * @param modelDir Directory containing the exported model artifacts. * @returns Predicted public screen and safety labels for each image. */ export async function classifyBatch( imagePaths: readonly string[], modelDir = MODEL_DIR, modelFormat: ModelFormat = 'fp32', ): Promise> { const classifier = await Classifier.create(modelDir, modelFormat); if (imagePaths.length === 0) { return []; } return classifier.classifyBatch(imagePaths); } async function preprocessImage( imagePath: string, preprocess: { resize_longest_side_px: number; mean: [number, number, number]; std: [number, number, number]; }, ): Promise<{ data: Float32Array; height: number; width: number }> { const { data, info: metadata } = await sharp(imagePath) .rotate() .resize({ width: preprocess.resize_longest_side_px, height: preprocess.resize_longest_side_px, fit: 'inside', kernel: 'cubic', }) .removeAlpha() .toColourspace('srgb') .raw() .toBuffer({ resolveWithObject: true }); if (metadata.channels !== 3) { throw new Error(`Expected RGB image data, got ${String(metadata.channels)} channels.`); } const pixels = metadata.width * metadata.height; const tensor = new Float32Array(3 * pixels); for (let pixel = 0; pixel < pixels; pixel += 1) { const offset = pixel * 3; tensor[pixel] = (channel(data, offset) / 255 - preprocess.mean[0]) / preprocess.std[0]; tensor[pixels + pixel] = (channel(data, offset + 1) / 255 - preprocess.mean[1]) / preprocess.std[1]; tensor[2 * pixels + pixel] = (channel(data, offset + 2) / 255 - preprocess.mean[2]) / preprocess.std[2]; } return { data: tensor, height: metadata.height, width: metadata.width, }; } function collateImages( images: ReadonlyArray<{ data: Float32Array; height: number; width: number }>, ): ort.TypedTensor<'float32'> { const height = Math.ceil(Math.max(...images.map((image) => image.height)) / PADDING_MULTIPLE) * PADDING_MULTIPLE; const width = Math.ceil(Math.max(...images.map((image) => image.width)) / PADDING_MULTIPLE) * PADDING_MULTIPLE; const imagePixels = height * width; const tensor = new Float32Array(images.length * 3 * imagePixels); for (let imageIndex = 0; imageIndex < images.length; imageIndex += 1) { const image = imageAt(images, imageIndex); const sourcePixels = image.height * image.width; for (let channelIndex = 0; channelIndex < 3; channelIndex += 1) { for (let row = 0; row < image.height; row += 1) { const sourceStart = channelIndex * sourcePixels + row * image.width; const targetStart = imageIndex * 3 * imagePixels + channelIndex * imagePixels + row * width; tensor.set(image.data.subarray(sourceStart, sourceStart + image.width), targetStart); } } } return new ort.Tensor('float32', tensor, [images.length, 3, height, width]); } function decodePredictions( screen: ort.TypedTensor<'float32'>, safety: ort.TypedTensor<'float32'>, labels: { screen: { labels: string[] }; safety: { labels: string[] }; }, ): Array<{ screen: string; safety: string }> { // ONNX emits flat screen and safety logits. const batchSize = tensorDim(screen, 0); const screenClassCount = tensorDim(screen, 1); const safetyBatchSize = tensorDim(safety, 0); const safetyClassCount = tensorDim(safety, 1); if (safetyBatchSize !== batchSize) { throw new Error( `Safety batch size ${String(safetyBatchSize)} does not match screen batch size ${String(batchSize)}.`, ); } const predictions: Array<{ screen: string; safety: string }> = []; for (let row = 0; row < batchSize; row += 1) { predictions.push({ screen: labelAt(labels.screen.labels, topIndex(screen.data, row * screenClassCount, screenClassCount)), safety: labelAt(labels.safety.labels, topIndex(safety.data, row * safetyClassCount, safetyClassCount)), }); } return predictions; } function outputTensor(output: ort.InferenceSession.ReturnType, name: string): ort.TypedTensor<'float32'> { const tensor = output[name]; if (tensor === undefined) { throw new Error(`Missing ONNX output "${name}".`); } if (tensor.type !== 'float32') { throw new Error(`Expected "${name}" to be float32, got ${tensor.type}.`); } return tensor as ort.TypedTensor<'float32'>; } function tensorDim(tensor: ort.TypedTensor<'float32'>, index: number): number { const dimension = tensor.dims[index]; if (typeof dimension !== 'number') { throw new Error(`Missing tensor dimension ${String(index)}.`); } return dimension; } function labelAt(labels: readonly string[], index: number): string { const label = labels[index]; if (label === undefined) { throw new Error(`Missing label for class ${String(index)}.`); } return label; } function topIndex(data: Float32Array, offset: number, count: number): number { let top = 0; let score = valueAt(data, offset); for (let index = 1; index < count; index += 1) { const candidate = valueAt(data, offset + index); if (candidate > score) { top = index; score = candidate; } } return top; } function imageAt( images: ReadonlyArray<{ data: Float32Array; height: number; width: number }>, index: number, ): { data: Float32Array; height: number; width: number } { const image = images[index]; if (image === undefined) { throw new Error(`Missing image at index ${String(index)}.`); } return image; } function valueAt(data: Float32Array, index: number): number { const value = data[index]; if (value === undefined) { throw new Error(`Missing tensor value at offset ${String(index)}.`); } return value; } function channel(data: Buffer, index: number): number { const channelByte = data[index]; if (channelByte === undefined) { throw new Error(`Missing image channel at offset ${String(index)}.`); } return channelByte; } const script = process.argv[1]; if (script !== undefined && import.meta.url === pathToFileURL(resolve(script)).href) { const args = process.argv.slice(2); const modelFlag = args.indexOf('--model-dir'); const modelDir = modelFlag >= 0 ? args[modelFlag + 1] : MODEL_DIR; if (modelFlag >= 0) { args.splice(modelFlag, 2); } const formatFlag = args.indexOf('--model-format'); const modelFormat = formatFlag >= 0 ? parseModelFormat(args[formatFlag + 1]) : 'fp32'; if (formatFlag >= 0) { args.splice(formatFlag, 2); } if (args.length === 0 || modelDir === undefined) { throw new Error('Usage: bun inference/typescript.ts [--model-dir export_dir] [--model-format fp32|fp16]'); } const predictions = await classifyBatch(args, modelDir, modelFormat); process.stdout.write(`${JSON.stringify(predictions.length === 1 ? predictions[0] : predictions, null, 2)}\n`); } function parseModelFormat(value: string | undefined): ModelFormat { if (value === 'fp32' || value === 'fp16') { return value; } throw new Error('Model format must be fp32 or fp16.'); }