Spaces:
Running
Running
File size: 4,559 Bytes
6c30253 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | import { selectEmbeddingPrecision } from '../model-precision.js';
/**
* Format-neutral runtime boundary.
*
* An ONNX or GGUF implementation must satisfy the same adapter contract. The
* product UI intentionally has no dependency on either runtime.
*/
class Runtime extends EventTarget {
adapter = null;
status = 'idle';
backend = 'Engine not selected';
webgpu = null;
embeddingPrecision = null;
emit(type, detail) {
this.dispatchEvent(new CustomEvent(type, { detail }));
}
log(level, message, detail = '') {
if (level !== 'error' && level !== 'warn') return;
console[level](`[LFM edge runtime] ${message}`, detail || '');
}
async probe() {
if (!globalThis.isSecureContext) throw new Error('WebGPU requires HTTPS or localhost.');
if (!navigator.gpu) throw new Error('WebGPU is unavailable in this browser.');
const adapter = await navigator.gpu.requestAdapter({ powerPreference: 'high-performance' });
if (!adapter) throw new Error('The browser did not return a WebGPU adapter.');
const info = adapter.info || {};
this.webgpu = {
adapter,
name: info.description || info.device || info.architecture || 'WebGPU adapter',
vendor: info.vendor || 'Not exposed',
architecture: info.architecture || 'Not exposed',
features: [...adapter.features].sort(),
limits: adapter.limits,
};
return this.webgpu;
}
async load() {
if (this.status === 'loading') return;
this.status = 'loading';
this.emit('status', { status: this.status, backend: this.backend });
try {
await this.probe();
const baseManifest = await fetch('/model-manifest.json', { cache: 'no-store' }).then(response => response.json());
const selection = selectEmbeddingPrecision(baseManifest, this.webgpu.features);
const manifest = selection.manifest;
this.embeddingPrecision = selection.precision;
this.emit('status', { status: this.status, backend: this.backend });
const loaders = {
'onnx-transformers': () => import('../engines/onnx-transformers-engine.js'),
};
const loadEngine = loaders[manifest.engine?.adapter];
if (!loadEngine) throw new Error(`Unknown or unselected inference adapter: ${manifest.engine?.adapter || 'none'}.`);
const module = await loadEngine();
this.adapter = await module.createEngine({ manifest, telemetry: event => this.log(event.level, event.message, event.detail) });
await this.adapter.load(progress => this.emit('progress', progress));
this.backend = this.adapter.backend;
this.status = 'ready';
} catch (error) {
this.status = 'error';
this.log('error', 'Engine load stopped', error.message);
throw error;
} finally {
this.emit('status', { status: this.status, backend: this.backend });
}
}
/** Returns { text, toolCalls, finishReason } for every inference adapter. */
async generate(messages, options) {
if (!this.adapter || this.status !== 'ready') throw new Error('No inference engine is configured.');
this.status = 'generating';
this.emit('status', { status: this.status, backend: this.backend });
try {
return await this.adapter.generate(messages, options);
} finally {
this.status = 'ready';
this.emit('status', { status: this.status, backend: this.backend });
}
}
async clearCache() {
const result = this.adapter?.clearCache ? await this.adapter.clearCache() : { cleared: false, entriesDeleted: 0 };
const cleared = typeof result === 'object' ? result.cleared : Boolean(result);
return cleared;
}
async cacheInfo() {
if (this.adapter?.cacheInfo) return this.adapter.cacheInfo();
if (globalThis.caches) {
const cacheNames = await caches.keys();
const modelCacheNames = cacheNames.filter(name => name === 'liquid-lfm-models-v4');
let used = 0;
for (const cacheName of modelCacheNames) {
const cache = await caches.open(cacheName);
for (const request of await cache.keys()) {
const response = await cache.match(request);
used += Number(response?.headers.get('content-length') || 0);
}
}
const estimate = await navigator.storage?.estimate?.();
return { used, available: estimate?.quota || 0 };
}
const estimate = await navigator.storage?.estimate?.();
return { used: 0, available: estimate?.quota || 0 };
}
clearConversationCache() {
this.adapter?.clearConversationCache?.();
}
}
export const runtime = new Runtime();
|