AksharaLP-Sentence-2.8M / tools /run_examples_sent.mjs
sourojitdotcom's picture
AksharaLP-Sentence-2.8M v3.0.0 — word model (v2) + 2.77M-param sentence LM
8948114 verified
Raw History Blame Contribute Delete
1.28 kB
// Node: measured word-mode / sentence-mode outputs for the model card tables.
// usage: node tools/run_examples_sent.mjs <bundleDir> <nlmFile> <out.json> [text ...]
// e.g. node tools/run_examples_sent.mjs . ./nlm.q8 /tmp/out.json "$(sed -n 1p cases/cases.txt)"
import fs from "node:fs";
const root = new URL("../", import.meta.url);
const { loadFromDir } = await import(new URL("aksharalp.mjs", root).href);
const { attachSentence } = await import(new URL("sent.mjs", root).href);
const [dir, nlmFile, outp, ...texts] = process.argv.slice(2);
const list = texts.length ? texts
: fs.readFileSync(new URL("cases/cases.txt", root), "utf8").split("\n").map((s) => s.trim()).filter(Boolean);
const e = await loadFromDir(dir);
const v2 = list.map((t) => e.transliterate(t, { k: 6 }));
attachSentence(e, fs.readFileSync(nlmFile));
const rows = list.map((t, i) => {
const t0 = performance.now();
const r = e.transliterate(t, { k: 6 });
return { text: t, v2: v2[i].output, v3: r.output, same: v2[i].output === r.output, guard: r.guard, ms: +(performance.now() - t0).toFixed(1) };
});
fs.writeFileSync(outp, JSON.stringify(rows, null, 1));
for (const r of rows) console.log(`${r.text}\t|\t${r.v2}\t|\t${r.v3}\t|\t${r.same ? "same" : "CHANGED"}\t| ${JSON.stringify(r.guard)}`);