Spaces:
Runtime error
Runtime error
Update index.html
Browse files- index.html +43 -35
index.html
CHANGED
|
@@ -275,54 +275,61 @@
|
|
| 275 |
return { setSources, load, drawBase, highlight };
|
| 276 |
})();
|
| 277 |
|
| 278 |
-
// --- Local Qwen tokenizer loader (supports both JSON and BPE files) ---
|
| 279 |
async function loadTokenizerQwenLocal() {
|
| 280 |
resetProgress("Tokenizer");
|
| 281 |
setStatus("Tokenizer: starting…");
|
| 282 |
|
| 283 |
-
const base = MODELS.qwen.base;
|
| 284 |
-
|
| 285 |
-
|
|
|
|
| 286 |
const tcfg = new URL("tokenizer_config.json", base).href;
|
| 287 |
const smap = new URL("special_tokens_map.json", base).href;
|
|
|
|
| 288 |
|
| 289 |
-
//
|
| 290 |
-
const
|
| 291 |
-
const merges = new URL("merges.txt", base).href;
|
| 292 |
-
|
| 293 |
-
// Try new-format first; if missing, fall back to vocab/merges
|
| 294 |
-
const tjsonHead = await fetch(tjson, { method: "HEAD" });
|
| 295 |
-
const useNewFormat = tjsonHead.ok;
|
| 296 |
-
|
| 297 |
-
const files = new Map();
|
| 298 |
|
| 299 |
-
if (
|
| 300 |
const [tokJSON, tokCfgJSON, smapJSON] = await Promise.all([
|
| 301 |
-
fetch(tjson).then(r => { if (!r.ok) throw new Error("missing tokenizer.json");
|
| 302 |
-
fetch(tcfg).then(r => { if (!r.ok) throw new Error("missing tokenizer_config.json"); return r.json(); }),
|
| 303 |
-
fetch(smap).then(r => r.ok ? r.json() : null),
|
| 304 |
]);
|
| 305 |
-
|
| 306 |
-
files
|
|
|
|
|
|
|
| 307 |
if (smapJSON) files.set("special_tokens_map.json", new Blob([JSON.stringify(smapJSON)], { type: "application/json" }));
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
]);
|
| 316 |
-
files.set("vocab.json", new Blob([vocabText], { type: "application/json" }));
|
| 317 |
-
files.set("merges.txt", new Blob([mergesText], { type: "text/plain" }));
|
| 318 |
-
files.set("tokenizer_config.json",
|
| 319 |
-
new Blob([JSON.stringify(tokCfgJSON)], { type: "application/json" })
|
| 320 |
-
);
|
| 321 |
-
if (smapJSON) files.set("special_tokens_map.json",
|
| 322 |
-
new Blob([JSON.stringify(smapJSON)], { type: "application/json" })
|
| 323 |
-
);
|
| 324 |
}
|
| 325 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 326 |
tokenizer = await AutoTokenizer.from_pretrained("", {
|
| 327 |
files,
|
| 328 |
local_files_only: true,
|
|
@@ -331,6 +338,7 @@
|
|
| 331 |
}
|
| 332 |
|
| 333 |
|
|
|
|
| 334 |
// Core model state (module scope)
|
| 335 |
let tokenizer = null, model = null;
|
| 336 |
let loadSeq = 0;
|
|
|
|
| 275 |
return { setSources, load, drawBase, highlight };
|
| 276 |
})();
|
| 277 |
|
|
|
|
| 278 |
async function loadTokenizerQwenLocal() {
|
| 279 |
resetProgress("Tokenizer");
|
| 280 |
setStatus("Tokenizer: starting…");
|
| 281 |
|
| 282 |
+
const base = MODELS.qwen.base; // absolute href into assets/models/qwen/
|
| 283 |
+
|
| 284 |
+
// try tokenizer.json first (preferred)
|
| 285 |
+
const tjson = new URL("tokenizer.json", base).href;
|
| 286 |
const tcfg = new URL("tokenizer_config.json", base).href;
|
| 287 |
const smap = new URL("special_tokens_map.json", base).href;
|
| 288 |
+
const merges= new URL("merges.txt", base).href;
|
| 289 |
|
| 290 |
+
// detect whether tokenizer.json exists
|
| 291 |
+
const hasTokenizerJSON = await fetch(tjson, { method: "HEAD" }).then(r => r.ok).catch(() => false);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 292 |
|
| 293 |
+
if (hasTokenizerJSON) {
|
| 294 |
const [tokJSON, tokCfgJSON, smapJSON] = await Promise.all([
|
| 295 |
+
fetch(tjson).then(r => { if (!r.ok) throw new Error("missing tokenizer.json"); return r.json(); }),
|
| 296 |
+
fetch(tcfg ).then(r => { if (!r.ok) throw new Error("missing tokenizer_config.json"); return r.json(); }),
|
| 297 |
+
fetch(smap ).then(r => r.ok ? r.json() : null),
|
| 298 |
]);
|
| 299 |
+
|
| 300 |
+
const files = new Map();
|
| 301 |
+
files.set("tokenizer.json", new Blob([JSON.stringify(tokJSON)], { type: "application/json" }));
|
| 302 |
+
files.set("tokenizer_config.json", new Blob([JSON.stringify(tokCfgJSON)], { type: "application/json" }));
|
| 303 |
if (smapJSON) files.set("special_tokens_map.json", new Blob([JSON.stringify(smapJSON)], { type: "application/json" }));
|
| 304 |
+
|
| 305 |
+
tokenizer = await AutoTokenizer.from_pretrained("", {
|
| 306 |
+
files,
|
| 307 |
+
local_files_only: true,
|
| 308 |
+
progress_callback: onProgress,
|
| 309 |
+
});
|
| 310 |
+
return;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 311 |
}
|
| 312 |
|
| 313 |
+
// fallback: legacy BPE (vocab.txt + merges.txt)
|
| 314 |
+
const vocab = new URL("vocab.json", base).href;
|
| 315 |
+
const mergesTxt = await fetch(merges).then(r => {
|
| 316 |
+
if (!r.ok) throw new Error("missing merges.txt");
|
| 317 |
+
return r.text();
|
| 318 |
+
});
|
| 319 |
+
const mergesList = mergesTxt
|
| 320 |
+
.split(/\r?\n/)
|
| 321 |
+
.filter(line => line && !line.startsWith("#"));
|
| 322 |
+
|
| 323 |
+
const tokCfgJSON = await fetch(tcfg).then(r => {
|
| 324 |
+
if (!r.ok) throw new Error("missing tokenizer_config.json");
|
| 325 |
+
return r.json();
|
| 326 |
+
});
|
| 327 |
+
|
| 328 |
+
const files = new Map();
|
| 329 |
+
files.set("vocab.json", new Blob([await (await fetch(vocab)).text()], { type: "application/json" }));
|
| 330 |
+
files.set("merges.txt", new Blob([mergesList.join("\n")], { type: "text/plain" }));
|
| 331 |
+
files.set("tokenizer_config.json", new Blob([JSON.stringify(tokCfgJSON)], { type: "application/json" }));
|
| 332 |
+
|
| 333 |
tokenizer = await AutoTokenizer.from_pretrained("", {
|
| 334 |
files,
|
| 335 |
local_files_only: true,
|
|
|
|
| 338 |
}
|
| 339 |
|
| 340 |
|
| 341 |
+
|
| 342 |
// Core model state (module scope)
|
| 343 |
let tokenizer = null, model = null;
|
| 344 |
let loadSeq = 0;
|