Spaces:
Runtime error
Runtime error
Update index.html
Browse files- index.html +33 -12
index.html
CHANGED
|
@@ -279,8 +279,7 @@
|
|
| 279 |
async function loadTokenizerQwenLocal() {
|
| 280 |
resetProgress("Tokenizer");
|
| 281 |
setStatus("Tokenizer: starting…");
|
| 282 |
-
|
| 283 |
-
// Always resolve from the current document URL and force the /qwen/ segment.
|
| 284 |
let base = new URL("assets/models/qwen/", document.baseURI).href;
|
| 285 |
if (!/\/qwen\/$/.test(base)) {
|
| 286 |
base = new URL("assets/models/qwen/", base).href;
|
|
@@ -290,20 +289,15 @@
|
|
| 290 |
const tjson = new URL("tokenizer.json", base).href;
|
| 291 |
const tcfg = new URL("tokenizer_config.json", base).href;
|
| 292 |
const smap = new URL("special_tokens_map.json", base).href;
|
| 293 |
-
const merges= new URL("merges.txt", base).href;
|
| 294 |
-
|
| 295 |
-
// DEBUG: verify what the browser will fetch
|
| 296 |
-
console.log("[Qwen tokenizer paths]", { base, tjson, tcfg, smap, merges });
|
| 297 |
|
| 298 |
// Detect tokenizer.json first
|
| 299 |
-
const hasTokenizerJSON = await fetch(tjson, { method: "HEAD" })
|
| 300 |
-
.then(r => r.ok)
|
| 301 |
-
.catch(() => false);
|
| 302 |
|
| 303 |
if (hasTokenizerJSON) {
|
|
|
|
| 304 |
const [tokJSON, tokCfgJSON, smapJSON] = await Promise.all([
|
| 305 |
-
fetch(tjson).then(r => { if (!r.ok) throw new Error("missing tokenizer.json");
|
| 306 |
-
fetch(tcfg ).then(r => { if (!r.ok) throw new Error("missing tokenizer_config.json");
|
| 307 |
fetch(smap ).then(r => r.ok ? r.json() : null),
|
| 308 |
]);
|
| 309 |
|
|
@@ -317,8 +311,35 @@
|
|
| 317 |
progress_callback: onProgress,
|
| 318 |
});
|
| 319 |
return;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 320 |
}
|
| 321 |
-
}
|
|
|
|
| 322 |
// ---- fallback: legacy BPE (vocab.json + merges.txt) ----
|
| 323 |
const vocabURL = new URL("vocab.json", base).href;
|
| 324 |
const mergesURL = new URL("merges.txt", base).href;
|
|
|
|
| 279 |
async function loadTokenizerQwenLocal() {
|
| 280 |
resetProgress("Tokenizer");
|
| 281 |
setStatus("Tokenizer: starting…");
|
| 282 |
+
// Always resolve from current URL and force the /qwen/ segment.
|
|
|
|
| 283 |
let base = new URL("assets/models/qwen/", document.baseURI).href;
|
| 284 |
if (!/\/qwen\/$/.test(base)) {
|
| 285 |
base = new URL("assets/models/qwen/", base).href;
|
|
|
|
| 289 |
const tjson = new URL("tokenizer.json", base).href;
|
| 290 |
const tcfg = new URL("tokenizer_config.json", base).href;
|
| 291 |
const smap = new URL("special_tokens_map.json", base).href;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 292 |
|
| 293 |
// Detect tokenizer.json first
|
| 294 |
+
const hasTokenizerJSON = await fetch(tjson, { method: "HEAD" }).then(r => r.ok).catch(() => false);
|
|
|
|
|
|
|
| 295 |
|
| 296 |
if (hasTokenizerJSON) {
|
| 297 |
+
// load tokenizer.json, tokenizer_config.json and special tokens map
|
| 298 |
const [tokJSON, tokCfgJSON, smapJSON] = await Promise.all([
|
| 299 |
+
fetch(tjson).then(r => { if (!r.ok) throw new Error("missing tokenizer.json"); return r.json(); }),
|
| 300 |
+
fetch(tcfg ).then(r => { if (!r.ok) throw new Error("missing tokenizer_config.json"); return r.json(); }),
|
| 301 |
fetch(smap ).then(r => r.ok ? r.json() : null),
|
| 302 |
]);
|
| 303 |
|
|
|
|
| 311 |
progress_callback: onProgress,
|
| 312 |
});
|
| 313 |
return;
|
| 314 |
+
} else {
|
| 315 |
+
// ---- fallback: legacy BPE (vocab.json + merges.txt) ----
|
| 316 |
+
const vocabURL = new URL("vocab.json", base).href;
|
| 317 |
+
const mergesURL = new URL("merges.txt", base).href;
|
| 318 |
+
|
| 319 |
+
const [vocabText, mergesText, cfgMaybe] = await Promise.all([
|
| 320 |
+
fetch(vocabURL ).then(r => { if (!r.ok) throw new Error("missing vocab.json"); return r.text(); }),
|
| 321 |
+
fetch(mergesURL).then(r => { if (!r.ok) throw new Error("missing merges.txt"); return r.text(); }),
|
| 322 |
+
fetch(new URL("tokenizer_config.json", base).href).then(r => r.ok ? r.json() : null),
|
| 323 |
+
]);
|
| 324 |
+
|
| 325 |
+
const files = new Map();
|
| 326 |
+
files.set("vocab.json", new Blob([vocabText], { type: "application/json" }));
|
| 327 |
+
files.set("merges.txt", new Blob([mergesText], { type: "text/plain" }));
|
| 328 |
+
if (cfgMaybe) {
|
| 329 |
+
const cfg = { ...cfgMaybe };
|
| 330 |
+
delete cfg.merges;
|
| 331 |
+
if (cfg.tokenizer_file) delete cfg.tokenizer_file;
|
| 332 |
+
files.set("tokenizer_config.json", new Blob([JSON.stringify(cfg)], { type: "application/json" }));
|
| 333 |
+
}
|
| 334 |
+
|
| 335 |
+
tokenizer = await AutoTokenizer.from_pretrained("", {
|
| 336 |
+
files,
|
| 337 |
+
local_files_only: true,
|
| 338 |
+
progress_callback: onProgress,
|
| 339 |
+
});
|
| 340 |
}
|
| 341 |
+
} // end of loadTokenizerQwenLocal
|
| 342 |
+
|
| 343 |
// ---- fallback: legacy BPE (vocab.json + merges.txt) ----
|
| 344 |
const vocabURL = new URL("vocab.json", base).href;
|
| 345 |
const mergesURL = new URL("merges.txt", base).href;
|