PeterPinetree commited on
Commit
2150d6e
·
verified ·
1 Parent(s): afe3de1

Update index.html

Browse files
Files changed (1) hide show
  1. index.html +43 -35
index.html CHANGED
@@ -275,54 +275,61 @@
275
  return { setSources, load, drawBase, highlight };
276
  })();
277
 
278
- // --- Local Qwen tokenizer loader (supports both JSON and BPE files) ---
279
  async function loadTokenizerQwenLocal() {
280
  resetProgress("Tokenizer");
281
  setStatus("Tokenizer: starting…");
282
 
283
- const base = MODELS.qwen.base; // e.g. .../assets/models/qwen/
284
- // New-format filenames
285
- const tjson = new URL("tokenizer.json", base).href;
 
286
  const tcfg = new URL("tokenizer_config.json", base).href;
287
  const smap = new URL("special_tokens_map.json", base).href;
 
288
 
289
- // Classic BPE filenames (present in many ONNX repos, incl. Qwen)
290
- const vocab = new URL("vocab.json", base).href;
291
- const merges = new URL("merges.txt", base).href;
292
-
293
- // Try new-format first; if missing, fall back to vocab/merges
294
- const tjsonHead = await fetch(tjson, { method: "HEAD" });
295
- const useNewFormat = tjsonHead.ok;
296
-
297
- const files = new Map();
298
 
299
- if (useNewFormat) {
300
  const [tokJSON, tokCfgJSON, smapJSON] = await Promise.all([
301
- fetch(tjson).then(r => { if (!r.ok) throw new Error("missing tokenizer.json"); return r.json(); }),
302
- fetch(tcfg).then(r => { if (!r.ok) throw new Error("missing tokenizer_config.json"); return r.json(); }),
303
- fetch(smap).then(r => r.ok ? r.json() : null),
304
  ]);
305
- files.set("tokenizer.json", new Blob([JSON.stringify(tokJSON)], { type: "application/json" }));
306
- files.set("tokenizer_config.json", new Blob([JSON.stringify(tokCfgJSON)],{ type: "application/json" }));
 
 
307
  if (smapJSON) files.set("special_tokens_map.json", new Blob([JSON.stringify(smapJSON)], { type: "application/json" }));
308
- } else {
309
- // Fall back to BPE pair; this avoids the "e.split is not a function" crash
310
- const [vocabText, mergesText, tokCfgJSON, smapJSON] = await Promise.all([
311
- fetch(vocab).then(r => { if (!r.ok) throw new Error("missing vocab.json"); return r.text(); }),
312
- fetch(merges).then(r => { if (!r.ok) throw new Error("missing merges.txt"); return r.text(); }),
313
- fetch(tcfg).then(r => r.ok ? r.json() : { model_max_length: 32768 }),
314
- fetch(smap).then(r => r.ok ? r.json() : null),
315
- ]);
316
- files.set("vocab.json", new Blob([vocabText], { type: "application/json" }));
317
- files.set("merges.txt", new Blob([mergesText], { type: "text/plain" }));
318
- files.set("tokenizer_config.json",
319
- new Blob([JSON.stringify(tokCfgJSON)], { type: "application/json" })
320
- );
321
- if (smapJSON) files.set("special_tokens_map.json",
322
- new Blob([JSON.stringify(smapJSON)], { type: "application/json" })
323
- );
324
  }
325
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
326
  tokenizer = await AutoTokenizer.from_pretrained("", {
327
  files,
328
  local_files_only: true,
@@ -331,6 +338,7 @@
331
  }
332
 
333
 
 
334
  // Core model state (module scope)
335
  let tokenizer = null, model = null;
336
  let loadSeq = 0;
 
275
  return { setSources, load, drawBase, highlight };
276
  })();
277
 
 
278
  async function loadTokenizerQwenLocal() {
279
  resetProgress("Tokenizer");
280
  setStatus("Tokenizer: starting…");
281
 
282
+ const base = MODELS.qwen.base; // absolute href into assets/models/qwen/
283
+
284
+ // try tokenizer.json first (preferred)
285
+ const tjson = new URL("tokenizer.json", base).href;
286
  const tcfg = new URL("tokenizer_config.json", base).href;
287
  const smap = new URL("special_tokens_map.json", base).href;
288
+ const merges= new URL("merges.txt", base).href;
289
 
290
+ // detect whether tokenizer.json exists
291
+ const hasTokenizerJSON = await fetch(tjson, { method: "HEAD" }).then(r => r.ok).catch(() => false);
 
 
 
 
 
 
 
292
 
293
+ if (hasTokenizerJSON) {
294
  const [tokJSON, tokCfgJSON, smapJSON] = await Promise.all([
295
+ fetch(tjson).then(r => { if (!r.ok) throw new Error("missing tokenizer.json"); return r.json(); }),
296
+ fetch(tcfg ).then(r => { if (!r.ok) throw new Error("missing tokenizer_config.json"); return r.json(); }),
297
+ fetch(smap ).then(r => r.ok ? r.json() : null),
298
  ]);
299
+
300
+ const files = new Map();
301
+ files.set("tokenizer.json", new Blob([JSON.stringify(tokJSON)], { type: "application/json" }));
302
+ files.set("tokenizer_config.json", new Blob([JSON.stringify(tokCfgJSON)], { type: "application/json" }));
303
  if (smapJSON) files.set("special_tokens_map.json", new Blob([JSON.stringify(smapJSON)], { type: "application/json" }));
304
+
305
+ tokenizer = await AutoTokenizer.from_pretrained("", {
306
+ files,
307
+ local_files_only: true,
308
+ progress_callback: onProgress,
309
+ });
310
+ return;
 
 
 
 
 
 
 
 
 
311
  }
312
 
313
+ // fallback: legacy BPE (vocab.txt + merges.txt)
314
+ const vocab = new URL("vocab.json", base).href;
315
+ const mergesTxt = await fetch(merges).then(r => {
316
+ if (!r.ok) throw new Error("missing merges.txt");
317
+ return r.text();
318
+ });
319
+ const mergesList = mergesTxt
320
+ .split(/\r?\n/)
321
+ .filter(line => line && !line.startsWith("#"));
322
+
323
+ const tokCfgJSON = await fetch(tcfg).then(r => {
324
+ if (!r.ok) throw new Error("missing tokenizer_config.json");
325
+ return r.json();
326
+ });
327
+
328
+ const files = new Map();
329
+ files.set("vocab.json", new Blob([await (await fetch(vocab)).text()], { type: "application/json" }));
330
+ files.set("merges.txt", new Blob([mergesList.join("\n")], { type: "text/plain" }));
331
+ files.set("tokenizer_config.json", new Blob([JSON.stringify(tokCfgJSON)], { type: "application/json" }));
332
+
333
  tokenizer = await AutoTokenizer.from_pretrained("", {
334
  files,
335
  local_files_only: true,
 
338
  }
339
 
340
 
341
+
342
  // Core model state (module scope)
343
  let tokenizer = null, model = null;
344
  let loadSeq = 0;