PeterPinetree commited on
Commit
67ccc05
·
verified ·
1 Parent(s): dc46d57

Update index.html

Browse files
Files changed (1) hide show
  1. index.html +33 -12
index.html CHANGED
@@ -279,8 +279,7 @@
279
  async function loadTokenizerQwenLocal() {
280
  resetProgress("Tokenizer");
281
  setStatus("Tokenizer: starting…");
282
-
283
- // Always resolve from the current document URL and force the /qwen/ segment.
284
  let base = new URL("assets/models/qwen/", document.baseURI).href;
285
  if (!/\/qwen\/$/.test(base)) {
286
  base = new URL("assets/models/qwen/", base).href;
@@ -290,20 +289,15 @@
290
  const tjson = new URL("tokenizer.json", base).href;
291
  const tcfg = new URL("tokenizer_config.json", base).href;
292
  const smap = new URL("special_tokens_map.json", base).href;
293
- const merges= new URL("merges.txt", base).href;
294
-
295
- // DEBUG: verify what the browser will fetch
296
- console.log("[Qwen tokenizer paths]", { base, tjson, tcfg, smap, merges });
297
 
298
  // Detect tokenizer.json first
299
- const hasTokenizerJSON = await fetch(tjson, { method: "HEAD" })
300
- .then(r => r.ok)
301
- .catch(() => false);
302
 
303
  if (hasTokenizerJSON) {
 
304
  const [tokJSON, tokCfgJSON, smapJSON] = await Promise.all([
305
- fetch(tjson).then(r => { if (!r.ok) throw new Error("missing tokenizer.json"); return r.json(); }),
306
- fetch(tcfg ).then(r => { if (!r.ok) throw new Error("missing tokenizer_config.json"); return r.json(); }),
307
  fetch(smap ).then(r => r.ok ? r.json() : null),
308
  ]);
309
 
@@ -317,8 +311,35 @@
317
  progress_callback: onProgress,
318
  });
319
  return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
320
  }
321
- }
 
322
  // ---- fallback: legacy BPE (vocab.json + merges.txt) ----
323
  const vocabURL = new URL("vocab.json", base).href;
324
  const mergesURL = new URL("merges.txt", base).href;
 
279
  async function loadTokenizerQwenLocal() {
280
  resetProgress("Tokenizer");
281
  setStatus("Tokenizer: starting…");
282
+ // Always resolve from current URL and force the /qwen/ segment.
 
283
  let base = new URL("assets/models/qwen/", document.baseURI).href;
284
  if (!/\/qwen\/$/.test(base)) {
285
  base = new URL("assets/models/qwen/", base).href;
 
289
  const tjson = new URL("tokenizer.json", base).href;
290
  const tcfg = new URL("tokenizer_config.json", base).href;
291
  const smap = new URL("special_tokens_map.json", base).href;
 
 
 
 
292
 
293
  // Detect tokenizer.json first
294
+ const hasTokenizerJSON = await fetch(tjson, { method: "HEAD" }).then(r => r.ok).catch(() => false);
 
 
295
 
296
  if (hasTokenizerJSON) {
297
+ // load tokenizer.json, tokenizer_config.json and special tokens map
298
  const [tokJSON, tokCfgJSON, smapJSON] = await Promise.all([
299
+ fetch(tjson).then(r => { if (!r.ok) throw new Error("missing tokenizer.json"); return r.json(); }),
300
+ fetch(tcfg ).then(r => { if (!r.ok) throw new Error("missing tokenizer_config.json"); return r.json(); }),
301
  fetch(smap ).then(r => r.ok ? r.json() : null),
302
  ]);
303
 
 
311
  progress_callback: onProgress,
312
  });
313
  return;
314
+ } else {
315
+ // ---- fallback: legacy BPE (vocab.json + merges.txt) ----
316
+ const vocabURL = new URL("vocab.json", base).href;
317
+ const mergesURL = new URL("merges.txt", base).href;
318
+
319
+ const [vocabText, mergesText, cfgMaybe] = await Promise.all([
320
+ fetch(vocabURL ).then(r => { if (!r.ok) throw new Error("missing vocab.json"); return r.text(); }),
321
+ fetch(mergesURL).then(r => { if (!r.ok) throw new Error("missing merges.txt"); return r.text(); }),
322
+ fetch(new URL("tokenizer_config.json", base).href).then(r => r.ok ? r.json() : null),
323
+ ]);
324
+
325
+ const files = new Map();
326
+ files.set("vocab.json", new Blob([vocabText], { type: "application/json" }));
327
+ files.set("merges.txt", new Blob([mergesText], { type: "text/plain" }));
328
+ if (cfgMaybe) {
329
+ const cfg = { ...cfgMaybe };
330
+ delete cfg.merges;
331
+ if (cfg.tokenizer_file) delete cfg.tokenizer_file;
332
+ files.set("tokenizer_config.json", new Blob([JSON.stringify(cfg)], { type: "application/json" }));
333
+ }
334
+
335
+ tokenizer = await AutoTokenizer.from_pretrained("", {
336
+ files,
337
+ local_files_only: true,
338
+ progress_callback: onProgress,
339
+ });
340
  }
341
+ } // end of loadTokenizerQwenLocal
342
+
343
  // ---- fallback: legacy BPE (vocab.json + merges.txt) ----
344
  const vocabURL = new URL("vocab.json", base).href;
345
  const mergesURL = new URL("merges.txt", base).href;