PeterPinetree commited on
Commit
cb2b9a9
·
verified ·
1 Parent(s): 67ccc05

Update index.html

Browse files

Deleted redundant fallback code

Files changed (1) hide show
  1. index.html +0 -35
index.html CHANGED
@@ -340,41 +340,6 @@
340
  }
341
  } // end of loadTokenizerQwenLocal
342
 
343
- // ---- fallback: legacy BPE (vocab.json + merges.txt) ----
344
- const vocabURL = new URL("vocab.json", base).href;
345
- const mergesURL = new URL("merges.txt", base).href;
346
-
347
- // Load raw file contents
348
- const [vocabText, mergesText, cfgMaybe] = await Promise.all([
349
- fetch(vocabURL).then(r => { if (!r.ok) throw new Error("missing vocab.json"); return r.text(); }),
350
- fetch(mergesURL).then(r => { if (!r.ok) throw new Error("missing merges.txt"); return r.text(); }),
351
- // tokenizer_config.json is optional; we'll sanitize it if present
352
- fetch(new URL("tokenizer_config.json", base).href).then(r => r.ok ? r.json() : null),
353
- ]);
354
-
355
- // IMPORTANT: pass merges as the **raw text string**
356
- files.set("vocab.json", new Blob([vocabText], { type: "application/json" }));
357
- files.set("merges.txt", new Blob([mergesText], { type: "text/plain" }));
358
-
359
- // If you include tokenizer_config.json, strip fields that can confuse the BPE loader
360
- if (cfgMaybe) {
361
- const cfg = { ...cfgMaybe };
362
- // These sometimes show up as arrays/objects; the BPE loader expects to read merges.txt itself.
363
- if (Array.isArray(cfg.merges)) delete cfg.merges;
364
- if (typeof cfg.merges === "object") delete cfg.merges;
365
- // (Optional) also remove tokenizer_file pointers which can trigger alternate code paths
366
- if (cfg.tokenizer_file) delete cfg.tokenizer_file;
367
-
368
- files.set("tokenizer_config.json", new Blob([JSON.stringify(cfg)], { type: "application/json" }));
369
- }
370
-
371
- // Now build the tokenizer strictly from local files
372
- tokenizer = await AutoTokenizer.from_pretrained("", {
373
- files,
374
- local_files_only: true,
375
- progress_callback: onProgress,
376
- });
377
-
378
  // Core model state (module scope)
379
  let tokenizer = null, model = null;
380
  let loadSeq = 0;
 
340
  }
341
  } // end of loadTokenizerQwenLocal
342
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
343
  // Core model state (module scope)
344
  let tokenizer = null, model = null;
345
  let loadSeq = 0;