Spaces:
Runtime error
Runtime error
Update index.html
Browse filesDeleted redundant fallback code
- index.html +0 -35
index.html
CHANGED
|
@@ -340,41 +340,6 @@
|
|
| 340 |
}
|
| 341 |
} // end of loadTokenizerQwenLocal
|
| 342 |
|
| 343 |
-
// ---- fallback: legacy BPE (vocab.json + merges.txt) ----
|
| 344 |
-
const vocabURL = new URL("vocab.json", base).href;
|
| 345 |
-
const mergesURL = new URL("merges.txt", base).href;
|
| 346 |
-
|
| 347 |
-
// Load raw file contents
|
| 348 |
-
const [vocabText, mergesText, cfgMaybe] = await Promise.all([
|
| 349 |
-
fetch(vocabURL).then(r => { if (!r.ok) throw new Error("missing vocab.json"); return r.text(); }),
|
| 350 |
-
fetch(mergesURL).then(r => { if (!r.ok) throw new Error("missing merges.txt"); return r.text(); }),
|
| 351 |
-
// tokenizer_config.json is optional; we'll sanitize it if present
|
| 352 |
-
fetch(new URL("tokenizer_config.json", base).href).then(r => r.ok ? r.json() : null),
|
| 353 |
-
]);
|
| 354 |
-
|
| 355 |
-
// IMPORTANT: pass merges as the **raw text string**
|
| 356 |
-
files.set("vocab.json", new Blob([vocabText], { type: "application/json" }));
|
| 357 |
-
files.set("merges.txt", new Blob([mergesText], { type: "text/plain" }));
|
| 358 |
-
|
| 359 |
-
// If you include tokenizer_config.json, strip fields that can confuse the BPE loader
|
| 360 |
-
if (cfgMaybe) {
|
| 361 |
-
const cfg = { ...cfgMaybe };
|
| 362 |
-
// These sometimes show up as arrays/objects; the BPE loader expects to read merges.txt itself.
|
| 363 |
-
if (Array.isArray(cfg.merges)) delete cfg.merges;
|
| 364 |
-
if (typeof cfg.merges === "object") delete cfg.merges;
|
| 365 |
-
// (Optional) also remove tokenizer_file pointers which can trigger alternate code paths
|
| 366 |
-
if (cfg.tokenizer_file) delete cfg.tokenizer_file;
|
| 367 |
-
|
| 368 |
-
files.set("tokenizer_config.json", new Blob([JSON.stringify(cfg)], { type: "application/json" }));
|
| 369 |
-
}
|
| 370 |
-
|
| 371 |
-
// Now build the tokenizer strictly from local files
|
| 372 |
-
tokenizer = await AutoTokenizer.from_pretrained("", {
|
| 373 |
-
files,
|
| 374 |
-
local_files_only: true,
|
| 375 |
-
progress_callback: onProgress,
|
| 376 |
-
});
|
| 377 |
-
|
| 378 |
// Core model state (module scope)
|
| 379 |
let tokenizer = null, model = null;
|
| 380 |
let loadSeq = 0;
|
|
|
|
| 340 |
}
|
| 341 |
} // end of loadTokenizerQwenLocal
|
| 342 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 343 |
// Core model state (module scope)
|
| 344 |
let tokenizer = null, model = null;
|
| 345 |
let loadSeq = 0;
|