vomebook commited on
Commit
c3dfe9f
·
verified ·
1 Parent(s): 2f0752b

Skip damaged TXT bytes

Browse files
Files changed (1) hide show
  1. static/reader.js +5 -5
static/reader.js CHANGED
@@ -402,7 +402,7 @@ async function renderPlainText(response) {
402
  const textNode = document.createTextNode(""); pre.appendChild(textNode); content.appendChild(pre);
403
  if (!response.body || !response.body.getReader) {
404
  const bytes = new Uint8Array(await response.arrayBuffer());
405
- textNode.data = new TextDecoder(detectTextEncoding(bytes, title)).decode(bytes);
406
  return;
407
  }
408
  const reader = response.body.getReader(), chunks = [];
@@ -425,15 +425,15 @@ async function renderPlainText(response) {
425
  let pending = "", frame = 0;
426
  const flush = () => { frame = 0; if (pending) { textNode.appendData(pending); pending = ""; } };
427
  const scheduleFlush = () => { if (!frame) frame = requestAnimationFrame(flush); };
428
- pending = decoder.decode(sample.subarray(displayedSampleSize), { stream: !streamDone });
429
  scheduleFlush();
430
  while (!streamDone) {
431
  const { value, done } = await reader.read();
432
  if (done) { streamDone = true; break; }
433
- pending += decoder.decode(value, { stream: true });
434
  scheduleFlush();
435
  }
436
- pending += decoder.decode();
437
  if (frame) cancelAnimationFrame(frame);
438
  flush();
439
  }
@@ -447,7 +447,7 @@ function detectTextEncoding(bytes, hint = "") {
447
  if (evenNulls > bytes.length / 8 && evenNulls > oddNulls * 4) return "utf-16be";
448
  try { new TextDecoder("utf-8", { fatal: true }).decode(bytes, { stream: true }); return "utf-8"; } catch (_) {}
449
  if (/[\u0400-\u04ff]/.test(hint)) return "windows-1251";
450
- const candidates = ["gb18030", "big5", "windows-1251", "windows-1252"];
451
  let best = "gb18030", bestScore = -Infinity;
452
  for (const encoding of candidates) {
453
  try {
 
402
  const textNode = document.createTextNode(""); pre.appendChild(textNode); content.appendChild(pre);
403
  if (!response.body || !response.body.getReader) {
404
  const bytes = new Uint8Array(await response.arrayBuffer());
405
+ textNode.data = new TextDecoder(detectTextEncoding(bytes, title)).decode(bytes).replace(/\ufffd/g, "");
406
  return;
407
  }
408
  const reader = response.body.getReader(), chunks = [];
 
425
  let pending = "", frame = 0;
426
  const flush = () => { frame = 0; if (pending) { textNode.appendData(pending); pending = ""; } };
427
  const scheduleFlush = () => { if (!frame) frame = requestAnimationFrame(flush); };
428
+ pending = decoder.decode(sample.subarray(displayedSampleSize), { stream: !streamDone }).replace(/\ufffd/g, "");
429
  scheduleFlush();
430
  while (!streamDone) {
431
  const { value, done } = await reader.read();
432
  if (done) { streamDone = true; break; }
433
+ pending += decoder.decode(value, { stream: true }).replace(/\ufffd/g, "");
434
  scheduleFlush();
435
  }
436
+ pending += decoder.decode().replace(/\ufffd/g, "");
437
  if (frame) cancelAnimationFrame(frame);
438
  flush();
439
  }
 
447
  if (evenNulls > bytes.length / 8 && evenNulls > oddNulls * 4) return "utf-16be";
448
  try { new TextDecoder("utf-8", { fatal: true }).decode(bytes, { stream: true }); return "utf-8"; } catch (_) {}
449
  if (/[\u0400-\u04ff]/.test(hint)) return "windows-1251";
450
+ const candidates = /[\u3400-\u9fff]/.test(hint) ? ["gb18030", "big5"] : ["gb18030", "big5", "windows-1251", "windows-1252"];
451
  let best = "gb18030", bestScore = -Infinity;
452
  for (const encoding of candidates) {
453
  try {