vomebook commited on
Commit
a97d5a7
·
verified ·
1 Parent(s): 5f4a036

Improve TXT encoding detection

Browse files
Files changed (1) hide show
  1. static/reader.js +1 -2
static/reader.js CHANGED
@@ -458,8 +458,7 @@ function detectTextEncoding(bytes, hint = "") {
458
  const cjk = (text.match(/[\u3400-\u9fff]/g) || []).length;
459
  const cyrillic = (text.match(/[\u0400-\u04ff]/g) || []).length;
460
  const commonCjk = (text.match(/[的一是在不了有和人这中大为上个国我以要他时来用们生到作地于出就分对成会可主发年动同工也能下过子说产种面而方后多定行学法所民得经之进着等部家自理起现实都体制当本性应开合因由然前外政社义事相全与关各重新内正反明原利质向道命此变结解问意建公系军情者立代通题党程展料员革文总品活长求老基资级图统知组别期论运农指区战任处理世]/g) || []).length;
461
- const commonCyrillic = (text.toLowerCase().match(/[оеаинтсрвлкмдпуяызьгчбйхжюшцщэфъ]/g) || []).length;
462
- const score = Math.max(cjk + commonCjk * 5, cyrillic + commonCyrillic) - controls * 20 - replacements * 40;
463
  if (score > bestScore) { best = encoding; bestScore = score; }
464
  } catch (_) {}
465
  }
 
458
  const cjk = (text.match(/[\u3400-\u9fff]/g) || []).length;
459
  const cyrillic = (text.match(/[\u0400-\u04ff]/g) || []).length;
460
  const commonCjk = (text.match(/[的一是在不了有和人这中大为上个国我以要他时来用们生到作地于出就分对成会可主发年动同工也能下过子说产种面而方后多定行学法所民得经之进着等部家自理起现实都体制当本性应开合因由然前外政社义事相全与关各重新内正反明原利质向道命此变结解问意建公系军情者立代通题党程展料员革文总品活长求老基资级图统知组别期论运农指区战任处理世]/g) || []).length;
461
+ const score = Math.max(cjk + commonCjk * 5, cyrillic) - controls * 20 - replacements * 40;
 
462
  if (score > bestScore) { best = encoding; bestScore = score; }
463
  } catch (_) {}
464
  }