Spaces:
Sleeping
Sleeping
File size: 3,682 Bytes
2a2a176 a59c3c4 d1f47f9 2a2a176 d1f47f9 2a2a176 a59c3c4 d1f47f9 a59c3c4 d1f47f9 a59c3c4 d1f47f9 2a2a176 d1f47f9 a59c3c4 2a2a176 d1f47f9 a59c3c4 2a2a176 a59c3c4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 | """A picture's content (see reading.py): the content the most scanners return, raw bytes and the text a scanner makes
of them counting as one content, and the content as the page may show it, with digests that pair pictures of one
content whichever scanners read them.
"""
from __future__ import annotations
import hashlib
def _texts(p: bytes) -> set[bytes]:
"""In UTF-8, the texts a scanner may make of raw bytes that aren't UTF-8: ISO-8859-1 (QR byte mode's default)
and, where they decode, Shift JIS (QR kanji mode)."""
try:
p.decode("utf-8")
return set()
except UnicodeDecodeError:
pass
out = {p.decode("latin-1").encode("utf-8")}
try:
out.add(p.decode("shift_jis").encode("utf-8"))
except UnicodeDecodeError:
pass
return out
def _same(p: bytes, q: bytes) -> bool:
"""One content: the same bytes, or raw bytes and a text made of them."""
return p == q or p in _texts(q) or q in _texts(p)
def _is_text(p: bytes) -> bool:
try:
p.decode("utf-8")
return True
except UnicodeDecodeError:
return False
def _consensus(contents: dict[tuple[str, int], list[bytes]]) -> bytes | None:
forms = {p for ps in contents.values() for p in ps}
if not forms:
return None
def votes(c: bytes) -> tuple[int, int]:
hits = [(s, k) for (s, k), ps in contents.items() if any(_same(p, c) for p in ps)]
return len({s for s, _ in hits}), len(hits)
return max(forms, key=lambda c: (*votes(c), _is_text(c), -len(c), c))
def _raws(p: bytes) -> list[bytes]:
"""Every set of raw bytes a content stands for: p, when p is raw bytes already (not UTF-8); else, in this order,
the ISO-8859-1 and the Shift JIS bytes of which p is a text a scanner made, or p itself when it is neither. A text
with only ISO-8859-1 characters ("25°C") can be made of either, so it stands for both."""
try:
text = p.decode("utf-8")
except UnicodeDecodeError:
return [p] # raw bytes already
out = []
for encoding in ("latin-1", "shift_jis"):
try:
raw = text.encode(encoding)
except UnicodeEncodeError:
continue
if raw != p and p in _texts(raw):
out.append(raw)
return out or [p]
def _raw(p: bytes) -> bytes:
"""The raw bytes content.sha256 is taken from: the first of _raws, so one content (_same) has one digest whichever
scanners read it, but for a text that stands for two byte strings, which content.sha256s covers."""
return _raws(p)[0]
def describe(content: bytes | None) -> dict | None:
"""The content as the page may show it: printable text (UTF-8, or else ISO-8859-1), otherwise only its length;
its SHA-256 as one content's raw bytes (_raw); and "sha256s", the sorted SHA-256s of every set of raw bytes it
stands for (_raws), which two pictures of one content always share. Only the visitor's browser receives them, to
pair pictures."""
if content is None:
return None
digest = hashlib.sha256(_raw(content)).hexdigest()
digests = sorted({hashlib.sha256(r).hexdigest() for r in _raws(content)}) # pairing: one content shares one
try:
text = content.decode("utf-8")
except UnicodeDecodeError:
text = content.decode("latin-1") # QR byte mode's default; binary content shows control characters
if all(c.isprintable() or c in "\r\n\t" for c in text):
return {"text": text[:300], "cut": len(text) > 300, "bytes": len(content), "sha256": digest, "sha256s": digests}
return {"text": None, "cut": False, "bytes": len(content), "sha256": digest, "sha256s": digests}
|