File size: 607 Bytes
8f6d79d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | """Document ingest — raw text / bytes → UTF-8 string."""
from __future__ import annotations
def ingest_text(source: str | bytes | None, *, encoding: str = "utf-8") -> str:
"""Normalize input into a UTF-8 Unicode string."""
if source is None:
return ""
if isinstance(source, bytes):
for enc in (encoding, "utf-8", "utf-8-sig", "latin-1"):
try:
return source.decode(enc)
except (UnicodeDecodeError, LookupError):
continue
return source.decode("utf-8", errors="replace")
return str(source)
|