import hashlib import json from pathlib import Path from typing import Any def content_hash(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() def url_hash(url: str) -> str: return hashlib.md5(url.encode("utf-8")).hexdigest() def normalize_url(url: str) -> str: url = url.strip().rstrip("/") if url.startswith("http://"): url = url.replace("http://", "https://", 1) return url def save_json(data: Any, path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) with open(path, "w", encoding="utf-8") as f: json.dump(data, f, indent=2, ensure_ascii=False) def load_json(path: Path) -> Any: with open(path, "r", encoding="utf-8") as f: return json.load(f) def truncate_text(text: str, max_chars: int = 200) -> str: if len(text) <= max_chars: return text return text[: max_chars - 3] + "..."