Spaces:
Running
Running
| """URL-hash dedup (ADR-0015: dedup runs before classification, so we never | |
| spend a Groq call on something we're about to discard as already-seen).""" | |
| import hashlib | |
| def url_hash(url: str) -> str: | |
| # Trailing slashes and query strings are common no-op variants of the same | |
| # article URL — strip them so the same piece of content doesn't get | |
| # rescraped as "new" on a second poll just because of a tracking param. | |
| normalized = url.strip().rstrip("/").split("?", 1)[0].split("#", 1)[0] | |
| return hashlib.sha256(normalized.encode("utf-8")).hexdigest() | |