Spaces:
Running
Running
| """ | |
| News → permanent event history. The news pipeline (ADR-0015) decides what users | |
| *see* and lets items expire from the feed; this keeps every published item as a | |
| dated, located event because, for the correlation step, past events are the | |
| history — the reason the `threat` value was removed (no value in showing | |
| after-the-fact news) doesn't apply to *learning* from it. | |
| record_news_item() is called from the news pipeline and the admin publish route. | |
| It never raises and is a no-op when the intel store is off (INTEL_DB_PATH unset), | |
| so the deployed backend is unaffected. | |
| """ | |
| from datetime import date, datetime | |
| from intel import db | |
| from intel.locations import resolve | |
| SOURCE_ID = "news_pipeline" | |
| # news categories → event family; identical names except where noted. | |
| CATEGORY_TO_FAMILY = { | |
| "policy": "policy", "weather": "weather", "infrastructure": "infrastructure", | |
| "funding": "funding", "security": "security", "other": "other", | |
| } | |
| def _day(value) -> str | None: | |
| if value is None: | |
| return None | |
| if isinstance(value, (date, datetime)): | |
| return value.date().isoformat() if isinstance(value, datetime) else value.isoformat() | |
| return str(value)[:10] | |
| def record_news_item(item: dict) -> bool: | |
| """item: a news_items-shaped dict (url_hash, title, raw_text, category, | |
| location, published_at, scraped_at, deadline). Returns True if an event was | |
| written. Only published (status 'classified') items should be passed in.""" | |
| if not db.enabled(): | |
| return False | |
| try: | |
| con = db.connect() | |
| try: | |
| location_id = resolve(con, item.get("location")) or resolve(con, item.get("title")) | |
| con.execute( | |
| """insert into events (id, family, type, location_id, starts_on, ends_on, title, detail, | |
| is_estimate, source_id, source_ref, ingested_at) | |
| values (?, ?, 'news', ?, ?, ?, ?, ?, 0, ?, ?, ?) | |
| on conflict (source_id, source_ref) do update set family = excluded.family, | |
| location_id = excluded.location_id, title = excluded.title, detail = excluded.detail""", | |
| ( | |
| f"{SOURCE_ID}:{item['url_hash']}", | |
| CATEGORY_TO_FAMILY.get(item.get("category") or "other", "other"), | |
| location_id, | |
| _day(item.get("published_at")) or _day(item.get("scraped_at")), | |
| _day(item.get("deadline")), | |
| item["title"], | |
| (item.get("raw_text") or "")[:2000] or None, | |
| SOURCE_ID, | |
| item["url_hash"], | |
| db.now_iso(), | |
| ), | |
| ) | |
| con.commit() | |
| from intel.freshness import record_attempt | |
| record_attempt(con, SOURCE_ID, ok=True, latest_data_period=_day(item.get("published_at")) or _day(item.get("scraped_at"))) | |
| return True | |
| finally: | |
| con.close() | |
| except Exception as e: | |
| print(f"[intel] could not record news event {item.get('url_hash')}: {e}") | |
| return False | |