Spaces:
Running
Running
File size: 3,124 Bytes
b7c46f4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | """
News → permanent event history. The news pipeline (ADR-0015) decides what users
*see* and lets items expire from the feed; this keeps every published item as a
dated, located event because, for the correlation step, past events are the
history — the reason the `threat` value was removed (no value in showing
after-the-fact news) doesn't apply to *learning* from it.
record_news_item() is called from the news pipeline and the admin publish route.
It never raises and is a no-op when the intel store is off (INTEL_DB_PATH unset),
so the deployed backend is unaffected.
"""
from datetime import date, datetime
from intel import db
from intel.locations import resolve
SOURCE_ID = "news_pipeline"
# news categories → event family; identical names except where noted.
CATEGORY_TO_FAMILY = {
"policy": "policy", "weather": "weather", "infrastructure": "infrastructure",
"funding": "funding", "security": "security", "other": "other",
}
def _day(value) -> str | None:
if value is None:
return None
if isinstance(value, (date, datetime)):
return value.date().isoformat() if isinstance(value, datetime) else value.isoformat()
return str(value)[:10]
def record_news_item(item: dict) -> bool:
"""item: a news_items-shaped dict (url_hash, title, raw_text, category,
location, published_at, scraped_at, deadline). Returns True if an event was
written. Only published (status 'classified') items should be passed in."""
if not db.enabled():
return False
try:
con = db.connect()
try:
location_id = resolve(con, item.get("location")) or resolve(con, item.get("title"))
con.execute(
"""insert into events (id, family, type, location_id, starts_on, ends_on, title, detail,
is_estimate, source_id, source_ref, ingested_at)
values (?, ?, 'news', ?, ?, ?, ?, ?, 0, ?, ?, ?)
on conflict (source_id, source_ref) do update set family = excluded.family,
location_id = excluded.location_id, title = excluded.title, detail = excluded.detail""",
(
f"{SOURCE_ID}:{item['url_hash']}",
CATEGORY_TO_FAMILY.get(item.get("category") or "other", "other"),
location_id,
_day(item.get("published_at")) or _day(item.get("scraped_at")),
_day(item.get("deadline")),
item["title"],
(item.get("raw_text") or "")[:2000] or None,
SOURCE_ID,
item["url_hash"],
db.now_iso(),
),
)
con.commit()
from intel.freshness import record_attempt
record_attempt(con, SOURCE_ID, ok=True, latest_data_period=_day(item.get("published_at")) or _day(item.get("scraped_at")))
return True
finally:
con.close()
except Exception as e:
print(f"[intel] could not record news event {item.get('url_hash')}: {e}")
return False
|