import hashlib import feedparser from app.contracts.signal_models import SignalCard, SignalScores class SignalNormalizer: def normalize(self, item: dict) -> SignalCard: t = item.get("type") or "other" # arXiv path: parse atom feed from raw text if item.get("source_key") == "arxiv" and "raw" in item: feed = feedparser.parse(item["raw"]) cards = [] for e in feed.entries[:10]: url = getattr(e, "link", "") or "" title = getattr(e, "title", "") or "arXiv item" published = getattr(e, "published", None) summary = getattr(e, "summary", "") or "" sid = self._id(url or title) cards.append(SignalCard( id=sid, type="research", title=title.strip(), publisher="arXiv", published_at=published, url=url or item.get("fetch_url", ""), summary_1line=(summary.strip()[:180] + "…") if len(summary.strip()) > 180 else summary.strip(), scores=SignalScores(), raw={"entry": dict(e)} )) # return as a packed "bundle" marker? We'll flatten later; easiest: # if multiple parsed, return the first and stash rest in raw if cards: # put remaining in raw for later flatten in active intent first = cards[0] first.raw["__bundle__"] = [c.model_dump() for c in cards[1:]] return first # RSS/news path: url = item.get("url") or "" title = (item.get("title") or "").strip() or "Untitled" published = item.get("published") publisher = item.get("publisher") summary = (item.get("summary") or "").strip() summary_1line = (summary[:180] + "…") if len(summary) > 180 else summary return SignalCard( id=self._id(url or title), type=t, title=title, publisher=publisher, published_at=published, url=url, summary_1line=summary_1line, scores=SignalScores(), raw=item, ) def _id(self, s: str) -> str: return hashlib.sha1(s.encode("utf-8", errors="ignore")).hexdigest()[:16]