Spaces:
Sleeping
Sleeping
| import hashlib | |
| import feedparser | |
| from app.contracts.signal_models import SignalCard, SignalScores | |
| class SignalNormalizer: | |
| def normalize(self, item: dict) -> SignalCard: | |
| t = item.get("type") or "other" | |
| # arXiv path: parse atom feed from raw text | |
| if item.get("source_key") == "arxiv" and "raw" in item: | |
| feed = feedparser.parse(item["raw"]) | |
| cards = [] | |
| for e in feed.entries[:10]: | |
| url = getattr(e, "link", "") or "" | |
| title = getattr(e, "title", "") or "arXiv item" | |
| published = getattr(e, "published", None) | |
| summary = getattr(e, "summary", "") or "" | |
| sid = self._id(url or title) | |
| cards.append(SignalCard( | |
| id=sid, | |
| type="research", | |
| title=title.strip(), | |
| publisher="arXiv", | |
| published_at=published, | |
| url=url or item.get("fetch_url", ""), | |
| summary_1line=(summary.strip()[:180] + "…") if len(summary.strip()) > 180 else summary.strip(), | |
| scores=SignalScores(), | |
| raw={"entry": dict(e)} | |
| )) | |
| # return as a packed "bundle" marker? We'll flatten later; easiest: | |
| # if multiple parsed, return the first and stash rest in raw | |
| if cards: | |
| # put remaining in raw for later flatten in active intent | |
| first = cards[0] | |
| first.raw["__bundle__"] = [c.model_dump() for c in cards[1:]] | |
| return first | |
| # RSS/news path: | |
| url = item.get("url") or "" | |
| title = (item.get("title") or "").strip() or "Untitled" | |
| published = item.get("published") | |
| publisher = item.get("publisher") | |
| summary = (item.get("summary") or "").strip() | |
| summary_1line = (summary[:180] + "…") if len(summary) > 180 else summary | |
| return SignalCard( | |
| id=self._id(url or title), | |
| type=t, | |
| title=title, | |
| publisher=publisher, | |
| published_at=published, | |
| url=url, | |
| summary_1line=summary_1line, | |
| scores=SignalScores(), | |
| raw=item, | |
| ) | |
| def _id(self, s: str) -> str: | |
| return hashlib.sha1(s.encode("utf-8", errors="ignore")).hexdigest()[:16] | |