fix: extract dates from article pages via trafilatura instead of homepage HTML

main.py

@@ -66,6 +66,8 @@ def run_pipeline(items: list[NewsItem], cfg: Config, skip_extraction: bool = Fal
                     source_domain=item.post.source_domain or "reddit.com",
                     extraction_success=False,
                     image_url=item.post.image_url,
+                    published=item.post.published,
+                    published_iso=item.post.published_iso,
                 )
                 logger.info("  [%2d/%d] %-60s ✗  (title only)", i, len(items), item.post.title[:60])
             item.article = article

refresh_task.py

@@ -61,8 +61,8 @@ def serialize_clusters(clusters):
                     "comments": a.post.num_comments,
                     "url": a.post.url,
                     "image": a.article.image_url if a.article else a.post.image_url,
-                    "published": a.post.published,
-                    "published_iso": a.post.published_iso,
+                    "published": a.article.published or a.post.published,
+                    "published_iso": a.article.published_iso or a.post.published_iso,
                 }
                 for a in c.articles[:5]
             ],
@@ -92,8 +92,8 @@ def serialize_clusters(clusters):
                     "score": a.post.score,
                     "comments": a.post.num_comments,
                     "image": a.article.image_url if a.article else a.post.image_url,
-                    "published": a.post.published,
-                    "published_iso": a.post.published_iso,
+                    "published": a.article.published or a.post.published,
+                    "published_iso": a.article.published_iso or a.post.published_iso,
                 }
                 for a in c.articles[:5]
             ],

src/extractor.py

@@ -33,12 +33,17 @@ class ArticleExtractor:
         downloaded = self._trafilatura.fetch_url(url)
         if not downloaded:
             return None
-        text = self._trafilatura.extract(downloaded)
-        if not text:
+        result = self._trafilatura.extract(downloaded, output_format='python')
+        if not result or not result.get('text'):
             return None
-        title = self._extract_title_meta(downloaded) or ""
+        text = result['text']
+        title = result.get('title') or self._extract_title_meta(downloaded) or ""
         image = self._extract_og_image(downloaded)
-        return Article(url=url, title=title, text=text, source_domain=domain, image_url=image)
+        published, published_iso = self._parse_trafilatura_date(result.get('date'))
+        return Article(
+            url=url, title=title, text=text, source_domain=domain,
+            image_url=image, published=published, published_iso=published_iso,
+        )
 
     def _extract_fallback(self, url: str, domain: str) -> Optional[Article]:
         resp = requests.get(url, headers={"User-Agent": "newsapp/1.0"}, timeout=15)
@@ -50,12 +55,40 @@ class ArticleExtractor:
             title = soup.title.get_text(strip=True)
 
         image = self._extract_og_image_soup(soup)
+        published, published_iso = self._extract_fallback_date(soup)
 
         paragraphs = soup.find_all("p")
         text = "\n\n".join(p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True))
         if not text:
             return None
-        return Article(url=url, title=title, text=text, source_domain=domain, image_url=image)
+        return Article(
+            url=url, title=title, text=text, source_domain=domain,
+            image_url=image, published=published, published_iso=published_iso,
+        )
+
+    @staticmethod
+    def _parse_trafilatura_date(raw_date: str | None) -> tuple[str, str]:
+        if not raw_date:
+            return ("", "")
+        from datetime import datetime
+        for fmt in ("%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
+            try:
+                dt = datetime.strptime(raw_date.strip(), fmt)
+                return (dt.strftime("%d %b %Y"), dt.strftime("%Y-%m-%d"))
+            except ValueError:
+                continue
+        try:
+            dt = datetime.fromisoformat(raw_date.strip())
+            return (dt.strftime("%d %b %Y"), dt.strftime("%Y-%m-%d"))
+        except (ValueError, TypeError):
+            pass
+        if len(raw_date) >= 10:
+            try:
+                dt = datetime.strptime(raw_date[:10], "%Y-%m-%d")
+                return (dt.strftime("%d %b %Y"), raw_date[:10])
+            except ValueError:
+                pass
+        return ("", "")
 
     @staticmethod
     def _extract_title_meta(html: str) -> Optional[str]:
@@ -79,6 +112,47 @@ class ArticleExtractor:
         )
         return m.group(1) if m else ""
 
+    @staticmethod
+    def _extract_fallback_date(soup) -> tuple[str, str]:
+        from datetime import datetime
+        time_tag = soup.find("time")
+        if time_tag:
+            raw = time_tag.get("datetime") or time_tag.get_text(strip=True)
+            if raw:
+                for fmt in ("%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
+                    try:
+                        dt = datetime.strptime(raw.strip(), fmt)
+                        return (dt.strftime("%d %b %Y"), dt.strftime("%Y-%m-%d"))
+                    except ValueError:
+                        continue
+                try:
+                    dt = datetime.fromisoformat(raw.strip())
+                    return (dt.strftime("%d %b %Y"), dt.strftime("%Y-%m-%d"))
+                except (ValueError, TypeError):
+                    pass
+        for prop in ("article:published_time", "date"):
+            tag = soup.find("meta", property=prop) or soup.find("meta", attrs={"name": prop})
+            if tag and tag.get("content"):
+                raw = tag["content"].strip()
+                for fmt in ("%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
+                    try:
+                        dt = datetime.strptime(raw, fmt)
+                        return (dt.strftime("%d %b %Y"), dt.strftime("%Y-%m-%d"))
+                    except ValueError:
+                        continue
+                try:
+                    dt = datetime.fromisoformat(raw)
+                    return (dt.strftime("%d %b %Y"), dt.strftime("%Y-%m-%d"))
+                except (ValueError, TypeError):
+                    pass
+                if len(raw) >= 10:
+                    try:
+                        dt = datetime.strptime(raw[:10], "%Y-%m-%d")
+                        return (dt.strftime("%d %b %Y"), raw[:10])
+                    except ValueError:
+                        pass
+        return ("", "")
+
     @staticmethod
     def _extract_og_image_soup(soup) -> str:
         for prop in ("og:image", "twitter:image"):

src/models.py

@@ -24,6 +24,8 @@ class Article:
     source_domain: str
     extraction_success: bool = True
     image_url: str = ""
+    published: str = ""
+    published_iso: str = ""
 
 
 @dataclass

src/presenter.py

@@ -26,7 +26,7 @@ class NewsPresenter:
 
         for cat in NewsPresenter.CATEGORIES:
             items = by_cat.get(cat, [])
-            items.sort(key=lambda x: (x.articles[0].post.published_iso or "", x.final_score), reverse=True)
+            items.sort(key=lambda x: (x.articles[0].article.published_iso or x.articles[0].post.published_iso or "", x.final_score), reverse=True)
             items = items[:top_n]
             if not items:
                 continue
@@ -38,8 +38,9 @@ class NewsPresenter:
                 item = cluster.articles[0]
                 title = item.article.title[:72] + "…" if item.article.title and len(item.article.title) > 72 else (item.article.title or item.post.title)
                 print(f"       {title}")
-                if item.post.published:
-                    print(f"       📅 {item.post.published}")
+                published = item.article.published or item.post.published
+                if published:
+                    print(f"       📅 {published}")
                 if item.analysis and item.analysis.summary:
                     short = item.analysis.summary[:72] + "…" if len(item.analysis.summary) > 72 else item.analysis.summary
                     print(f"       → {short}")

webapp.py

@@ -66,8 +66,8 @@ def _cluster_to_html_dict(c):
                 "score": a.post.score,
                 "comments": a.post.num_comments,
                 "image": a.article.image_url if a.article else a.post.image_url,
-                "published": a.post.published,
-                "published_iso": a.post.published_iso,
+                "published": a.article.published or a.post.published,
+                "published_iso": a.article.published_iso or a.post.published_iso,
             }
             for a in c.articles[:5]
         ],
@@ -105,8 +105,8 @@ def _cluster_to_api_dict(c):
                 "score": a.post.score,
                 "comments": a.post.num_comments,
                 "image": a.article.image_url if a.article else a.post.image_url,
-                "published": a.post.published,
-                "published_iso": a.post.published_iso,
+                "published": a.article.published or a.post.published,
+                "published_iso": a.article.published_iso or a.post.published_iso,
             }
             for a in c.articles[:5]
         ],