scripts/seed_all_posts.py

#!/usr/bin/env python3
"""
Scan the drafts/ directory and seed every markdown file into the database.
Also seeds a few hardcoded sample posts if no drafts exist.
Safe to re-run — skips existing slugs.
"""

from __future__ import annotations

import re
import sys
import os
from datetime import datetime
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from app import app, db, Post, Subscriber, prepare_post_content, generate_slug, calculate_content_hash

DRAFTS_DIR = PROJECT_ROOT / "drafts"

# Simple YAML frontmatter parser (avoids pyyaml dependency)
FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL)
HEADING_RE = re.compile(r"^#\s+(.+)$", re.MULTILINE)
DATE_PREFIX_RE = re.compile(r"^(\d{4}-\d{2}-\d{2})-(.+)$")


def parse_frontmatter(text: str) -> tuple[dict[str, str], str]:
    """Return (metadata_dict, body_without_frontmatter)."""
    match = FRONTMATTER_RE.match(text)
    if not match:
        return {}, text

    raw = match.group(1)
    body = text[match.end():]
    meta: dict[str, str] = {}
    for line in raw.splitlines():
        if ":" in line:
            key, _, value = line.partition(":")
            meta[key.strip().lower()] = value.strip().strip('"').strip("'")
    return meta, body


def slug_from_filename(filename: str) -> str:
    """Derive a URL slug from the markdown filename."""
    stem = Path(filename).stem
    m = DATE_PREFIX_RE.match(stem)
    if m:
        return m.group(2)
    return stem


def date_from_filename(filename: str) -> datetime | None:
    m = DATE_PREFIX_RE.match(Path(filename).stem)
    if m:
        try:
            return datetime.strptime(m.group(1), "%Y-%m-%d")
        except ValueError:
            return None
    return None


def title_from_content(meta: dict[str, str], body: str, slug: str) -> str:
    """Extract title from frontmatter, first heading, or slug."""
    if meta.get("title"):
        return meta["title"]
    heading_match = HEADING_RE.search(body)
    if heading_match:
        return heading_match.group(1).strip()
    return slug.replace("-", " ").title()


def summary_from_content(body: str, max_len: int = 300) -> str:
    """Extract a plain-text summary from the body."""
    # Strip markdown formatting for a clean summary
    text = re.sub(r'#+ ', '', body)
    text = re.sub(r'\*\*([^*]+)\*\*', r'\1', text)
    text = re.sub(r'\*([^*]+)\*', r'\1', text)
    text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', text)
    text = re.sub(r'[-*] ', '', text)
    text = re.sub(r'\n+', ' ', text).strip()
    if len(text) > max_len:
        text = text[:max_len].rsplit(' ', 1)[0] + '...'
    return text


def seed_from_drafts():
    """Scan drafts/ and add all markdown files to the database."""
    if not DRAFTS_DIR.is_dir():
        print(f"  No drafts directory found at {DRAFTS_DIR}")
        return 0

    md_files = sorted(DRAFTS_DIR.glob("*.md"))
    if not md_files:
        print("  No markdown files found in drafts/")
        return 0

    added = 0
    skipped = 0

    for md_path in md_files:
        raw = md_path.read_text(encoding="utf-8")
        meta, body = parse_frontmatter(raw)

        slug = slug_from_filename(md_path.name)
        title = title_from_content(meta, body, slug)
        category = meta.get("category", "Excavation Reports")
        source_url = meta.get("source_url", "")
        source_name = meta.get("source_name", "")

        pub_date = date_from_filename(md_path.name)
        if not pub_date and meta.get("date"):
            try:
                pub_date = datetime.strptime(meta["date"], "%Y-%m-%d")
            except ValueError:
                pub_date = None
        if not pub_date:
            pub_date = datetime.utcnow()

        # Skip if slug already exists
        if Post.query.filter_by(slug=slug).first():
            print(f"  [skip] {slug}")
            skipped += 1
            continue

        # Render markdown to HTML
        rendered = prepare_post_content(body.strip())
        content_hash = calculate_content_hash(body.strip())
        summary = summary_from_content(body)

        post = Post(
            title=title,
            slug=slug,
            content=rendered,
            summary=summary,
            pub_date=pub_date,
            category=category,
            source_url=source_url,
            source_name=source_name,
            published=True,
            content_hash=content_hash,
        )
        db.session.add(post)
        db.session.commit()
        print(f"  [added] {slug} — {title}")
        added += 1

        # Notify subscribers about the new article (if email is configured)
        if os.environ.get('BREVO_API_KEY') and os.environ.get('NOTIFY_ON_SEED', '').lower() == 'true':
            try:
                from email_service import send_new_article_notification
                subs = [s.email for s in Subscriber.query.filter_by(confirmed=True).all()]
                if subs:
                    sent, failed = send_new_article_notification(
                        subscribers=subs,
                        post_title=title,
                        post_slug=slug,
                        post_summary=summary,
                        post_category=category,
                    )
                    print(f"    [email] Notified {sent} subscribers ({failed} failed)")
            except Exception as exc:
                print(f"    [email] Notification failed: {exc}")

    print(f"\nSeeding complete: {added} added, {skipped} skipped.")
    return added


if __name__ == "__main__":
    with app.app_context():
        db.create_all()
        seed_from_drafts()