scraper/items.py

"""Scraped data items definition."""
from typing import Optional, List, Dict, Any
from dataclasses import dataclass, field
from datetime import datetime


@dataclass
class ScrapedItem:
    """Base scraped item."""

    url: str
    scraped_at: datetime = field(default_factory=datetime.utcnow)
    source: Optional[str] = None


@dataclass
class ProductItem(ScrapedItem):
    """Product item for e-commerce scraping."""

    name: Optional[str] = None
    price: Optional[float] = None
    currency: str = "TWD"
    description: Optional[str] = None
    images: List[str] = field(default_factory=list)
    category: Optional[str] = None
    sku: Optional[str] = None
    availability: Optional[str] = None
    rating: Optional[float] = None
    reviews_count: int = 0
    metadata: Dict[str, Any] = field(default_factory=dict)


@dataclass
class ArticleItem(ScrapedItem):
    """Article item for content scraping."""

    title: Optional[str] = None
    author: Optional[str] = None
    publish_date: Optional[datetime] = None
    content: Optional[str] = None
    summary: Optional[str] = None
    tags: List[str] = field(default_factory=list)
    images: List[str] = field(default_factory=list)
    reading_time: int = 0  # minutes


@dataclass
class ProfileItem(ScrapedItem):
    """Profile item for social media scraping."""

    username: Optional[str] = None
    display_name: Optional[str] = None
    bio: Optional[str] = None
    avatar_url: Optional[str] = None
    followers: int = 0
    following: int = 0
    posts_count: int = 0
    verified: bool = False
    location: Optional[str] = None
    website: Optional[str] = None


def item_to_dict(item: ScrapedItem) -> Dict[str, Any]:
    """Convert item to dictionary."""
    result = {
        "url": item.url,
        "scraped_at": item.scraped_at.isoformat() if item.scraped_at else None,
        "source": item.source,
    }

    # Add item-specific fields
    if isinstance(item, ProductItem):
        result.update(
            {
                "type": "product",
                "name": item.name,
                "price": item.price,
                "currency": item.currency,
                "description": item.description,
                "images": item.images,
                "category": item.category,
                "sku": item.sku,
                "availability": item.availability,
                "rating": item.rating,
                "reviews_count": item.reviews_count,
                "metadata": item.metadata,
            }
        )
    elif isinstance(item, ArticleItem):
        result.update(
            {
                "type": "article",
                "title": item.title,
                "author": item.author,
                "publish_date": item.publish_date.isoformat() if item.publish_date else None,
                "content": item.content,
                "summary": item.summary,
                "tags": item.tags,
                "images": item.images,
                "reading_time": item.reading_time,
            }
        )
    elif isinstance(item, ProfileItem):
        result.update(
            {
                "type": "profile",
                "username": item.username,
                "display_name": item.display_name,
                "bio": item.bio,
                "avatar_url": item.avatar_url,
                "followers": item.followers,
                "following": item.following,
                "posts_count": item.posts_count,
                "verified": item.verified,
                "location": item.location,
                "website": item.website,
            }
        )

    return result