Upload 4 files

app/database/__init__.py

@@ -0,0 +1 @@
+# Database Package

app/database/connection.py

@@ -0,0 +1,170 @@
+"""
+============================================
+Database Connection Manager
+- Async SQLAlchemy Engine with Connection Pooling
+- Handles Render's free tier quirks:
+  * Sleep mode reconnection
+  * Connection drop recovery
+  * Pool pre-ping for stale connections
+============================================
+"""
+
+import logging
+from sqlalchemy.ext.asyncio import (
+    create_async_engine,
+    AsyncSession,
+    async_sessionmaker,
+    AsyncEngine,
+)
+from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy.pool import AsyncAdaptedQueuePool
+from sqlalchemy import text
+
+from app.config import get_database_url, settings
+
+logger = logging.getLogger(__name__)
+
+
+# ============================================
+# Base Model Class (all models inherit this)
+# ============================================
+class Base(DeclarativeBase):
+    """Base class for all database models."""
+    pass
+
+
+# ============================================
+# Engine Configuration
+# ============================================
+_engine: AsyncEngine | None = None
+_session_factory: async_sessionmaker[AsyncSession] | None = None
+
+
+def get_engine() -> AsyncEngine:
+    """
+    Create or return the async SQLAlchemy engine.
+    
+    Key settings for Render Free Tier:
+    - pool_pre_ping=True: Tests connection before using it
+      (prevents "connection closed" errors after DB sleep)
+    - pool_size=5: Small pool (free tier has limited connections)
+    - max_overflow=3: Allow 3 extra connections under load
+    - pool_recycle=300: Recycle connections every 5 minutes
+      (prevents stale connections from Render's idle timeout)
+    - pool_timeout=30: Wait max 30s for a connection from pool
+    """
+    global _engine
+
+    if _engine is None:
+        database_url = get_database_url()
+
+        logger.info("Creating database engine...")
+        logger.info(f"Database host: {database_url.split('@')[1].split('/')[0] if '@' in database_url else 'unknown'}")
+
+        _engine = create_async_engine(
+            database_url,
+            # --- Connection Pool Settings ---
+            poolclass=AsyncAdaptedQueuePool,
+            pool_pre_ping=True,        # CRITICAL: Check connection health
+            pool_size=5,               # Base connections in pool
+            max_overflow=3,            # Extra connections allowed
+            pool_recycle=300,          # Recycle every 5 min (300 sec)
+            pool_timeout=30,           # Wait 30s for available connection
+            
+            # --- Query Settings ---
+            echo=settings.DEBUG,       # Log SQL queries in debug mode
+            
+            # --- Connection Arguments ---
+            connect_args={
+                "timeout": 30,                    # Connection timeout
+                "command_timeout": 60,             # Query timeout
+                "server_settings": {
+                    "application_name": "novel_scraper",
+                    "idle_in_transaction_session_timeout": "60000",  # 60s
+                },
+            },
+        )
+        logger.info("Database engine created successfully.")
+
+    return _engine
+
+
+def get_session_factory() -> async_sessionmaker[AsyncSession]:
+    """
+    Create or return the async session factory.
+    """
+    global _session_factory
+
+    if _session_factory is None:
+        engine = get_engine()
+        _session_factory = async_sessionmaker(
+            bind=engine,
+            class_=AsyncSession,
+            expire_on_commit=False,    # Don't expire objects after commit
+            autocommit=False,
+            autoflush=False,
+        )
+        logger.info("Session factory created.")
+
+    return _session_factory
+
+
+async def get_db_session() -> AsyncSession:
+    """
+    Dependency for FastAPI routes.
+    Yields an async database session.
+    
+    Usage in FastAPI:
+        @router.get("/something")
+        async def my_route(db: AsyncSession = Depends(get_db_session)):
+            ...
+    """
+    factory = get_session_factory()
+    async with factory() as session:
+        try:
+            yield session
+            await session.commit()
+        except Exception as e:
+            await session.rollback()
+            logger.error(f"Database session error: {e}")
+            raise
+        finally:
+            await session.close()
+
+
+async def init_database():
+    """
+    Initialize database: Create all tables.
+    Called on application startup.
+    """
+    engine = get_engine()
+
+    try:
+        # Test connection first
+        async with engine.begin() as conn:
+            await conn.execute(text("SELECT 1"))
+        logger.info("✅ Database connection test successful!")
+
+        # Create all tables
+        async with engine.begin() as conn:
+            await conn.run_sync(Base.metadata.create_all)
+        logger.info("✅ Database tables created/verified!")
+
+    except Exception as e:
+        logger.error(f"❌ Database initialization failed: {e}")
+        logger.error("Check your DATABASE_URL environment variable!")
+        raise
+
+
+async def close_database():
+    """
+    Close database connections.
+    Called on application shutdown.
+    """
+    global _engine, _session_factory
+
+    if _engine is not None:
+        await _engine.dispose()
+        _engine = None
+        _session_factory = None
+        logger.info("Database connections closed.")

app/database/crud.py

@@ -0,0 +1,275 @@
+"""
+============================================
+CRUD Operations (Create, Read, Update, Delete)
+All database operations go through here.
+Includes comprehensive error handling.
+============================================
+"""
+
+import logging
+from typing import Optional, List
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import select, update, func, delete
+from sqlalchemy.exc import (
+    IntegrityError,
+    OperationalError,
+    DisconnectionError,
+    TimeoutError as SATimeoutError,
+)
+
+from app.database.models import Novel, Chapter, NovelStatus
+
+logger = logging.getLogger(__name__)
+
+
+# ============================================
+# Retry Decorator for DB Operations
+# ============================================
+def handle_db_errors(func):
+    """
+    Decorator to handle common database errors gracefully.
+    Catches connection drops, timeouts, and retries logic.
+    """
+    async def wrapper(*args, **kwargs):
+        try:
+            return await func(*args, **kwargs)
+        except DisconnectionError as e:
+            logger.error(f"Database disconnected during {func.__name__}: {e}")
+            logger.info("This usually happens when Render DB wakes from sleep. Retrying...")
+            raise
+        except OperationalError as e:
+            logger.error(f"Database operational error in {func.__name__}: {e}")
+            raise
+        except SATimeoutError as e:
+            logger.error(f"Database timeout in {func.__name__}: {e}")
+            raise
+        except IntegrityError as e:
+            logger.warning(f"Integrity error in {func.__name__}: {e}")
+            raise
+        except Exception as e:
+            logger.error(f"Unexpected DB error in {func.__name__}: {e}")
+            raise
+    return wrapper
+
+
+# ============================================
+# Novel CRUD
+# ============================================
+@handle_db_errors
+async def create_novel(
+    db: AsyncSession,
+    title: str,
+    url: str,
+    login_email: Optional[str] = None,
+    login_password: Optional[str] = None,
+    next_button_selector: Optional[str] = None,
+    content_selector: Optional[str] = None,
+) -> Novel:
+    """Create a new novel entry in the database."""
+    novel = Novel(
+        title=title,
+        url=url,
+        current_url=url,
+        login_email=login_email,
+        login_password=login_password,
+        status=NovelStatus.QUEUED,
+    )
+    
+    if next_button_selector:
+        novel.next_button_selector = next_button_selector
+    if content_selector:
+        novel.content_selector = content_selector
+
+    db.add(novel)
+    await db.flush()  # Get the ID without committing
+    await db.refresh(novel)
+    
+    logger.info(f"Created novel: {novel.title} (ID: {novel.id})")
+    return novel
+
+
+@handle_db_errors
+async def get_novel_by_id(db: AsyncSession, novel_id: int) -> Optional[Novel]:
+    """Get a single novel by its ID."""
+    result = await db.execute(
+        select(Novel).where(Novel.id == novel_id)
+    )
+    return result.scalar_one_or_none()
+
+
+@handle_db_errors
+async def get_all_novels(db: AsyncSession) -> List[Novel]:
+    """Get all novels ordered by creation date."""
+    result = await db.execute(
+        select(Novel).order_by(Novel.created_at.desc())
+    )
+    return list(result.scalars().all())
+
+
+@handle_db_errors
+async def get_novels_needing_intervention(db: AsyncSession) -> List[Novel]:
+    """Get novels that are paused and need manual captcha solving."""
+    result = await db.execute(
+        select(Novel).where(
+            Novel.needs_intervention == True,
+            Novel.status == NovelStatus.PAUSED_CAPTCHA,
+        )
+    )
+    return list(result.scalars().all())
+
+
+@handle_db_errors
+async def update_novel_status(
+    db: AsyncSession,
+    novel_id: int,
+    status: NovelStatus,
+    error_message: Optional[str] = None,
+    current_url: Optional[str] = None,
+    screenshot_path: Optional[str] = None,
+    needs_intervention: Optional[bool] = None,
+) -> Optional[Novel]:
+    """Update the status and related fields of a novel."""
+    update_data = {"status": status}
+    
+    if error_message is not None:
+        update_data["last_error"] = error_message
+    if current_url is not None:
+        update_data["current_url"] = current_url
+    if screenshot_path is not None:
+        update_data["screenshot_path"] = screenshot_path
+    if needs_intervention is not None:
+        update_data["needs_intervention"] = needs_intervention
+
+    await db.execute(
+        update(Novel)
+        .where(Novel.id == novel_id)
+        .values(**update_data)
+    )
+    await db.flush()
+
+    # Return updated novel
+    return await get_novel_by_id(db, novel_id)
+
+
+@handle_db_errors
+async def increment_chapter_count(db: AsyncSession, novel_id: int) -> None:
+    """Increment the chapters_scraped counter for a novel."""
+    await db.execute(
+        update(Novel)
+        .where(Novel.id == novel_id)
+        .values(chapters_scraped=Novel.chapters_scraped + 1)
+    )
+    await db.flush()
+
+
+@handle_db_errors
+async def delete_novel(db: AsyncSession, novel_id: int) -> bool:
+    """Delete a novel and all its chapters (cascade)."""
+    result = await db.execute(
+        delete(Novel).where(Novel.id == novel_id)
+    )
+    await db.flush()
+    deleted = result.rowcount > 0
+    if deleted:
+        logger.info(f"Deleted novel ID: {novel_id}")
+    return deleted
+
+
+# ============================================
+# Chapter CRUD
+# ============================================
+@handle_db_errors
+async def save_chapter(
+    db: AsyncSession,
+    novel_id: int,
+    chapter_number: int,
+    content: str,
+    title: Optional[str] = None,
+    url: Optional[str] = None,
+) -> Chapter:
+    """
+    Save a chapter to the database.
+    If a chapter with the same number already exists for this novel,
+    it will raise an IntegrityError (caught by caller).
+    """
+    word_count = len(content.split()) if content else 0
+
+    chapter = Chapter(
+        novel_id=novel_id,
+        chapter_number=chapter_number,
+        title=title or f"Chapter {chapter_number}",
+        content=content,
+        url=url,
+        word_count=word_count,
+    )
+
+    db.add(chapter)
+    await db.flush()
+    await db.refresh(chapter)
+
+    logger.info(
+        f"Saved Chapter {chapter_number} for Novel {novel_id} "
+        f"({word_count} words)"
+    )
+    return chapter
+
+
+@handle_db_errors
+async def get_chapters_for_novel(
+    db: AsyncSession,
+    novel_id: int,
+) -> List[Chapter]:
+    """Get all chapters for a novel, ordered by chapter number."""
+    result = await db.execute(
+        select(Chapter)
+        .where(Chapter.novel_id == novel_id)
+        .order_by(Chapter.chapter_number.asc())
+    )
+    return list(result.scalars().all())
+
+
+@handle_db_errors
+async def get_chapter_count(db: AsyncSession, novel_id: int) -> int:
+    """Get the total number of chapters saved for a novel."""
+    result = await db.execute(
+        select(func.count(Chapter.id))
+        .where(Chapter.novel_id == novel_id)
+    )
+    return result.scalar() or 0
+
+
+@handle_db_errors
+async def get_last_chapter_number(db: AsyncSession, novel_id: int) -> int:
+    """Get the highest chapter number saved for a novel."""
+    result = await db.execute(
+        select(func.max(Chapter.chapter_number))
+        .where(Chapter.novel_id == novel_id)
+    )
+    return result.scalar() or 0
+
+
+@handle_db_errors
+async def chapter_exists(
+    db: AsyncSession,
+    novel_id: int,
+    chapter_number: int,
+) -> bool:
+    """Check if a specific chapter already exists."""
+    result = await db.execute(
+        select(func.count(Chapter.id))
+        .where(
+            Chapter.novel_id == novel_id,
+            Chapter.chapter_number == chapter_number,
+        )
+    )
+    return (result.scalar() or 0) > 0
+
+
+@handle_db_errors
+async def get_total_word_count(db: AsyncSession, novel_id: int) -> int:
+    """Get total word count across all chapters of a novel."""
+    result = await db.execute(
+        select(func.sum(Chapter.word_count))
+        .where(Chapter.novel_id == novel_id)
+    )
+    return result.scalar() or 0

app/database/models.py

@@ -0,0 +1,156 @@
+"""
+============================================
+Database Models (Tables)
+- Novel: Stores novel metadata & scraping status
+- Chapter: Stores individual chapter content
+============================================
+"""
+
+import enum
+from datetime import datetime, timezone
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    Text,
+    DateTime,
+    ForeignKey,
+    Enum as SQLEnum,
+    Boolean,
+    Float,
+    Index,
+    UniqueConstraint,
+)
+from sqlalchemy.orm import relationship
+from app.database.connection import Base
+
+
+# ============================================
+# Enums for Status Tracking
+# ============================================
+class NovelStatus(str, enum.Enum):
+    """Status of a novel's scraping process."""
+    QUEUED = "queued"                    # Waiting in queue
+    LOGGING_IN = "logging_in"           # Attempting login
+    SCRAPING = "scraping"               # Actively scraping chapters
+    PAUSED_CAPTCHA = "paused_captcha"   # Waiting for manual intervention
+    PAUSED_ERROR = "paused_error"       # Paused due to error
+    COMPLETED = "completed"             # All chapters scraped
+    FAILED = "failed"                   # Permanently failed
+
+
+# ============================================
+# Novel Model
+# ============================================
+class Novel(Base):
+    """
+    Stores metadata about each novel being scraped.
+    """
+    __tablename__ = "novels"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    
+    # --- Basic Info ---
+    title = Column(String(500), nullable=False, default="Unknown Novel")
+    url = Column(String(2000), nullable=False)  # Starting URL
+    current_url = Column(String(2000), nullable=True)  # Current page URL
+    
+    # --- Credentials (for login) ---
+    login_email = Column(String(500), nullable=True)
+    login_password = Column(String(500), nullable=True)  # In production, encrypt this!
+    
+    # --- Scraping Config ---
+    next_button_selector = Column(
+        String(500), 
+        nullable=False,
+        default="a.next_page, a[rel='next'], .next-chap, button.next-chapter"
+    )
+    content_selector = Column(
+        String(500),
+        nullable=False,
+        default=".chapter-content, .reading-content, #chapter-content, .text-left"
+    )
+    
+    # --- Status Tracking ---
+    status = Column(
+        SQLEnum(NovelStatus),
+        nullable=False,
+        default=NovelStatus.QUEUED
+    )
+    chapters_scraped = Column(Integer, default=0)
+    last_error = Column(Text, nullable=True)
+    
+    # --- Captcha/Intervention ---
+    screenshot_path = Column(String(1000), nullable=True)
+    needs_intervention = Column(Boolean, default=False)
+    
+    # --- Timestamps ---
+    created_at = Column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now(timezone.utc)
+    )
+    updated_at = Column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now(timezone.utc),
+        onupdate=lambda: datetime.now(timezone.utc)
+    )
+
+    # --- Relationships ---
+    chapters = relationship(
+        "Chapter",
+        back_populates="novel",
+        cascade="all, delete-orphan",
+        order_by="Chapter.chapter_number",
+        lazy="selectin",
+    )
+
+    def __repr__(self):
+        return f"<Novel(id={self.id}, title='{self.title}', status={self.status})>"
+
+
+# ============================================
+# Chapter Model
+# ============================================
+class Chapter(Base):
+    """
+    Stores individual chapter content.
+    Each chapter belongs to a Novel.
+    """
+    __tablename__ = "chapters"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    
+    # --- Foreign Key ---
+    novel_id = Column(
+        Integer,
+        ForeignKey("novels.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    
+    # --- Chapter Data ---
+    chapter_number = Column(Integer, nullable=False)
+    title = Column(String(1000), nullable=True, default="")
+    content = Column(Text, nullable=False)  # The actual chapter text
+    url = Column(String(2000), nullable=True)  # URL where this was scraped from
+    word_count = Column(Integer, default=0)
+    
+    # --- Timestamps ---
+    scraped_at = Column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now(timezone.utc)
+    )
+
+    # --- Relationships ---
+    novel = relationship("Novel", back_populates="chapters")
+
+    # --- Constraints ---
+    __table_args__ = (
+        # Ensure no duplicate chapters for same novel
+        UniqueConstraint("novel_id", "chapter_number", name="uq_novel_chapter"),
+        # Index for faster queries
+        Index("ix_chapter_novel_number", "novel_id", "chapter_number"),
+    )
+
+    def __repr__(self):
+        return f"<Chapter(id={self.id}, novel_id={self.novel_id}, ch={self.chapter_number})>"