Upload 51 files

.env

@@ -0,0 +1,30 @@
+# Application
+FLASK_ENV=development
+SECRET_KEY=dev-secret-key-change-in-production-use-32-chars
+DEBUG=True
+
+# Database (leave unset to use default SQLite in project/database/)
+# DATABASE_URL=sqlite:///database/scraper.db
+
+# Security
+WTF_CSRF_ENABLED=True
+SESSION_COOKIE_SECURE=False
+SESSION_COOKIE_HTTPONLY=True
+SESSION_COOKIE_SAMESITE=Lax
+
+# Rate Limiting
+RATELIMIT_DEFAULT=100 per hour
+RATELIMIT_STORAGE_URL=memory://
+
+# Scraping
+MAX_CONCURRENT_JOBS=5
+REQUEST_TIMEOUT=30
+MAX_RETRIES=3
+DEFAULT_DELAY=1.0
+
+# Exports
+EXPORT_DIR=exports
+
+# Logging
+LOG_LEVEL=DEBUG
+LOG_DIR=logs

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+database/scraper.db filter=lfs diff=lfs merge=lfs -text

app/__init__.py

@@ -0,0 +1,80 @@
+"""
+Application factory.
+Create and configure the Flask app with all extensions, blueprints, and middleware.
+"""
+from __future__ import annotations
+
+import logging
+
+from flask import Flask
+from flask_wtf.csrf import CSRFProtect
+from flask_limiter import Limiter
+from flask_limiter.util import get_remote_address
+
+from app.config import get_config
+from app.models import db
+from app.middleware import register_middleware
+from app.utils.logging_config import configure_logging
+
+csrf = CSRFProtect()
+limiter = Limiter(key_func=get_remote_address)
+
+logger = logging.getLogger(__name__)
+
+
+def create_app(config_class=None) -> Flask:
+    """Application factory — creates a fully configured Flask app."""
+    app = Flask(
+        __name__,
+        template_folder="templates",
+        static_folder="static",
+    )
+
+    # --- Load config ---
+    cfg = config_class or get_config()
+    app.config.from_object(cfg)
+    cfg.init_app(app)
+
+    # --- Logging ---
+    configure_logging(
+        log_level=app.config.get("LOG_LEVEL", "INFO"),
+        log_dir=str(app.config.get("LOG_DIR", "logs")),
+    )
+
+    # --- Extensions ---
+    db.init_app(app)
+    csrf.init_app(app)
+    limiter.init_app(app)
+
+    # --- Middleware ---
+    register_middleware(app)
+
+    # --- Blueprints ---
+    from app.routes import main_bp, jobs_bp
+    app.register_blueprint(main_bp)
+    app.register_blueprint(jobs_bp)
+
+    # Apply rate limiting to API routes
+    limiter.limit("60 per minute")(jobs_bp)
+
+    # --- Database init ---
+    with app.app_context():
+        db.create_all()
+        _seed_defaults()
+
+    logger.info("WebScraper Platform started [%s]", app.config.get("FLASK_ENV", "development"))
+    return app
+
+
+def _seed_defaults() -> None:
+    """Seed default app settings if not present."""
+    from app.models import AppSetting
+    defaults = [
+        ("max_concurrent_jobs", "5", "Maximum parallel scrape jobs"),
+        ("default_delay", "1.0", "Default seconds between requests"),
+        ("default_timeout", "30", "Default HTTP timeout in seconds"),
+    ]
+    for key, value, desc in defaults:
+        if not AppSetting.query.filter_by(key=key).first():
+            db.session.add(AppSetting(key=key, value=value, description=desc))
+    db.session.commit()

app/config/__init__.py

@@ -0,0 +1,3 @@
+from .settings import get_config, BaseConfig, DevelopmentConfig, ProductionConfig, TestingConfig
+
+__all__ = ["get_config", "BaseConfig", "DevelopmentConfig", "ProductionConfig", "TestingConfig"]

app/config/settings.py

@@ -0,0 +1,107 @@
+"""
+Application configuration classes for different environments.
+Uses environment variables with sensible defaults.
+"""
+import os
+from pathlib import Path
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# app/config/settings.py -> parent (config) -> parent (app) -> parent (project root)
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+
+
+class BaseConfig:
+    """Base configuration shared across all environments."""
+
+    # Flask
+    SECRET_KEY: str = os.getenv("SECRET_KEY", "change-me-in-production-32-chars!!")
+    DEBUG: bool = False
+    TESTING: bool = False
+
+    # SQLAlchemy
+    SQLALCHEMY_DATABASE_URI: str = os.getenv(
+        "DATABASE_URL", f"sqlite:///{BASE_DIR / 'database' / 'scraper.db'}"
+    )
+    SQLALCHEMY_TRACK_MODIFICATIONS: bool = False
+    SQLALCHEMY_ENGINE_OPTIONS: dict = {
+        "pool_pre_ping": True,
+        "pool_recycle": 300,
+    }
+
+    # CSRF
+    WTF_CSRF_ENABLED: bool = os.getenv("WTF_CSRF_ENABLED", "True") == "True"
+    WTF_CSRF_TIME_LIMIT: int = 3600
+
+    # Session
+    SESSION_COOKIE_SECURE: bool = os.getenv("SESSION_COOKIE_SECURE", "False") == "True"
+    SESSION_COOKIE_HTTPONLY: bool = True
+    SESSION_COOKIE_SAMESITE: str = "Lax"
+    PERMANENT_SESSION_LIFETIME: int = 86400
+
+    # Rate limiting
+    RATELIMIT_DEFAULT: str = os.getenv("RATELIMIT_DEFAULT", "200 per hour")
+    RATELIMIT_STORAGE_URL: str = os.getenv("RATELIMIT_STORAGE_URL", "memory://")
+    RATELIMIT_HEADERS_ENABLED: bool = True
+
+    # Scraping engine
+    MAX_CONCURRENT_JOBS: int = int(os.getenv("MAX_CONCURRENT_JOBS", "5"))
+    REQUEST_TIMEOUT: int = int(os.getenv("REQUEST_TIMEOUT", "30"))
+    MAX_RETRIES: int = int(os.getenv("MAX_RETRIES", "3"))
+    DEFAULT_DELAY: float = float(os.getenv("DEFAULT_DELAY", "1.0"))
+    MAX_PAGES: int = int(os.getenv("MAX_PAGES", "50"))
+
+    # Directories
+    EXPORT_DIR: Path = BASE_DIR / os.getenv("EXPORT_DIR", "exports")
+    LOG_DIR: Path = BASE_DIR / os.getenv("LOG_DIR", "logs")
+
+    # Logging
+    LOG_LEVEL: str = os.getenv("LOG_LEVEL", "INFO")
+    LOG_FORMAT: str = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
+
+    @classmethod
+    def init_app(cls, app) -> None:
+        """Hook for environment-specific initialization."""
+        # Ensure required directories exist
+        cls.EXPORT_DIR.mkdir(parents=True, exist_ok=True)
+        cls.LOG_DIR.mkdir(parents=True, exist_ok=True)
+        (BASE_DIR / "database").mkdir(parents=True, exist_ok=True)
+
+
+class DevelopmentConfig(BaseConfig):
+    DEBUG: bool = True
+    LOG_LEVEL: str = "DEBUG"
+    SESSION_COOKIE_SECURE: bool = False
+
+
+class ProductionConfig(BaseConfig):
+    DEBUG: bool = False
+    SESSION_COOKIE_SECURE: bool = True
+    LOG_LEVEL: str = "WARNING"
+    SQLALCHEMY_ENGINE_OPTIONS: dict = {
+        "pool_pre_ping": True,
+        "pool_recycle": 300,
+        "pool_size": 10,
+        "max_overflow": 20,
+    }
+
+
+class TestingConfig(BaseConfig):
+    TESTING: bool = True
+    DEBUG: bool = True
+    WTF_CSRF_ENABLED: bool = False
+    SQLALCHEMY_DATABASE_URI: str = "sqlite:///:memory:"
+
+
+config_map = {
+    "development": DevelopmentConfig,
+    "production": ProductionConfig,
+    "testing": TestingConfig,
+    "default": DevelopmentConfig,
+}
+
+
+def get_config() -> type:
+    env = os.getenv("FLASK_ENV", "development").lower()
+    return config_map.get(env, DevelopmentConfig)

app/middleware/__init__.py

@@ -0,0 +1,3 @@
+from .security import register_middleware
+
+__all__ = ["register_middleware"]

app/middleware/security.py

@@ -0,0 +1,67 @@
+"""
+Flask middleware:
+- Request timing
+- Security headers (XSS, clickjacking, content-type sniffing)
+- Global error handlers (404, 429, 500)
+"""
+from __future__ import annotations
+
+import logging
+import time
+
+from flask import Flask, jsonify, request, render_template
+
+logger = logging.getLogger(__name__)
+
+
+def register_middleware(app: Flask) -> None:
+    """Register all middleware and error handlers on the Flask app."""
+
+    @app.before_request
+    def start_timer() -> None:
+        request._start_time = time.monotonic()
+
+    @app.after_request
+    def add_security_headers(response):
+        duration = time.monotonic() - getattr(request, "_start_time", time.monotonic())
+        response.headers["X-Response-Time"] = f"{duration * 1000:.2f}ms"
+        response.headers["X-Content-Type-Options"] = "nosniff"
+        response.headers["X-Frame-Options"] = "SAMEORIGIN"
+        response.headers["X-XSS-Protection"] = "1; mode=block"
+        response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
+        response.headers["Permissions-Policy"] = "geolocation=(), microphone=()"
+        if app.config.get("SESSION_COOKIE_SECURE"):
+            response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
+        logger.debug(
+            "%s %s -> %d (%.2fms)",
+            request.method,
+            request.path,
+            response.status_code,
+            duration * 1000,
+        )
+        return response
+
+    @app.errorhandler(400)
+    def bad_request(exc):
+        if request.is_json:
+            return jsonify({"error": "Bad request", "detail": str(exc)}), 400
+        return render_template("pages/error.html", code=400, message="Bad Request"), 400
+
+    @app.errorhandler(404)
+    def not_found(exc):
+        if request.is_json:
+            return jsonify({"error": "Not found"}), 404
+        return render_template("pages/error.html", code=404, message="Page Not Found"), 404
+
+    @app.errorhandler(429)
+    def rate_limited(exc):
+        if request.is_json:
+            return jsonify({"error": "Rate limit exceeded. Please slow down."}), 429
+        return render_template("pages/error.html", code=429, message="Rate limit exceeded"), 429
+
+    @app.errorhandler(500)
+    def internal_error(exc):
+        logger.exception("Internal server error: %s", exc)
+        if request.is_json:
+            return jsonify({"error": "Internal server error"}), 500
+        return render_template("pages/error.html", code=500, message="Internal Server Error"), 500

app/models/__init__.py

@@ -0,0 +1,22 @@
+from .models import (
+    db,
+    BaseModel,
+    User,
+    ScrapeJob,
+    ScrapeResult,
+    JobLog,
+    ExportRecord,
+    Schedule,
+    AppSetting,
+)
+
+__all__ = [
+    "db",
+    "User",
+    "ScrapeJob",
+    "ScrapeResult",
+    "JobLog",
+    "ExportRecord",
+    "Schedule",
+    "AppSetting",
+]

app/models/models.py

@@ -0,0 +1,371 @@
+"""
+SQLAlchemy ORM models with relationships, indexes, and cascading deletes.
+All timestamps are UTC. All text fields are sanitized at the service layer.
+"""
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import Optional
+
+from flask_sqlalchemy import SQLAlchemy
+from sqlalchemy import (
+    Boolean,
+    Column,
+    DateTime,
+    Float,
+    ForeignKey,
+    Index,
+    Integer,
+    String,
+    Text,
+    Enum,
+    UniqueConstraint,
+)
+from sqlalchemy.orm import relationship
+
+db = SQLAlchemy()
+
+
+class BaseModel(db.Model):
+    """Base model with __allow_unmapped__ for legacy annotations."""
+    __abstract__ = True
+    __allow_unmapped__ = True
+
+
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc).replace(tzinfo=None)
+
+
+# ---------------------------------------------------------------------------
+# Enumerations (stored as VARCHAR for SQLite compatibility)
+# ---------------------------------------------------------------------------
+
+JOB_STATUS = ("pending", "running", "completed", "failed", "cancelled")
+SCRAPE_TYPE = ("static", "dynamic")
+EXTRACTION_TYPE = ("text", "images", "links", "attributes", "table", "json_ld", "full_html")
+EXPORT_FORMAT = ("json", "csv", "excel")
+
+
+# ---------------------------------------------------------------------------
+# Models
+# ---------------------------------------------------------------------------
+
+
+class User(BaseModel):
+    """Simple user model — single-user by default, extendable to multi-user."""
+
+    __tablename__ = "users"
+
+    id: int = Column(Integer, primary_key=True)
+    username: str = Column(String(80), unique=True, nullable=False, index=True)
+    email: str = Column(String(120), unique=True, nullable=False, index=True)
+    password_hash: str = Column(String(256), nullable=False)
+    is_active: bool = Column(Boolean, default=True, nullable=False)
+    is_admin: bool = Column(Boolean, default=False, nullable=False)
+    api_key: Optional[str] = Column(String(64), unique=True, index=True)
+    created_at: datetime = Column(DateTime, default=utcnow, nullable=False)
+    updated_at: datetime = Column(DateTime, default=utcnow, onupdate=utcnow, nullable=False)
+
+    # Relationships
+    jobs: list[ScrapeJob] = relationship(
+        "ScrapeJob", back_populates="user", cascade="all, delete-orphan", lazy="select"
+    )
+    schedules: list[Schedule] = relationship(
+        "Schedule", back_populates="user", cascade="all, delete-orphan", lazy="select"
+    )
+
+    def __repr__(self) -> str:
+        return f"<User {self.username}>"
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "username": self.username,
+            "email": self.email,
+            "is_active": self.is_active,
+            "is_admin": self.is_admin,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+        }
+
+
+class ScrapeJob(BaseModel):
+    """Represents a single scraping job with all configuration."""
+
+    __tablename__ = "scrape_jobs"
+    __table_args__ = (
+        Index("ix_scrape_jobs_status", "status"),
+        Index("ix_scrape_jobs_created_at", "created_at"),
+        Index("ix_scrape_jobs_user_id", "user_id"),
+    )
+
+    id: int = Column(Integer, primary_key=True)
+    user_id: Optional[int] = Column(Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
+    name: str = Column(String(200), nullable=False)
+    url: str = Column(Text, nullable=False)
+
+    # Selectors
+    html_tag: Optional[str] = Column(String(100))
+    css_selector: Optional[str] = Column(Text)
+    xpath_selector: Optional[str] = Column(Text)
+    attribute_name: Optional[str] = Column(String(100))
+
+    # Configuration
+    extraction_type: str = Column(
+        Enum(*EXTRACTION_TYPE, name="extraction_type"), default="text", nullable=False
+    )
+    scrape_type: str = Column(
+        Enum(*SCRAPE_TYPE, name="scrape_type"), default="static", nullable=False
+    )
+    status: str = Column(
+        Enum(*JOB_STATUS, name="job_status"), default="pending", nullable=False
+    )
+
+    # Advanced options
+    follow_pagination: bool = Column(Boolean, default=False)
+    max_pages: int = Column(Integer, default=1)
+    infinite_scroll: bool = Column(Boolean, default=False)
+    scroll_count: int = Column(Integer, default=3)
+    download_images: bool = Column(Boolean, default=False)
+    custom_headers: Optional[str] = Column(Text)  # JSON string
+    user_agent: Optional[str] = Column(Text)
+    delay_seconds: float = Column(Float, default=1.0)
+    timeout_seconds: int = Column(Integer, default=30)
+    max_retries: int = Column(Integer, default=3)
+    check_robots_txt: bool = Column(Boolean, default=True)
+    deduplicate: bool = Column(Boolean, default=True)
+
+    # Stats
+    total_items: int = Column(Integer, default=0)
+    pages_scraped: int = Column(Integer, default=0)
+    error_count: int = Column(Integer, default=0)
+    duration_seconds: Optional[float] = Column(Float)
+
+    # Timestamps
+    created_at: datetime = Column(DateTime, default=utcnow, nullable=False)
+    updated_at: datetime = Column(DateTime, default=utcnow, onupdate=utcnow)
+    started_at: Optional[datetime] = Column(DateTime)
+    completed_at: Optional[datetime] = Column(DateTime)
+
+    # Relationships
+    user: Optional[User] = relationship("User", back_populates="jobs")
+    results: list[ScrapeResult] = relationship(
+        "ScrapeResult", back_populates="job", cascade="all, delete-orphan", lazy="select"
+    )
+    logs: list[JobLog] = relationship(
+        "JobLog", back_populates="job", cascade="all, delete-orphan", lazy="select"
+    )
+    exports: list[ExportRecord] = relationship(
+        "ExportRecord", back_populates="job", cascade="all, delete-orphan", lazy="select"
+    )
+
+    def __repr__(self) -> str:
+        return f"<ScrapeJob {self.id} [{self.status}] {self.url[:50]}>"
+
+    @property
+    def custom_headers_dict(self) -> dict:
+        if self.custom_headers:
+            try:
+                return json.loads(self.custom_headers)
+            except (json.JSONDecodeError, TypeError):
+                return {}
+        return {}
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "name": self.name,
+            "url": self.url,
+            "html_tag": self.html_tag,
+            "css_selector": self.css_selector,
+            "xpath_selector": self.xpath_selector,
+            "extraction_type": self.extraction_type,
+            "scrape_type": self.scrape_type,
+            "status": self.status,
+            "follow_pagination": self.follow_pagination,
+            "max_pages": self.max_pages,
+            "infinite_scroll": self.infinite_scroll,
+            "total_items": self.total_items,
+            "pages_scraped": self.pages_scraped,
+            "error_count": self.error_count,
+            "duration_seconds": self.duration_seconds,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+            "started_at": self.started_at.isoformat() if self.started_at else None,
+            "completed_at": self.completed_at.isoformat() if self.completed_at else None,
+        }
+
+
+class ScrapeResult(BaseModel):
+    """Individual scraped data item linked to a job."""
+
+    __tablename__ = "scrape_results"
+    __table_args__ = (
+        Index("ix_scrape_results_job_id", "job_id"),
+        Index("ix_scrape_results_page_num", "page_num"),
+        Index("ix_scrape_results_content_hash", "content_hash"),
+    )
+
+    id: int = Column(Integer, primary_key=True)
+    job_id: int = Column(Integer, ForeignKey("scrape_jobs.id", ondelete="CASCADE"), nullable=False)
+    page_url: str = Column(Text, nullable=False)
+    page_num: int = Column(Integer, default=1, nullable=False)
+    item_index: int = Column(Integer, default=0)
+    content: Optional[str] = Column(Text)
+    content_type: str = Column(String(50), default="text")
+    content_hash: Optional[str] = Column(String(64))
+    metadata_: Optional[str] = Column("metadata", Text)  # JSON
+    created_at: datetime = Column(DateTime, default=utcnow, nullable=False)
+
+    # Relationship
+    job: ScrapeJob = relationship("ScrapeJob", back_populates="results")
+
+    def __repr__(self) -> str:
+        return f"<ScrapeResult job={self.job_id} page={self.page_num} idx={self.item_index}>"
+
+    @property
+    def metadata_dict(self) -> dict:
+        if self.metadata_:
+            try:
+                return json.loads(self.metadata_)
+            except (json.JSONDecodeError, TypeError):
+                return {}
+        return {}
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "job_id": self.job_id,
+            "page_url": self.page_url,
+            "page_num": self.page_num,
+            "item_index": self.item_index,
+            "content": self.content,
+            "content_type": self.content_type,
+            "metadata": self.metadata_dict,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+        }
+
+
+class JobLog(BaseModel):
+    """Structured logs for each scraping job."""
+
+    __tablename__ = "job_logs"
+    __table_args__ = (
+        Index("ix_job_logs_job_id", "job_id"),
+        Index("ix_job_logs_level", "level"),
+        Index("ix_job_logs_created_at", "created_at"),
+    )
+
+    id: int = Column(Integer, primary_key=True)
+    job_id: int = Column(Integer, ForeignKey("scrape_jobs.id", ondelete="CASCADE"), nullable=False)
+    level: str = Column(String(10), default="INFO", nullable=False)
+    message: str = Column(Text, nullable=False)
+    details: Optional[str] = Column(Text)  # JSON for structured extras
+    created_at: datetime = Column(DateTime, default=utcnow, nullable=False)
+
+    job: ScrapeJob = relationship("ScrapeJob", back_populates="logs")
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "job_id": self.job_id,
+            "level": self.level,
+            "message": self.message,
+            "details": self.details,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+        }
+
+
+class ExportRecord(BaseModel):
+    """Tracks export files generated for each job."""
+
+    __tablename__ = "export_records"
+    __table_args__ = (Index("ix_export_records_job_id", "job_id"),)
+
+    id: int = Column(Integer, primary_key=True)
+    job_id: int = Column(Integer, ForeignKey("scrape_jobs.id", ondelete="CASCADE"), nullable=False)
+    format: str = Column(Enum(*EXPORT_FORMAT, name="export_format"), nullable=False)
+    filename: str = Column(String(255), nullable=False)
+    filepath: str = Column(Text, nullable=False)
+    file_size_bytes: Optional[int] = Column(Integer)
+    row_count: int = Column(Integer, default=0)
+    created_at: datetime = Column(DateTime, default=utcnow, nullable=False)
+
+    job: ScrapeJob = relationship("ScrapeJob", back_populates="exports")
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "job_id": self.job_id,
+            "format": self.format,
+            "filename": self.filename,
+            "file_size_bytes": self.file_size_bytes,
+            "row_count": self.row_count,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+        }
+
+
+class Schedule(BaseModel):
+    """APScheduler-backed recurring scrape schedules."""
+
+    __tablename__ = "schedules"
+    __table_args__ = (Index("ix_schedules_user_id", "user_id"),)
+
+    id: int = Column(Integer, primary_key=True)
+    user_id: Optional[int] = Column(Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
+    name: str = Column(String(200), nullable=False)
+    cron_expression: str = Column(String(100), nullable=False)  # e.g. "0 9 * * 1"
+    job_config: str = Column(Text, nullable=False)  # JSON of ScrapeJob config
+    is_active: bool = Column(Boolean, default=True, nullable=False)
+    last_run_at: Optional[datetime] = Column(DateTime)
+    next_run_at: Optional[datetime] = Column(DateTime)
+    run_count: int = Column(Integer, default=0)
+    created_at: datetime = Column(DateTime, default=utcnow, nullable=False)
+    updated_at: datetime = Column(DateTime, default=utcnow, onupdate=utcnow)
+
+    user: Optional[User] = relationship("User", back_populates="schedules")
+
+    @property
+    def job_config_dict(self) -> dict:
+        try:
+            return json.loads(self.job_config)
+        except (json.JSONDecodeError, TypeError):
+            return {}
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "name": self.name,
+            "cron_expression": self.cron_expression,
+            "is_active": self.is_active,
+            "last_run_at": self.last_run_at.isoformat() if self.last_run_at else None,
+            "next_run_at": self.next_run_at.isoformat() if self.next_run_at else None,
+            "run_count": self.run_count,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+        }
+
+
+class AppSetting(BaseModel):
+    """Key-value application settings stored in the DB."""
+
+    __tablename__ = "app_settings"
+
+    id: int = Column(Integer, primary_key=True)
+    key: str = Column(String(100), unique=True, nullable=False, index=True)
+    value: str = Column(Text, nullable=False)
+    description: Optional[str] = Column(Text)
+    updated_at: datetime = Column(DateTime, default=utcnow, onupdate=utcnow)
+
+    @classmethod
+    def get(cls, key: str, default: str = "") -> str:
+        row = cls.query.filter_by(key=key).first()
+        return row.value if row else default
+
+    @classmethod
+    def set(cls, key: str, value: str, description: str = "") -> None:
+        row = cls.query.filter_by(key=key).first()
+        if row:
+            row.value = value
+        else:
+            row = cls(key=key, value=value, description=description)
+            db.session.add(row)
+        db.session.commit()

app/routes/__init__.py

@@ -0,0 +1,4 @@
+from .main import main_bp
+from .jobs import jobs_bp
+
+__all__ = ["main_bp", "jobs_bp"]

app/routes/jobs.py

@@ -0,0 +1,265 @@
+"""
+Scrape job routes (UI + REST API).
+Blueprint: 'jobs' — prefix: /jobs
+"""
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from flask import (
+    Blueprint,
+    abort,
+    current_app,
+    flash,
+    jsonify,
+    redirect,
+    render_template,
+    request,
+    send_file,
+    url_for,
+)
+
+from app.services import (
+    create_job,
+    cancel_job,
+    delete_job,
+    execute_job_async,
+    get_job,
+    get_job_logs,
+    get_job_results,
+    list_jobs,
+    export_json,
+    export_csv,
+    export_excel,
+    get_job_exports,
+)
+from app.utils.validators import validate_job_data, sanitize_string
+
+logger = logging.getLogger(__name__)
+
+jobs_bp = Blueprint("jobs", __name__, url_prefix="/jobs")
+
+
+# ---------------------------------------------------------------------------
+# UI Routes
+# ---------------------------------------------------------------------------
+
+
+@jobs_bp.route("/")
+def list_view():
+    page = request.args.get("page", 1, type=int)
+    status = request.args.get("status")
+    search = request.args.get("search")
+    pagination = list_jobs(page=page, per_page=20, status=status, search=search)
+    return render_template(
+        "pages/jobs.html",
+        pagination=pagination,
+        status_filter=status,
+        search=search,
+    )
+
+
+@jobs_bp.route("/new", methods=["GET", "POST"])
+def new_job():
+    if request.method == "POST":
+        data = {k: v for k, v in request.form.items()}
+        # Sanitize string fields
+        for field in ("name", "html_tag", "css_selector", "xpath_selector", "attribute_name", "user_agent"):
+            if field in data:
+                data[field] = sanitize_string(data[field])
+
+        # Checkboxes
+        data["follow_pagination"] = "follow_pagination" in request.form
+        data["infinite_scroll"] = "infinite_scroll" in request.form
+        data["download_images"] = "download_images" in request.form
+        data["check_robots_txt"] = "check_robots_txt" in request.form
+        data["deduplicate"] = "deduplicate" in request.form
+
+        valid, errors = validate_job_data(data)
+        if not valid:
+            return render_template("pages/new_job.html", errors=errors, form_data=data)
+
+        job = create_job(data)
+        execute_job_async(job.id, current_app._get_current_object())
+        flash(f"Job #{job.id} started successfully!", "success")
+        return redirect(url_for("jobs.detail", job_id=job.id))
+
+    return render_template("pages/new_job.html", errors={}, form_data={})
+
+
+@jobs_bp.route("/<int:job_id>")
+def detail(job_id: int):
+    job = get_job(job_id)
+    if not job:
+        abort(404)
+    results_page = get_job_results(job_id, page=request.args.get("rpage", 1, type=int), per_page=50)
+    logs_page = get_job_logs(job_id, page=1, per_page=200)
+    exports = get_job_exports(job_id)
+    return render_template(
+        "pages/job_detail.html",
+        job=job,
+        results=results_page,
+        logs=logs_page,
+        exports=exports,
+    )
+
+
+@jobs_bp.route("/<int:job_id>/delete", methods=["POST"])
+def delete(job_id: int):
+    if not delete_job(job_id):
+        abort(404)
+    flash(f"Job #{job_id} deleted.", "info")
+    return redirect(url_for("jobs.list_view"))
+
+
+@jobs_bp.route("/<int:job_id>/cancel", methods=["POST"])
+def cancel(job_id: int):
+    if not cancel_job(job_id):
+        flash("Cannot cancel this job.", "warning")
+    else:
+        flash(f"Job #{job_id} cancelled.", "info")
+    return redirect(url_for("jobs.detail", job_id=job_id))
+
+
+@jobs_bp.route("/<int:job_id>/rerun", methods=["POST"])
+def rerun(job_id: int):
+    job = get_job(job_id)
+    if not job:
+        abort(404)
+    job.status = "pending"
+    from app.models import db
+    db.session.commit()
+    execute_job_async(job_id, current_app._get_current_object())
+    flash(f"Job #{job_id} queued for re-run.", "success")
+    return redirect(url_for("jobs.detail", job_id=job_id))
+
+
+# ---------------------------------------------------------------------------
+# Export Routes
+# ---------------------------------------------------------------------------
+
+
+@jobs_bp.route("/<int:job_id>/export/<string:fmt>")
+def export(job_id: int, fmt: str):
+    job = get_job(job_id)
+    if not job:
+        abort(404)
+
+    exporters = {"json": export_json, "csv": export_csv, "excel": export_excel}
+    if fmt not in exporters:
+        abort(400)
+
+    filepath = exporters[fmt](job_id)
+    if not filepath or not Path(filepath).exists():
+        flash("Export failed. No results to export.", "danger")
+        return redirect(url_for("jobs.detail", job_id=job_id))
+
+    mime_map = {
+        "json": "application/json",
+        "csv": "text/csv",
+        "excel": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    }
+    return send_file(filepath, mimetype=mime_map[fmt], as_attachment=True)
+
+
+# ---------------------------------------------------------------------------
+# REST API Routes
+# ---------------------------------------------------------------------------
+
+
+@jobs_bp.route("/api/jobs", methods=["GET"])
+def api_list():
+    page = request.args.get("page", 1, type=int)
+    per_page = min(request.args.get("per_page", 20, type=int), 100)
+    status = request.args.get("status")
+    search = request.args.get("search")
+    pagination = list_jobs(page=page, per_page=per_page, status=status, search=search)
+    return jsonify(
+        {
+            "jobs": [j.to_dict() for j in pagination.items],
+            "total": pagination.total,
+            "page": pagination.page,
+            "pages": pagination.pages,
+            "per_page": pagination.per_page,
+        }
+    )
+
+
+@jobs_bp.route("/api/jobs", methods=["POST"])
+def api_create():
+    data = request.get_json(silent=True)
+    if not data:
+        return jsonify({"error": "JSON body required"}), 400
+
+    valid, errors = validate_job_data(data)
+    if not valid:
+        return jsonify({"error": "Validation failed", "details": errors}), 422
+
+    job = create_job(data)
+    execute_job_async(job.id, current_app._get_current_object())
+    return jsonify({"job": job.to_dict(), "message": "Job created and queued"}), 201
+
+
+@jobs_bp.route("/api/jobs/<int:job_id>", methods=["GET"])
+def api_get(job_id: int):
+    job = get_job(job_id)
+    if not job:
+        return jsonify({"error": "Job not found"}), 404
+    return jsonify({"job": job.to_dict()})
+
+
+@jobs_bp.route("/api/jobs/<int:job_id>", methods=["DELETE"])
+def api_delete(job_id: int):
+    if not delete_job(job_id):
+        return jsonify({"error": "Job not found"}), 404
+    return jsonify({"message": f"Job {job_id} deleted"}), 200
+
+
+@jobs_bp.route("/api/jobs/<int:job_id>/results", methods=["GET"])
+def api_results(job_id: int):
+    job = get_job(job_id)
+    if not job:
+        return jsonify({"error": "Job not found"}), 404
+    page = request.args.get("page", 1, type=int)
+    per_page = min(request.args.get("per_page", 50, type=int), 500)
+    pagination = get_job_results(job_id, page=page, per_page=per_page)
+    return jsonify(
+        {
+            "results": [r.to_dict() for r in pagination.items],
+            "total": pagination.total,
+            "page": pagination.page,
+            "pages": pagination.pages,
+        }
+    )
+
+
+@jobs_bp.route("/api/jobs/<int:job_id>/logs", methods=["GET"])
+def api_logs(job_id: int):
+    job = get_job(job_id)
+    if not job:
+        return jsonify({"error": "Job not found"}), 404
+    page = request.args.get("page", 1, type=int)
+    pagination = get_job_logs(job_id, page=page, per_page=100)
+    return jsonify(
+        {
+            "logs": [l.to_dict() for l in pagination.items],
+            "total": pagination.total,
+        }
+    )
+
+
+@jobs_bp.route("/api/jobs/<int:job_id>/status", methods=["GET"])
+def api_status(job_id: int):
+    """Lightweight polling endpoint for live status updates."""
+    job = get_job(job_id)
+    if not job:
+        return jsonify({"error": "Job not found"}), 404
+    return jsonify(
+        {
+            "status": job.status,
+            "total_items": job.total_items,
+            "pages_scraped": job.pages_scraped,
+            "error_count": job.error_count,
+        }
+    )

app/routes/main.py

@@ -0,0 +1,34 @@
+"""
+Dashboard and main UI routes.
+Blueprint: 'main' — prefix: /
+"""
+from __future__ import annotations
+
+import logging
+
+from flask import Blueprint, render_template, jsonify
+
+from app.services import get_dashboard_stats
+
+logger = logging.getLogger(__name__)
+
+main_bp = Blueprint("main", __name__)
+
+
+@main_bp.route("/")
+def index():
+    stats = get_dashboard_stats()
+    return render_template("pages/dashboard.html", stats=stats)
+
+
+@main_bp.route("/health")
+def health():
+    """Health check endpoint for load balancers and monitoring."""
+    return jsonify({"status": "ok", "service": "WebScraper Platform"}), 200
+
+
+@main_bp.route("/metrics")
+def metrics():
+    """Simple performance metrics endpoint."""
+    stats = get_dashboard_stats()
+    return jsonify(stats), 200

app/scrapers/__init__.py

@@ -0,0 +1,3 @@
+from .engine import run_scrape, ScrapeRequest, ScrapeResponse, ScrapedItem
+
+__all__ = ["run_scrape", "ScrapeRequest", "ScrapeResponse", "ScrapedItem"]

app/scrapers/engine.py

@@ -0,0 +1,572 @@
+"""
+Core scraping engine.
+Handles static (Requests + BS4) and dynamic (Playwright) scraping.
+Implements UA rotation, retry logic, robots.txt checking, pagination, and infinite scroll.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import re
+import time
+import urllib.robotparser
+from dataclasses import dataclass, field
+from typing import Any, Generator, Optional
+from urllib.parse import urljoin, urlparse
+
+import requests
+from bs4 import BeautifulSoup
+from fake_useragent import UserAgent
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Data structures
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ScrapeRequest:
+    """Fully-typed request configuration for the scraping engine."""
+
+    url: str
+    html_tag: Optional[str] = None
+    css_selector: Optional[str] = None
+    xpath_selector: Optional[str] = None
+    attribute_name: Optional[str] = None
+    extraction_type: str = "text"  # text | images | links | attributes | table | json_ld | full_html
+    scrape_type: str = "static"  # static | dynamic
+    follow_pagination: bool = False
+    max_pages: int = 1
+    infinite_scroll: bool = False
+    scroll_count: int = 3
+    download_images: bool = False
+    custom_headers: dict = field(default_factory=dict)
+    user_agent: Optional[str] = None
+    delay_seconds: float = 1.0
+    timeout_seconds: int = 30
+    max_retries: int = 3
+    check_robots_txt: bool = True
+    deduplicate: bool = True
+
+
+@dataclass
+class ScrapedItem:
+    """A single scraped datum."""
+
+    content: str
+    content_type: str
+    page_url: str
+    page_num: int
+    item_index: int
+    content_hash: str
+    metadata: dict = field(default_factory=dict)
+
+
+@dataclass
+class ScrapeResponse:
+    """Final response from the scraping engine."""
+
+    items: list[ScrapedItem]
+    pages_scraped: int
+    error_count: int
+    errors: list[str]
+    duration_seconds: float
+
+
+# ---------------------------------------------------------------------------
+# User-Agent rotation
+# ---------------------------------------------------------------------------
+
+_ua_instance: Optional[UserAgent] = None
+
+
+def _get_user_agent(preferred: Optional[str] = None) -> str:
+    global _ua_instance
+    if preferred:
+        return preferred
+    try:
+        if _ua_instance is None:
+            _ua_instance = UserAgent(fallback="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
+        return _ua_instance.random
+    except Exception:
+        return "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+
+
+# ---------------------------------------------------------------------------
+# robots.txt checker
+# ---------------------------------------------------------------------------
+
+
+def _is_allowed_by_robots(url: str, user_agent: str = "*") -> bool:
+    """Check whether a URL is allowed by the target site's robots.txt."""
+    try:
+        parsed = urlparse(url)
+        robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
+        rp = urllib.robotparser.RobotFileParser()
+        rp.set_url(robots_url)
+        rp.read()
+        return rp.can_fetch(user_agent, url)
+    except Exception as exc:
+        logger.warning("robots.txt check failed for %s: %s", url, exc)
+        return True  # Allow on error — be conservative
+
+
+# ---------------------------------------------------------------------------
+# Content hashing for deduplication
+# ---------------------------------------------------------------------------
+
+
+def _content_hash(content: str) -> str:
+    return hashlib.sha256(content.encode("utf-8", errors="replace")).hexdigest()[:16]
+
+
+# ---------------------------------------------------------------------------
+# HTML Parsers
+# ---------------------------------------------------------------------------
+
+
+def _parse_html(html: str) -> BeautifulSoup:
+    try:
+        return BeautifulSoup(html, "lxml")
+    except Exception:
+        return BeautifulSoup(html, "html.parser")
+
+
+def _extract_with_css(soup: BeautifulSoup, selector: str) -> list[Any]:
+    return soup.select(selector)
+
+
+def _extract_with_xpath(html: str, xpath: str) -> list[str]:
+    """XPath extraction via lxml."""
+    try:
+        from lxml import etree
+
+        tree = etree.fromstring(html.encode(), parser=etree.HTMLParser())
+        results = tree.xpath(xpath)
+        texts = []
+        for r in results:
+            if isinstance(r, str):
+                texts.append(r.strip())
+            elif hasattr(r, "text_content"):
+                texts.append(r.text_content().strip())
+            elif hasattr(r, "text"):
+                texts.append((r.text or "").strip())
+        return [t for t in texts if t]
+    except Exception as exc:
+        logger.warning("XPath extraction failed: %s", exc)
+        return []
+
+
+def _extract_json_ld(soup: BeautifulSoup) -> list[dict]:
+    results = []
+    for tag in soup.find_all("script", type="application/ld+json"):
+        try:
+            data = json.loads(tag.string or "")
+            results.append(data)
+        except (json.JSONDecodeError, TypeError):
+            pass
+    return results
+
+
+def _extract_tables(soup: BeautifulSoup) -> list[list[list[str]]]:
+    tables = []
+    for table in soup.find_all("table"):
+        rows = []
+        for tr in table.find_all("tr"):
+            cells = [td.get_text(strip=True) for td in tr.find_all(["th", "td"])]
+            if cells:
+                rows.append(cells)
+        if rows:
+            tables.append(rows)
+    return tables
+
+
+# ---------------------------------------------------------------------------
+# Content extractors
+# ---------------------------------------------------------------------------
+
+
+def _extract_items(
+    soup: BeautifulSoup,
+    raw_html: str,
+    req: ScrapeRequest,
+    page_url: str,
+    page_num: int,
+) -> list[ScrapedItem]:
+    """Route extraction to the appropriate extractor."""
+    items: list[ScrapedItem] = []
+
+    if req.extraction_type == "json_ld":
+        for idx, data in enumerate(_extract_json_ld(soup)):
+            content = json.dumps(data, ensure_ascii=False)
+            items.append(
+                ScrapedItem(
+                    content=content,
+                    content_type="json_ld",
+                    page_url=page_url,
+                    page_num=page_num,
+                    item_index=idx,
+                    content_hash=_content_hash(content),
+                )
+            )
+        return items
+
+    if req.extraction_type == "table":
+        for t_idx, table in enumerate(_extract_tables(soup)):
+            content = json.dumps(table, ensure_ascii=False)
+            items.append(
+                ScrapedItem(
+                    content=content,
+                    content_type="table",
+                    page_url=page_url,
+                    page_num=page_num,
+                    item_index=t_idx,
+                    content_hash=_content_hash(content),
+                )
+            )
+        return items
+
+    if req.extraction_type == "full_html":
+        content = str(soup)
+        items.append(
+            ScrapedItem(
+                content=content,
+                content_type="html",
+                page_url=page_url,
+                page_num=page_num,
+                item_index=0,
+                content_hash=_content_hash(content),
+            )
+        )
+        return items
+
+    # --- Resolve elements via CSS or XPath or Tag ---
+    elements = []
+
+    if req.xpath_selector:
+        texts = _extract_with_xpath(raw_html, req.xpath_selector)
+        for idx, text in enumerate(texts):
+            items.append(
+                ScrapedItem(
+                    content=text,
+                    content_type="text",
+                    page_url=page_url,
+                    page_num=page_num,
+                    item_index=idx,
+                    content_hash=_content_hash(text),
+                )
+            )
+        return items
+
+    if req.css_selector:
+        elements = _extract_with_css(soup, req.css_selector)
+    elif req.html_tag:
+        elements = soup.find_all(req.html_tag)
+    else:
+        elements = soup.find_all(True)  # All elements
+
+    for idx, el in enumerate(elements):
+        if req.extraction_type == "text":
+            content = el.get_text(separator=" ", strip=True)
+        elif req.extraction_type == "links":
+            href = el.get("href", "") if el.name == "a" else ""
+            if not href:
+                link_el = el.find("a")
+                href = link_el.get("href", "") if link_el else ""
+            if href:
+                content = urljoin(page_url, href)
+            else:
+                continue
+        elif req.extraction_type == "images":
+            src = el.get("src", "") if el.name == "img" else ""
+            if not src:
+                img_el = el.find("img")
+                src = img_el.get("src", "") if img_el else ""
+            if src:
+                content = urljoin(page_url, src)
+            else:
+                continue
+        elif req.extraction_type == "attributes":
+            if req.attribute_name:
+                content = el.get(req.attribute_name, "")
+            else:
+                content = json.dumps(dict(el.attrs), ensure_ascii=False)
+        else:
+            content = el.get_text(separator=" ", strip=True)
+
+        content = content.strip()
+        if not content:
+            continue
+
+        items.append(
+            ScrapedItem(
+                content=content,
+                content_type=req.extraction_type,
+                page_url=page_url,
+                page_num=page_num,
+                item_index=idx,
+                content_hash=_content_hash(content),
+            )
+        )
+
+    return items
+
+
+# ---------------------------------------------------------------------------
+# Static scraper (Requests + BeautifulSoup)
+# ---------------------------------------------------------------------------
+
+
+class StaticScraper:
+    """HTTP scraper using Requests with retry, UA rotation, and timeout handling."""
+
+    def __init__(self, req: ScrapeRequest) -> None:
+        self.req = req
+        self.session = requests.Session()
+        self._configure_session()
+
+    def _configure_session(self) -> None:
+        ua = _get_user_agent(self.req.user_agent)
+        self.session.headers.update(
+            {
+                "User-Agent": ua,
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.5",
+                "Connection": "keep-alive",
+                "Upgrade-Insecure-Requests": "1",
+            }
+        )
+        if self.req.custom_headers:
+            self.session.headers.update(self.req.custom_headers)
+
+    @retry(
+        retry=retry_if_exception_type((requests.ConnectionError, requests.Timeout)),
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=1, max=10),
+        reraise=True,
+    )
+    def _fetch(self, url: str) -> requests.Response:
+        response = self.session.get(url, timeout=self.req.timeout_seconds, allow_redirects=True)
+        response.raise_for_status()
+        return response
+
+    def scrape(self) -> Generator[tuple[str, str, int], None, None]:
+        """Yields (html, page_url, page_num) for each page."""
+        url = self.req.url
+        page_num = 1
+
+        while url and page_num <= self.req.max_pages:
+            try:
+                # Rotate UA per request
+                self.session.headers["User-Agent"] = _get_user_agent(self.req.user_agent)
+                response = self._fetch(url)
+                html = response.text
+                yield html, response.url, page_num
+
+                if not self.req.follow_pagination or page_num >= self.req.max_pages:
+                    break
+
+                # Find next page link
+                soup = _parse_html(html)
+                next_url = _find_next_page(soup, response.url)
+                if not next_url or next_url == url:
+                    break
+
+                url = next_url
+                page_num += 1
+                time.sleep(self.req.delay_seconds)
+
+            except requests.HTTPError as exc:
+                logger.error("HTTP error fetching %s: %s", url, exc)
+                raise
+            except Exception as exc:
+                logger.error("Error fetching %s: %s", url, exc)
+                raise
+
+    def close(self) -> None:
+        self.session.close()
+
+
+def _find_next_page(soup: BeautifulSoup, current_url: str) -> Optional[str]:
+    """Heuristic: find next pagination link."""
+    patterns = [
+        "a[rel='next']",
+        "a.next",
+        "a.pagination-next",
+        "li.next a",
+        "a[aria-label='Next']",
+        ".next-page a",
+        "#next a",
+    ]
+    for sel in patterns:
+        el = soup.select_one(sel)
+        if el and el.get("href"):
+            return urljoin(current_url, el["href"])
+
+    # Fallback: look for links with text "next"
+    for a in soup.find_all("a", href=True):
+        text = a.get_text(strip=True).lower()
+        if text in ("next", "next »", "»", "›", "next page"):
+            return urljoin(current_url, a["href"])
+
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Dynamic scraper (Playwright)
+# ---------------------------------------------------------------------------
+
+
+class DynamicScraper:
+    """Playwright-based scraper for JS-rendered content with infinite scroll support."""
+
+    def scrape(self, req: ScrapeRequest) -> Generator[tuple[str, str, int], None, None]:
+        try:
+            from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout
+        except ImportError:
+            raise RuntimeError("Playwright not installed. Run: playwright install chromium")
+
+        ua = _get_user_agent(req.user_agent)
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"])
+            context = browser.new_context(
+                user_agent=ua,
+                viewport={"width": 1920, "height": 1080},
+                extra_http_headers=req.custom_headers or {},
+            )
+            page = context.new_page()
+            page.set_default_timeout(req.timeout_seconds * 1000)
+
+            url = req.url
+            page_num = 1
+
+            while url and page_num <= req.max_pages:
+                try:
+                    page.goto(url, wait_until="networkidle", timeout=req.timeout_seconds * 1000)
+
+                    if req.infinite_scroll:
+                        _perform_infinite_scroll(page, req.scroll_count)
+
+                    html = page.content()
+                    final_url = page.url
+                    yield html, final_url, page_num
+
+                    if not req.follow_pagination or page_num >= req.max_pages:
+                        break
+
+                    # Try clicking next page button
+                    next_url = _playwright_next_page(page, final_url)
+                    if not next_url or next_url == url:
+                        break
+
+                    url = next_url
+                    page_num += 1
+                    time.sleep(req.delay_seconds)
+
+                except PWTimeout as exc:
+                    logger.error("Playwright timeout on %s: %s", url, exc)
+                    raise
+                except Exception as exc:
+                    logger.error("Playwright error on %s: %s", url, exc)
+                    raise
+
+            context.close()
+            browser.close()
+
+
+def _perform_infinite_scroll(page, scroll_count: int) -> None:
+    """Scroll to bottom repeatedly to trigger lazy loading."""
+    for _ in range(scroll_count):
+        prev_height = page.evaluate("document.body.scrollHeight")
+        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        page.wait_for_timeout(1500)
+        new_height = page.evaluate("document.body.scrollHeight")
+        if new_height == prev_height:
+            break
+
+
+def _playwright_next_page(page, current_url: str) -> Optional[str]:
+    """Try to find and return next page URL from Playwright page."""
+    selectors = ["a[rel='next']", "a.next", "li.next a", "[aria-label='Next']"]
+    for sel in selectors:
+        try:
+            el = page.query_selector(sel)
+            if el:
+                href = el.get_attribute("href")
+                if href:
+                    return urljoin(current_url, href)
+        except Exception:
+            pass
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Main engine entrypoint
+# ---------------------------------------------------------------------------
+
+
+def run_scrape(req: ScrapeRequest) -> ScrapeResponse:
+    """
+    Execute a scrape job and return structured results.
+    Handles robots.txt, deduplication, pagination, and error accounting.
+    """
+    start_time = time.monotonic()
+    items: list[ScrapedItem] = []
+    errors: list[str] = []
+    pages_scraped = 0
+    seen_hashes: set[str] = set()
+
+    # robots.txt
+    if req.check_robots_txt:
+        if not _is_allowed_by_robots(req.url):
+            return ScrapeResponse(
+                items=[],
+                pages_scraped=0,
+                error_count=1,
+                errors=[f"robots.txt disallows scraping: {req.url}"],
+                duration_seconds=time.monotonic() - start_time,
+            )
+
+    try:
+        if req.scrape_type == "dynamic":
+            scraper = DynamicScraper()
+            pages = scraper.scrape(req)
+        else:
+            scraper = StaticScraper(req)
+            pages = scraper.scrape()
+
+        for html, page_url, page_num in pages:
+            pages_scraped += 1
+            soup = _parse_html(html)
+            page_items = _extract_items(soup, html, req, page_url, page_num)
+
+            for item in page_items:
+                if req.deduplicate and item.content_hash in seen_hashes:
+                    continue
+                seen_hashes.add(item.content_hash)
+                items.append(item)
+
+        if req.scrape_type == "static" and hasattr(scraper, "close"):
+            scraper.close()
+
+    except Exception as exc:
+        error_msg = f"Scrape failed: {type(exc).__name__}: {exc}"
+        logger.exception(error_msg)
+        errors.append(error_msg)
+
+    return ScrapeResponse(
+        items=items,
+        pages_scraped=pages_scraped,
+        error_count=len(errors),
+        errors=errors,
+        duration_seconds=time.monotonic() - start_time,
+    )

app/services/__init__.py

@@ -0,0 +1,28 @@
+from .job_service import (
+    create_job,
+    get_job,
+    list_jobs,
+    delete_job,
+    cancel_job,
+    execute_job_async,
+    get_job_logs,
+    get_job_results,
+    get_dashboard_stats,
+)
+from .export_service import export_json, export_csv, export_excel, get_job_exports
+
+__all__ = [
+    "create_job",
+    "get_job",
+    "list_jobs",
+    "delete_job",
+    "cancel_job",
+    "execute_job_async",
+    "get_job_logs",
+    "get_job_results",
+    "get_dashboard_stats",
+    "export_json",
+    "export_csv",
+    "export_excel",
+    "get_job_exports",
+]

app/services/export_service.py

@@ -0,0 +1,165 @@
+"""
+Export service — generates JSON, CSV, and Excel files from scraped results.
+Files are stored in the exports/ directory and tracked in the database.
+"""
+from __future__ import annotations
+
+import csv
+import json
+import logging
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+import openpyxl
+from openpyxl.styles import Font, PatternFill, Alignment
+
+from app.models import db, ScrapeJob, ScrapeResult, ExportRecord
+
+logger = logging.getLogger(__name__)
+
+EXPORT_DIR = Path("exports")
+
+
+def _ensure_export_dir() -> Path:
+    EXPORT_DIR.mkdir(parents=True, exist_ok=True)
+    return EXPORT_DIR
+
+
+def _get_results(job_id: int) -> list[ScrapeResult]:
+    return (
+        ScrapeResult.query.filter_by(job_id=job_id)
+        .order_by(ScrapeResult.page_num, ScrapeResult.item_index)
+        .all()
+    )
+
+
+def _record_export(job_id: int, fmt: str, filename: str, filepath: str, row_count: int) -> ExportRecord:
+    size = os.path.getsize(filepath) if os.path.exists(filepath) else 0
+    record = ExportRecord(
+        job_id=job_id,
+        format=fmt,
+        filename=filename,
+        filepath=filepath,
+        file_size_bytes=size,
+        row_count=row_count,
+    )
+    db.session.add(record)
+    db.session.commit()
+    return record
+
+
+def export_json(job_id: int) -> Optional[str]:
+    """Export results as a JSON file. Returns the filepath."""
+    job = ScrapeJob.query.get(job_id)
+    if not job:
+        return None
+
+    results = _get_results(job_id)
+    export_dir = _ensure_export_dir()
+    ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+    filename = f"job_{job_id}_{ts}.json"
+    filepath = str(export_dir / filename)
+
+    data = {
+        "job": job.to_dict(),
+        "total_items": len(results),
+        "exported_at": datetime.now(timezone.utc).isoformat(),
+        "results": [r.to_dict() for r in results],
+    }
+
+    with open(filepath, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+
+    _record_export(job_id, "json", filename, filepath, len(results))
+    logger.info("JSON export for job %s: %s", job_id, filepath)
+    return filepath
+
+
+def export_csv(job_id: int) -> Optional[str]:
+    """Export results as a CSV file."""
+    job = ScrapeJob.query.get(job_id)
+    if not job:
+        return None
+
+    results = _get_results(job_id)
+    export_dir = _ensure_export_dir()
+    ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+    filename = f"job_{job_id}_{ts}.csv"
+    filepath = str(export_dir / filename)
+
+    fieldnames = ["id", "job_id", "page_num", "item_index", "page_url", "content_type", "content", "created_at"]
+
+    with open(filepath, "w", newline="", encoding="utf-8-sig") as f:  # BOM for Excel compat
+        writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
+        writer.writeheader()
+        for r in results:
+            row = r.to_dict()
+            row.pop("metadata", None)
+            writer.writerow(row)
+
+    _record_export(job_id, "csv", filename, filepath, len(results))
+    logger.info("CSV export for job %s: %s", job_id, filepath)
+    return filepath
+
+
+def export_excel(job_id: int) -> Optional[str]:
+    """Export results as a styled Excel (.xlsx) file."""
+    job = ScrapeJob.query.get(job_id)
+    if not job:
+        return None
+
+    results = _get_results(job_id)
+    export_dir = _ensure_export_dir()
+    ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+    filename = f"job_{job_id}_{ts}.xlsx"
+    filepath = str(export_dir / filename)
+
+    wb = openpyxl.Workbook()
+
+    # --- Results sheet ---
+    ws = wb.active
+    ws.title = "Results"
+
+    header_fill = PatternFill("solid", fgColor="1A1A2E")
+    header_font = Font(color="FFFFFF", bold=True)
+    headers = ["#", "Page", "Index", "URL", "Type", "Content", "Scraped At"]
+
+    for col_idx, header in enumerate(headers, 1):
+        cell = ws.cell(row=1, column=col_idx, value=header)
+        cell.fill = header_fill
+        cell.font = header_font
+        cell.alignment = Alignment(horizontal="center")
+
+    for row_idx, r in enumerate(results, 2):
+        ws.cell(row=row_idx, column=1, value=r.id)
+        ws.cell(row=row_idx, column=2, value=r.page_num)
+        ws.cell(row=row_idx, column=3, value=r.item_index)
+        ws.cell(row=row_idx, column=4, value=r.page_url)
+        ws.cell(row=row_idx, column=5, value=r.content_type)
+        ws.cell(row=row_idx, column=6, value=(r.content or "")[:32767])  # Excel cell limit
+        ws.cell(row=row_idx, column=7, value=r.created_at.isoformat() if r.created_at else "")
+
+    ws.column_dimensions["D"].width = 40
+    ws.column_dimensions["F"].width = 60
+
+    # --- Summary sheet ---
+    ws2 = wb.create_sheet("Summary")
+    job_dict = job.to_dict()
+    ws2.cell(1, 1, "Field").font = Font(bold=True)
+    ws2.cell(1, 2, "Value").font = Font(bold=True)
+    for i, (k, v) in enumerate(job_dict.items(), 2):
+        ws2.cell(i, 1, k)
+        ws2.cell(i, 2, str(v))
+    ws2.column_dimensions["A"].width = 25
+    ws2.column_dimensions["B"].width = 50
+
+    wb.save(filepath)
+    _record_export(job_id, "excel", filename, filepath, len(results))
+    logger.info("Excel export for job %s: %s", job_id, filepath)
+    return filepath
+
+
+def get_job_exports(job_id: int) -> list[ExportRecord]:
+    return ExportRecord.query.filter_by(job_id=job_id).order_by(ExportRecord.created_at.desc()).all()

app/services/job_service.py

@@ -0,0 +1,281 @@
+"""
+Service layer for scrape job management.
+Handles job creation, execution, status updates, and result persistence.
+Keeps routes thin and business logic centralized here.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import threading
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+from app.models import db, ScrapeJob, ScrapeResult, JobLog, ExportRecord
+from app.scrapers.engine import ScrapeRequest, run_scrape, ScrapedItem
+
+logger = logging.getLogger(__name__)
+
+
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc).replace(tzinfo=None)
+
+
+# ---------------------------------------------------------------------------
+# Job CRUD
+# ---------------------------------------------------------------------------
+
+
+def create_job(data: dict, user_id: Optional[int] = None) -> ScrapeJob:
+    """Create and persist a new ScrapeJob from validated form/API data."""
+    custom_headers = data.get("custom_headers") or {}
+    if isinstance(custom_headers, str):
+        try:
+            custom_headers = json.loads(custom_headers)
+        except json.JSONDecodeError:
+            custom_headers = {}
+
+    job = ScrapeJob(
+        user_id=user_id,
+        name=data.get("name", f"Job - {data['url'][:50]}"),
+        url=data["url"],
+        html_tag=data.get("html_tag") or None,
+        css_selector=data.get("css_selector") or None,
+        xpath_selector=data.get("xpath_selector") or None,
+        attribute_name=data.get("attribute_name") or None,
+        extraction_type=data.get("extraction_type", "text"),
+        scrape_type=data.get("scrape_type", "static"),
+        follow_pagination=bool(data.get("follow_pagination", False)),
+        max_pages=int(data.get("max_pages", 1)),
+        infinite_scroll=bool(data.get("infinite_scroll", False)),
+        scroll_count=int(data.get("scroll_count", 3)),
+        download_images=bool(data.get("download_images", False)),
+        custom_headers=json.dumps(custom_headers),
+        user_agent=data.get("user_agent") or None,
+        delay_seconds=float(data.get("delay_seconds", 1.0)),
+        timeout_seconds=int(data.get("timeout_seconds", 30)),
+        max_retries=int(data.get("max_retries", 3)),
+        check_robots_txt=bool(data.get("check_robots_txt", True)),
+        deduplicate=bool(data.get("deduplicate", True)),
+        status="pending",
+    )
+    db.session.add(job)
+    db.session.commit()
+    _add_log(job.id, "INFO", f"Job created: {job.name}")
+    return job
+
+
+def get_job(job_id: int) -> Optional[ScrapeJob]:
+    return ScrapeJob.query.get(job_id)
+
+
+def list_jobs(
+    page: int = 1,
+    per_page: int = 20,
+    status: Optional[str] = None,
+    search: Optional[str] = None,
+) -> Any:
+    q = ScrapeJob.query.order_by(ScrapeJob.created_at.desc())
+    if status:
+        q = q.filter(ScrapeJob.status == status)
+    if search:
+        q = q.filter(
+            ScrapeJob.name.ilike(f"%{search}%") | ScrapeJob.url.ilike(f"%{search}%")
+        )
+    return q.paginate(page=page, per_page=per_page, error_out=False)
+
+
+def delete_job(job_id: int) -> bool:
+    job = ScrapeJob.query.get(job_id)
+    if not job:
+        return False
+    db.session.delete(job)
+    db.session.commit()
+    return True
+
+
+def cancel_job(job_id: int) -> bool:
+    job = ScrapeJob.query.get(job_id)
+    if not job or job.status not in ("pending", "running"):
+        return False
+    job.status = "cancelled"
+    db.session.commit()
+    _add_log(job_id, "WARNING", "Job cancelled by user")
+    return True
+
+
+# ---------------------------------------------------------------------------
+# Execution
+# ---------------------------------------------------------------------------
+
+
+def execute_job(job_id: int, app=None) -> None:
+    """
+    Execute a scrape job synchronously.
+    Designed to be called in a background thread with app context.
+    """
+    if app:
+        with app.app_context():
+            _run_job(job_id)
+    else:
+        _run_job(job_id)
+
+
+def execute_job_async(job_id: int, app) -> threading.Thread:
+    """Spawn a daemon thread for async job execution."""
+    thread = threading.Thread(target=execute_job, args=(job_id, app), daemon=True)
+    thread.start()
+    return thread
+
+
+def _run_job(job_id: int) -> None:
+    job = ScrapeJob.query.get(job_id)
+    if not job:
+        logger.error("Job %s not found", job_id)
+        return
+
+    job.status = "running"
+    job.started_at = utcnow()
+    db.session.commit()
+
+    _add_log(job_id, "INFO", f"Starting scrape: {job.url}")
+
+    try:
+        req = ScrapeRequest(
+            url=job.url,
+            html_tag=job.html_tag,
+            css_selector=job.css_selector,
+            xpath_selector=job.xpath_selector,
+            attribute_name=job.attribute_name,
+            extraction_type=job.extraction_type,
+            scrape_type=job.scrape_type,
+            follow_pagination=job.follow_pagination,
+            max_pages=job.max_pages,
+            infinite_scroll=job.infinite_scroll,
+            scroll_count=job.scroll_count,
+            download_images=job.download_images,
+            custom_headers=job.custom_headers_dict,
+            user_agent=job.user_agent,
+            delay_seconds=job.delay_seconds,
+            timeout_seconds=job.timeout_seconds,
+            max_retries=job.max_retries,
+            check_robots_txt=job.check_robots_txt,
+            deduplicate=job.deduplicate,
+        )
+
+        response = run_scrape(req)
+
+        # Persist results
+        _save_results(job_id, response.items)
+
+        # Log errors
+        for err in response.errors:
+            _add_log(job_id, "ERROR", err)
+
+        job.status = "completed" if response.error_count == 0 else "failed"
+        job.total_items = len(response.items)
+        job.pages_scraped = response.pages_scraped
+        job.error_count = response.error_count
+        job.duration_seconds = response.duration_seconds
+        job.completed_at = utcnow()
+        db.session.commit()
+
+        _add_log(
+            job_id,
+            "INFO",
+            f"Completed: {len(response.items)} items from {response.pages_scraped} pages "
+            f"in {response.duration_seconds:.2f}s",
+        )
+
+    except Exception as exc:
+        logger.exception("Job %s failed with unexpected error", job_id)
+        job.status = "failed"
+        job.error_count = (job.error_count or 0) + 1
+        job.completed_at = utcnow()
+        db.session.commit()
+        _add_log(job_id, "ERROR", f"Fatal error: {type(exc).__name__}: {exc}")
+
+
+def _save_results(job_id: int, items: list[ScrapedItem]) -> None:
+    """Bulk insert scraped results."""
+    if not items:
+        return
+    objs = [
+        ScrapeResult(
+            job_id=job_id,
+            page_url=item.page_url,
+            page_num=item.page_num,
+            item_index=item.item_index,
+            content=item.content,
+            content_type=item.content_type,
+            content_hash=item.content_hash,
+            metadata_=json.dumps(item.metadata) if item.metadata else None,
+        )
+        for item in items
+    ]
+    db.session.bulk_save_objects(objs)
+    db.session.commit()
+
+
+# ---------------------------------------------------------------------------
+# Logs
+# ---------------------------------------------------------------------------
+
+
+def _add_log(job_id: int, level: str, message: str, details: Optional[dict] = None) -> None:
+    try:
+        log = JobLog(
+            job_id=job_id,
+            level=level,
+            message=message,
+            details=json.dumps(details) if details else None,
+        )
+        db.session.add(log)
+        db.session.commit()
+    except Exception as exc:
+        logger.warning("Could not persist log for job %s: %s", job_id, exc)
+
+
+def get_job_logs(job_id: int, page: int = 1, per_page: int = 100) -> Any:
+    return (
+        JobLog.query.filter_by(job_id=job_id)
+        .order_by(JobLog.created_at.asc())
+        .paginate(page=page, per_page=per_page, error_out=False)
+    )
+
+
+def get_job_results(job_id: int, page: int = 1, per_page: int = 50) -> Any:
+    return (
+        ScrapeResult.query.filter_by(job_id=job_id)
+        .order_by(ScrapeResult.page_num, ScrapeResult.item_index)
+        .paginate(page=page, per_page=per_page, error_out=False)
+    )
+
+
+# ---------------------------------------------------------------------------
+# Dashboard stats
+# ---------------------------------------------------------------------------
+
+
+def get_dashboard_stats() -> dict:
+    total = ScrapeJob.query.count()
+    completed = ScrapeJob.query.filter_by(status="completed").count()
+    failed = ScrapeJob.query.filter_by(status="failed").count()
+    running = ScrapeJob.query.filter_by(status="running").count()
+    pending = ScrapeJob.query.filter_by(status="pending").count()
+    total_items = db.session.query(db.func.sum(ScrapeJob.total_items)).scalar() or 0
+
+    recent_jobs = (
+        ScrapeJob.query.order_by(ScrapeJob.created_at.desc()).limit(5).all()
+    )
+
+    return {
+        "total_jobs": total,
+        "completed_jobs": completed,
+        "failed_jobs": failed,
+        "running_jobs": running,
+        "pending_jobs": pending,
+        "total_items_scraped": total_items,
+        "recent_jobs": [j.to_dict() for j in recent_jobs],
+        "success_rate": round((completed / total * 100) if total else 0, 1),
+    }

app/static/css/style.css

@@ -0,0 +1,379 @@
+/* ========================================
+   WebScraper.pro — Premium Dark Theme
+   ======================================== */
+
+/* --- CSS Variables --- */
+:root {
+    --bg-primary: #0a0a12;
+    --bg-secondary: #12121e;
+    --bg-card: #1a1a2e;
+    --bg-card-hover: #1f1f35;
+    --bg-input: #16162a;
+    --border: #2a2a45;
+    --border-focus: #6c5ce7;
+    --text-primary: #e8e8f0;
+    --text-secondary: #9898b8;
+    --text-muted: #6868a0;
+    --accent: #6c5ce7;
+    --accent-glow: rgba(108, 92, 231, 0.4);
+    --success: #00cec9;
+    --warning: #fdcb6e;
+    --danger: #ff6b6b;
+    --info: #74b9ff;
+    --font: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif;
+    --radius: 12px;
+    --radius-sm: 8px;
+    --shadow: 0 4px 24px rgba(0,0,0,0.3);
+    --shadow-lg: 0 8px 40px rgba(0,0,0,0.4);
+    --transition: 0.25s cubic-bezier(0.4, 0, 0.2, 1);
+    --nav-height: 64px;
+}
+
+/* --- Reset & Base --- */
+*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+html { scroll-behavior: smooth; }
+body {
+    font-family: var(--font);
+    background: var(--bg-primary);
+    color: var(--text-primary);
+    line-height: 1.6;
+    min-height: 100vh;
+    display: flex;
+    flex-direction: column;
+    -webkit-font-smoothing: antialiased;
+}
+a { color: var(--accent); text-decoration: none; transition: color var(--transition); }
+a:hover { color: #8b7ff0; }
+
+/* --- Layout --- */
+.container { max-width: 1280px; margin: 0 auto; padding: 0 24px; width: 100%; }
+.main-content { flex: 1; padding: calc(var(--nav-height) + 32px) 0 48px; }
+
+/* --- Navbar --- */
+.navbar {
+    position: fixed; top: 0; left: 0; right: 0; z-index: 100;
+    height: var(--nav-height);
+    background: rgba(10, 10, 18, 0.85);
+    backdrop-filter: blur(20px);
+    border-bottom: 1px solid var(--border);
+}
+.nav-container {
+    max-width: 1280px; margin: 0 auto; padding: 0 24px;
+    display: flex; align-items: center; justify-content: space-between; height: 100%;
+}
+.nav-brand { display: flex; align-items: center; gap: 10px; font-weight: 700; font-size: 1.2rem; color: var(--text-primary); }
+.brand-icon { font-size: 1.5rem; }
+.brand-accent { color: var(--accent); }
+.nav-links { display: flex; align-items: center; gap: 8px; }
+.nav-link {
+    display: flex; align-items: center; gap: 6px; padding: 8px 16px;
+    border-radius: var(--radius-sm); color: var(--text-secondary);
+    font-size: 0.9rem; font-weight: 500; transition: all var(--transition);
+}
+.nav-link:hover, .nav-link.active { color: var(--text-primary); background: rgba(108, 92, 231, 0.1); }
+.nav-link.active { color: var(--accent); }
+.nav-icon { width: 18px; height: 18px; }
+.btn-nav-primary {
+    background: linear-gradient(135deg, var(--accent), #8b5cf6);
+    color: #fff !important; border-radius: var(--radius-sm);
+}
+.btn-nav-primary:hover { transform: translateY(-1px); box-shadow: 0 4px 15px var(--accent-glow); }
+.nav-toggle { display: none; background: none; border: none; cursor: pointer; padding: 8px; }
+.nav-toggle span { display: block; width: 22px; height: 2px; background: var(--text-primary); margin: 5px 0; transition: var(--transition); }
+
+/* --- Page Header --- */
+.page-header {
+    display: flex; align-items: flex-start; justify-content: space-between;
+    margin-bottom: 32px; flex-wrap: wrap; gap: 16px;
+}
+.page-title { font-size: 2rem; font-weight: 800; letter-spacing: -0.02em; }
+.page-subtitle { color: var(--text-muted); margin-top: 4px; font-size: 0.95rem; }
+.page-actions { display: flex; gap: 8px; flex-wrap: wrap; }
+
+/* --- Cards --- */
+.card {
+    background: var(--bg-card); border: 1px solid var(--border);
+    border-radius: var(--radius); margin-bottom: 24px;
+    transition: border-color var(--transition);
+}
+.card:hover { border-color: rgba(108, 92, 231, 0.3); }
+.card-header {
+    display: flex; align-items: center; justify-content: space-between;
+    padding: 20px 24px; border-bottom: 1px solid var(--border);
+}
+.card-title { font-size: 1.1rem; font-weight: 600; }
+.card-subtitle { color: var(--text-muted); font-size: 0.85rem; }
+.card-body { padding: 24px; }
+
+/* --- Stats Grid --- */
+.stats-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 16px; margin-bottom: 32px; }
+.stat-card {
+    background: var(--bg-card); border: 1px solid var(--border); border-radius: var(--radius);
+    padding: 24px; display: flex; align-items: center; gap: 16px;
+    transition: all var(--transition); cursor: default;
+}
+.stat-card:hover { transform: translateY(-3px); box-shadow: var(--shadow); }
+.stat-icon { font-size: 2rem; }
+.stat-value { font-size: 1.8rem; font-weight: 800; display: block; line-height: 1; }
+.stat-label { font-size: 0.8rem; color: var(--text-muted); text-transform: uppercase; letter-spacing: 0.05em; margin-top: 4px; display: block; }
+.stat-card-primary { border-left: 3px solid var(--accent); }
+.stat-card-success { border-left: 3px solid var(--success); }
+.stat-card-warning { border-left: 3px solid var(--warning); }
+.stat-card-danger { border-left: 3px solid var(--danger); }
+.stat-card-info { border-left: 3px solid var(--info); }
+.stat-card-accent { border-left: 3px solid #a29bfe; }
+
+/* --- Buttons --- */
+.btn {
+    display: inline-flex; align-items: center; gap: 8px;
+    padding: 10px 20px; border-radius: var(--radius-sm);
+    font-family: var(--font); font-size: 0.9rem; font-weight: 600;
+    border: none; cursor: pointer; transition: all var(--transition);
+    text-decoration: none; line-height: 1.4;
+}
+.btn-primary { background: linear-gradient(135deg, var(--accent), #8b5cf6); color: #fff; }
+.btn-primary:hover { transform: translateY(-1px); box-shadow: 0 4px 20px var(--accent-glow); color: #fff; }
+.btn-secondary { background: var(--bg-card); color: var(--text-primary); border: 1px solid var(--border); }
+.btn-secondary:hover { border-color: var(--accent); }
+.btn-warning { background: rgba(253, 203, 110, 0.15); color: var(--warning); border: 1px solid rgba(253, 203, 110, 0.3); }
+.btn-danger { background: rgba(255, 107, 107, 0.15); color: var(--danger); border: 1px solid rgba(255, 107, 107, 0.3); }
+.btn-outline { background: transparent; color: var(--text-secondary); border: 1px solid var(--border); }
+.btn-outline:hover { border-color: var(--accent); color: var(--accent); }
+.btn-ghost { background: transparent; color: var(--text-secondary); }
+.btn-ghost:hover { color: var(--text-primary); }
+.btn-sm { padding: 6px 14px; font-size: 0.82rem; }
+.btn-lg { padding: 14px 28px; font-size: 1rem; }
+.btn-glow:hover { box-shadow: 0 0 30px var(--accent-glow), 0 4px 20px rgba(0,0,0,0.3); }
+.btn-icon { width: 18px; height: 18px; }
+.btn-icon-sm { background: none; border: none; cursor: pointer; padding: 6px; font-size: 1rem; border-radius: 6px; transition: var(--transition); }
+.btn-icon-sm:hover { background: rgba(255,255,255,0.05); }
+.btn-danger-ghost:hover { background: rgba(255, 107, 107, 0.1); }
+
+/* --- Tables --- */
+.table-responsive { overflow-x: auto; }
+.table { width: 100%; border-collapse: collapse; font-size: 0.9rem; }
+.table th {
+    padding: 12px 16px; text-align: left; font-weight: 600;
+    color: var(--text-muted); font-size: 0.78rem; text-transform: uppercase;
+    letter-spacing: 0.05em; border-bottom: 1px solid var(--border);
+}
+.table td { padding: 14px 16px; border-bottom: 1px solid rgba(42, 42, 69, 0.5); }
+.table-row-hover:hover { background: rgba(108, 92, 231, 0.04); cursor: pointer; }
+.table-link { color: var(--text-primary); font-weight: 500; }
+.table-link:hover { color: var(--accent); }
+.table-compact td { padding: 10px 14px; font-size: 0.85rem; }
+.text-truncate { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+.text-muted { color: var(--text-muted); }
+.content-cell { max-width: 400px; }
+.content-preview { font-size: 0.83rem; color: var(--text-secondary); word-break: break-word; max-height: 60px; overflow: hidden; }
+
+/* --- Badges --- */
+.badge {
+    display: inline-flex; align-items: center; padding: 3px 10px;
+    border-radius: 20px; font-size: 0.75rem; font-weight: 600;
+}
+.badge-muted { background: rgba(152, 152, 184, 0.1); color: var(--text-muted); }
+.badge-type { background: rgba(108, 92, 231, 0.12); color: var(--accent); }
+.count-badge { background: rgba(108, 92, 231, 0.15); color: var(--accent); padding: 2px 10px; border-radius: 12px; font-size: 0.8rem; margin-left: 8px; }
+
+/* --- Status Badges --- */
+.status-badge {
+    display: inline-flex; align-items: center; gap: 6px;
+    padding: 4px 12px; border-radius: 20px; font-size: 0.78rem; font-weight: 600;
+}
+.status-dot { width: 7px; height: 7px; border-radius: 50%; }
+.status-pending { background: rgba(253, 203, 110, 0.12); color: var(--warning); }
+.status-pending .status-dot { background: var(--warning); }
+.status-running { background: rgba(116, 185, 255, 0.12); color: var(--info); }
+.status-running .status-dot { background: var(--info); animation: pulse 1.5s infinite; }
+.status-completed { background: rgba(0, 206, 201, 0.12); color: var(--success); }
+.status-completed .status-dot { background: var(--success); }
+.status-failed { background: rgba(255, 107, 107, 0.12); color: var(--danger); }
+.status-failed .status-dot { background: var(--danger); }
+.status-cancelled { background: rgba(152, 152, 184, 0.12); color: var(--text-muted); }
+.status-cancelled .status-dot { background: var(--text-muted); }
+.status-lg { padding: 8px 18px; font-size: 0.85rem; }
+
+@keyframes pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.4; } }
+
+/* --- Status Bar --- */
+.status-bar {
+    background: var(--bg-card); border: 1px solid var(--border);
+    border-radius: var(--radius); margin-bottom: 24px; overflow: hidden;
+}
+.status-bar-inner { display: flex; align-items: center; justify-content: space-between; padding: 20px 24px; flex-wrap: wrap; gap: 12px; }
+.status-meta { display: flex; gap: 20px; color: var(--text-secondary); font-size: 0.9rem; flex-wrap: wrap; }
+.progress-bar { height: 3px; background: var(--bg-input); }
+.progress-bar-fill { height: 100%; background: linear-gradient(90deg, var(--accent), var(--info)); border-radius: 3px; }
+.progress-animate { width: 100%; animation: progress-sweep 2s ease-in-out infinite; }
+@keyframes progress-sweep {
+    0% { transform: translateX(-100%); }
+    100% { transform: translateX(100%); }
+}
+
+/* --- Detail Grid --- */
+.detail-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 24px; }
+.detail-list { display: flex; flex-direction: column; gap: 14px; }
+.detail-item { display: flex; justify-content: space-between; align-items: flex-start; gap: 16px; }
+.detail-key { font-size: 0.85rem; color: var(--text-muted); min-width: 120px; flex-shrink: 0; }
+.detail-value { font-size: 0.9rem; text-align: right; word-break: break-all; }
+.detail-value.code { font-family: 'Fira Code', monospace; background: var(--bg-input); padding: 2px 8px; border-radius: 4px; font-size: 0.82rem; }
+.detail-value.link { color: var(--accent); }
+
+/* --- Export Buttons --- */
+.export-buttons { display: flex; gap: 12px; flex-wrap: wrap; }
+.btn-export {
+    flex: 1; min-width: 100px; display: flex; flex-direction: column; align-items: center;
+    gap: 8px; padding: 20px; background: var(--bg-input); border: 1px solid var(--border);
+    border-radius: var(--radius-sm); color: var(--text-primary); font-weight: 600;
+    font-size: 0.9rem; cursor: pointer; transition: all var(--transition); text-decoration: none;
+}
+.btn-export:hover { border-color: var(--accent); transform: translateY(-2px); box-shadow: var(--shadow); color: var(--text-primary); }
+.export-icon { font-size: 1.5rem; }
+.export-history { margin-top: 20px; }
+.export-history h4 { font-size: 0.85rem; color: var(--text-muted); margin-bottom: 10px; }
+.export-item { display: flex; align-items: center; gap: 10px; padding: 8px 0; border-bottom: 1px solid rgba(42,42,69,0.3); font-size: 0.85rem; }
+
+/* --- Logs --- */
+.logs-container { font-family: 'Fira Code', 'Cascadia Code', monospace; font-size: 0.82rem; max-height: 400px; overflow-y: auto; }
+.log-entry { display: flex; gap: 12px; padding: 6px 8px; border-radius: 4px; }
+.log-entry:hover { background: rgba(255,255,255,0.02); }
+.log-time { color: var(--text-muted); min-width: 65px; }
+.log-level { font-weight: 700; min-width: 60px; }
+.log-info .log-level { color: var(--info); }
+.log-warning .log-level { color: var(--warning); }
+.log-error .log-level { color: var(--danger); }
+.log-debug .log-level { color: var(--text-muted); }
+.log-message { color: var(--text-secondary); word-break: break-word; }
+
+/* --- Forms --- */
+.form-section { margin-bottom: 24px; }
+.form-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; }
+.form-full { grid-column: 1 / -1; }
+.form-group { display: flex; flex-direction: column; gap: 6px; }
+.form-label { font-size: 0.85rem; font-weight: 600; color: var(--text-secondary); }
+.required { color: var(--danger); }
+.form-input, .form-select, .form-textarea {
+    padding: 12px 16px; background: var(--bg-input); border: 1px solid var(--border);
+    border-radius: var(--radius-sm); color: var(--text-primary); font-family: var(--font);
+    font-size: 0.9rem; transition: all var(--transition); outline: none;
+}
+.form-input:focus, .form-select:focus, .form-textarea:focus {
+    border-color: var(--accent); box-shadow: 0 0 0 3px var(--accent-glow);
+}
+.form-input.form-error, .form-select.form-error { border-color: var(--danger); }
+.form-error-text { color: var(--danger); font-size: 0.8rem; }
+.form-hint { color: var(--text-muted); font-size: 0.78rem; }
+.form-select { appearance: none; background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='0 0 12 12'%3E%3Cpath fill='%239898b8' d='M6 8L1 3h10z'/%3E%3C/svg%3E"); background-repeat: no-repeat; background-position: right 14px center; padding-right: 36px; }
+.form-textarea { resize: vertical; min-height: 80px; }
+
+/* --- Checkboxes --- */
+.form-checkboxes { display: flex; flex-wrap: wrap; gap: 16px; margin-top: 20px; padding-top: 20px; border-top: 1px solid var(--border); }
+.checkbox-label {
+    display: flex; align-items: center; gap: 10px; cursor: pointer;
+    font-size: 0.88rem; color: var(--text-secondary); user-select: none;
+}
+.checkbox-label input { display: none; }
+.checkbox-custom {
+    width: 20px; height: 20px; border: 2px solid var(--border); border-radius: 5px;
+    display: flex; align-items: center; justify-content: center; transition: all var(--transition);
+}
+.checkbox-label input:checked + .checkbox-custom {
+    background: var(--accent); border-color: var(--accent);
+}
+.checkbox-label input:checked + .checkbox-custom::after {
+    content: '✓'; color: #fff; font-size: 0.7rem; font-weight: 700;
+}
+
+/* --- Collapsible --- */
+.collapsible { cursor: pointer; user-select: none; }
+.collapse-icon { width: 20px; height: 20px; color: var(--text-muted); transition: transform var(--transition); }
+.collapsible.collapsed .collapse-icon { transform: rotate(-90deg); }
+
+/* --- Form Actions --- */
+.form-actions { display: flex; justify-content: flex-end; gap: 12px; margin-top: 8px; }
+
+/* --- Filter Bar --- */
+.filter-bar { margin-bottom: 20px; }
+.filter-bar .card-body, .filter-form { padding: 16px 20px; display: flex; align-items: flex-end; gap: 16px; flex-wrap: wrap; }
+.filter-group { display: flex; flex-direction: column; gap: 4px; }
+.filter-label { font-size: 0.75rem; font-weight: 600; color: var(--text-muted); text-transform: uppercase; letter-spacing: 0.05em; }
+.filter-search { flex: 1; min-width: 200px; }
+
+/* --- Pagination --- */
+.pagination { display: flex; justify-content: center; gap: 6px; margin-top: 24px; padding-top: 20px; border-top: 1px solid var(--border); }
+.pagination-link {
+    padding: 8px 14px; border-radius: var(--radius-sm); font-size: 0.85rem;
+    color: var(--text-secondary); background: var(--bg-input); border: 1px solid var(--border);
+    transition: all var(--transition);
+}
+.pagination-link:hover, .pagination-link.active { background: var(--accent); color: #fff; border-color: var(--accent); }
+.pagination-ellipsis { padding: 8px 6px; color: var(--text-muted); }
+
+/* --- Empty State --- */
+.empty-state { text-align: center; padding: 60px 24px; }
+.empty-icon { font-size: 3.5rem; margin-bottom: 16px; }
+.empty-state h3 { font-size: 1.2rem; margin-bottom: 8px; }
+.empty-state p { color: var(--text-muted); margin-bottom: 20px; }
+.empty-state-sm { text-align: center; padding: 32px 16px; }
+
+/* --- Error Page --- */
+.error-page { display: flex; align-items: center; justify-content: center; min-height: 60vh; gap: 60px; }
+.error-code { font-size: 7rem; font-weight: 900; background: linear-gradient(135deg, var(--accent), var(--danger)); -webkit-background-clip: text; -webkit-text-fill-color: transparent; line-height: 1; }
+.error-title { font-size: 1.5rem; margin: 12px 0; }
+.error-description { color: var(--text-muted); margin-bottom: 24px; max-width: 400px; }
+.error-actions { display: flex; gap: 12px; }
+.error-visual { font-size: 5rem; opacity: 0.15; }
+
+/* --- Toast --- */
+.toast-container { position: fixed; top: calc(var(--nav-height) + 16px); right: 24px; z-index: 200; display: flex; flex-direction: column; gap: 10px; }
+.toast {
+    display: flex; align-items: center; gap: 12px; padding: 14px 20px;
+    background: var(--bg-card); border: 1px solid var(--border); border-radius: var(--radius-sm);
+    box-shadow: var(--shadow-lg); animation: slideIn 0.3s ease; min-width: 300px; max-width: 500px;
+}
+.toast-success { border-left: 3px solid var(--success); }
+.toast-danger, .toast-error { border-left: 3px solid var(--danger); }
+.toast-warning { border-left: 3px solid var(--warning); }
+.toast-info { border-left: 3px solid var(--info); }
+.toast-icon { font-size: 1.1rem; font-weight: 700; }
+.toast-success .toast-icon { color: var(--success); }
+.toast-danger .toast-icon, .toast-error .toast-icon { color: var(--danger); }
+.toast-warning .toast-icon { color: var(--warning); }
+.toast-body { flex: 1; font-size: 0.9rem; }
+.toast-close { background: none; border: none; color: var(--text-muted); cursor: pointer; font-size: 1.2rem; padding: 0 4px; }
+@keyframes slideIn { from { transform: translateX(100%); opacity: 0; } to { transform: translateX(0); opacity: 1; } }
+
+/* --- Footer --- */
+.footer { border-top: 1px solid var(--border); padding: 24px 0; margin-top: auto; }
+.footer-inner { display: flex; justify-content: space-between; align-items: center; }
+.footer-text { font-size: 0.82rem; color: var(--text-muted); }
+.footer-links { display: flex; gap: 16px; }
+.footer-link { font-size: 0.82rem; color: var(--text-muted); }
+.footer-link:hover { color: var(--accent); }
+
+/* --- Inline form --- */
+.inline-form { display: inline; }
+
+/* --- Action Group --- */
+.action-group { display: flex; gap: 4px; }
+
+/* --- Responsive --- */
+@media (max-width: 768px) {
+    .nav-links { display: none; position: fixed; top: var(--nav-height); left: 0; right: 0; background: var(--bg-secondary); flex-direction: column; padding: 16px; border-bottom: 1px solid var(--border); }
+    .nav-links.open { display: flex; }
+    .nav-toggle { display: block; }
+    .page-header { flex-direction: column; }
+    .stats-grid { grid-template-columns: repeat(2, 1fr); }
+    .form-grid { grid-template-columns: 1fr; }
+    .detail-grid { grid-template-columns: 1fr; }
+    .filter-form { flex-direction: column; align-items: stretch; }
+    .error-page { flex-direction: column; text-align: center; }
+    .footer-inner { flex-direction: column; gap: 12px; text-align: center; }
+    .status-bar-inner { flex-direction: column; align-items: flex-start; }
+}
+@media (max-width: 480px) {
+    .stats-grid { grid-template-columns: 1fr; }
+    .container { padding: 0 16px; }
+    .page-title { font-size: 1.5rem; }
+}

app/static/js/app.js

@@ -0,0 +1,51 @@
+/* ========================================
+   WebScraper.pro — Interactive JS
+   ======================================== */
+
+document.addEventListener('DOMContentLoaded', () => {
+    // --- Mobile Nav Toggle ---
+    const toggle = document.getElementById('nav-toggle');
+    const links = document.getElementById('nav-links');
+    if (toggle && links) {
+        toggle.addEventListener('click', () => links.classList.toggle('open'));
+    }
+
+    // --- Auto-dismiss Toasts ---
+    document.querySelectorAll('.toast[data-auto-dismiss]').forEach(toast => {
+        const delay = parseInt(toast.dataset.autoDismiss, 10) || 5000;
+        setTimeout(() => {
+            toast.style.animation = 'slideOut 0.3s ease forwards';
+            setTimeout(() => toast.remove(), 300);
+        }, delay);
+    });
+
+    // --- Stat Card Entrance Animation ---
+    document.querySelectorAll('.stat-card').forEach((card, i) => {
+        card.style.opacity = '0';
+        card.style.transform = 'translateY(20px)';
+        setTimeout(() => {
+            card.style.transition = 'all 0.5s cubic-bezier(0.4, 0, 0.2, 1)';
+            card.style.opacity = '1';
+            card.style.transform = 'translateY(0)';
+        }, 80 * i);
+    });
+});
+
+// --- Collapsible Sections ---
+function toggleSection(id) {
+    const el = document.getElementById(id);
+    if (!el) return;
+    const header = el.previousElementSibling || el.parentElement.querySelector('.collapsible');
+    if (el.style.display === 'none') {
+        el.style.display = '';
+        if (header) header.classList.remove('collapsed');
+    } else {
+        el.style.display = 'none';
+        if (header) header.classList.add('collapsed');
+    }
+}
+
+// --- Slide Out animation for toasts ---
+const style = document.createElement('style');
+style.textContent = `@keyframes slideOut { to { transform: translateX(120%); opacity: 0; } }`;
+document.head.appendChild(style);

app/templates/base.html

@@ -0,0 +1,80 @@
+<!DOCTYPE html>
+<html lang="en" data-theme="dark">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="description" content="WebScraper Platform — Professional web scraping with visual configuration, exports, and scheduling.">
+    <title>{% block title %}WebScraper Platform{% endblock %}</title>
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&display=swap" rel="stylesheet">
+    <link rel="stylesheet" href="{{ url_for('static', filename='css/style.css') }}">
+    {% block extra_head %}{% endblock %}
+</head>
+<body>
+    <!-- Navigation -->
+    <nav class="navbar" id="main-nav">
+        <div class="nav-container">
+            <a href="{{ url_for('main.index') }}" class="nav-brand">
+                <span class="brand-icon">🕷️</span>
+                <span class="brand-text">WebScraper<span class="brand-accent">.pro</span></span>
+            </a>
+            <div class="nav-links" id="nav-links">
+                <a href="{{ url_for('main.index') }}" class="nav-link {% if request.endpoint == 'main.index' %}active{% endif %}">
+                    <svg class="nav-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M3 9l9-7 9 7v11a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2z"/><polyline points="9 22 9 12 15 12 15 22"/></svg>
+                    Dashboard
+                </a>
+                <a href="{{ url_for('jobs.list_view') }}" class="nav-link {% if request.endpoint and request.endpoint.startswith('jobs.') and request.endpoint != 'jobs.new_job' %}active{% endif %}">
+                    <svg class="nav-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><rect x="2" y="7" width="20" height="14" rx="2" ry="2"/><path d="M16 21V5a2 2 0 0 0-2-2h-4a2 2 0 0 0-2 2v16"/></svg>
+                    Jobs
+                </a>
+                <a href="{{ url_for('jobs.new_job') }}" class="nav-link btn-nav-primary {% if request.endpoint == 'jobs.new_job' %}active{% endif %}">
+                    <svg class="nav-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><circle cx="12" cy="12" r="10"/><line x1="12" y1="8" x2="12" y2="16"/><line x1="8" y1="12" x2="16" y2="12"/></svg>
+                    New Scrape
+                </a>
+            </div>
+            <button class="nav-toggle" id="nav-toggle" aria-label="Toggle navigation">
+                <span></span><span></span><span></span>
+            </button>
+        </div>
+    </nav>
+
+    <!-- Flash Messages -->
+    {% with messages = get_flashed_messages(with_categories=true) %}
+    {% if messages %}
+    <div class="toast-container" id="toast-container">
+        {% for category, message in messages %}
+        <div class="toast toast-{{ category }}" data-auto-dismiss="5000">
+            <div class="toast-icon">
+                {% if category == 'success' %}✓{% elif category == 'danger' or category == 'error' %}✕{% elif category == 'warning' %}⚠{% else %}ℹ{% endif %}
+            </div>
+            <div class="toast-body">{{ message }}</div>
+            <button class="toast-close" onclick="this.parentElement.remove()">×</button>
+        </div>
+        {% endfor %}
+    </div>
+    {% endif %}
+    {% endwith %}
+
+    <!-- Main Content -->
+    <main class="main-content">
+        <div class="container">
+            {% block content %}{% endblock %}
+        </div>
+    </main>
+
+    <!-- Footer -->
+    <footer class="footer">
+        <div class="container footer-inner">
+            <p class="footer-text">© 2026 WebScraper<span class="brand-accent">.pro</span> — Built with Flask & BeautifulSoup</p>
+            <div class="footer-links">
+                <a href="{{ url_for('main.health') }}" class="footer-link">Health</a>
+                <a href="{{ url_for('main.metrics') }}" class="footer-link">Metrics</a>
+            </div>
+        </div>
+    </footer>
+
+    <script src="{{ url_for('static', filename='js/app.js') }}"></script>
+    {% block extra_scripts %}{% endblock %}
+</body>
+</html>

app/templates/pages/dashboard.html

@@ -0,0 +1,113 @@
+{% extends "base.html" %}
+{% block title %}Dashboard — WebScraper.pro{% endblock %}
+
+{% block content %}
+<div class="page-header">
+    <div class="page-header-content">
+        <h1 class="page-title">Dashboard</h1>
+        <p class="page-subtitle">Monitor your scraping operations at a glance</p>
+    </div>
+    <a href="{{ url_for('jobs.new_job') }}" class="btn btn-primary btn-glow">
+        <svg class="btn-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><circle cx="12" cy="12" r="10"/><line x1="12" y1="8" x2="12" y2="16"/><line x1="8" y1="12" x2="16" y2="12"/></svg>
+        New Scrape Job
+    </a>
+</div>
+
+<!-- Stats Grid -->
+<div class="stats-grid" id="stats-grid">
+    <div class="stat-card stat-card-primary">
+        <div class="stat-icon">📊</div>
+        <div class="stat-info">
+            <span class="stat-value">{{ stats.total_jobs }}</span>
+            <span class="stat-label">Total Jobs</span>
+        </div>
+    </div>
+    <div class="stat-card stat-card-success">
+        <div class="stat-icon">✅</div>
+        <div class="stat-info">
+            <span class="stat-value">{{ stats.completed_jobs }}</span>
+            <span class="stat-label">Completed</span>
+        </div>
+    </div>
+    <div class="stat-card stat-card-warning">
+        <div class="stat-icon">⏳</div>
+        <div class="stat-info">
+            <span class="stat-value">{{ stats.running_jobs + stats.pending_jobs }}</span>
+            <span class="stat-label">In Progress</span>
+        </div>
+    </div>
+    <div class="stat-card stat-card-danger">
+        <div class="stat-icon">❌</div>
+        <div class="stat-info">
+            <span class="stat-value">{{ stats.failed_jobs }}</span>
+            <span class="stat-label">Failed</span>
+        </div>
+    </div>
+    <div class="stat-card stat-card-info">
+        <div class="stat-icon">📦</div>
+        <div class="stat-info">
+            <span class="stat-value">{{ stats.total_items_scraped }}</span>
+            <span class="stat-label">Items Scraped</span>
+        </div>
+    </div>
+    <div class="stat-card stat-card-accent">
+        <div class="stat-icon">🎯</div>
+        <div class="stat-info">
+            <span class="stat-value">{{ stats.success_rate }}%</span>
+            <span class="stat-label">Success Rate</span>
+        </div>
+    </div>
+</div>
+
+<!-- Recent Jobs -->
+<div class="card" id="recent-jobs-card">
+    <div class="card-header">
+        <h2 class="card-title">Recent Jobs</h2>
+        <a href="{{ url_for('jobs.list_view') }}" class="btn btn-outline btn-sm">View All →</a>
+    </div>
+    <div class="card-body">
+        {% if stats.recent_jobs %}
+        <div class="table-responsive">
+            <table class="table" id="recent-jobs-table">
+                <thead>
+                    <tr>
+                        <th>ID</th>
+                        <th>Name</th>
+                        <th>URL</th>
+                        <th>Status</th>
+                        <th>Items</th>
+                        <th>Duration</th>
+                        <th>Created</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for job in stats.recent_jobs %}
+                    <tr class="table-row-hover" onclick="window.location='/jobs/{{ job.id }}'">
+                        <td><span class="badge badge-muted">#{{ job.id }}</span></td>
+                        <td class="text-truncate" style="max-width: 200px;">{{ job.name }}</td>
+                        <td class="text-truncate text-muted" style="max-width: 250px;">{{ job.url }}</td>
+                        <td>
+                            <span class="status-badge status-{{ job.status }}">
+                                <span class="status-dot"></span>
+                                {{ job.status | capitalize }}
+                            </span>
+                        </td>
+                        <td>{{ job.total_items }}</td>
+                        <td>{% if job.duration_seconds %}{{ "%.1f"|format(job.duration_seconds) }}s{% else %}—{% endif %}</td>
+                        <td class="text-muted">{{ job.created_at[:16] if job.created_at else '—' }}</td>
+                    </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+        </div>
+        {% else %}
+        <div class="empty-state">
+            <div class="empty-icon">🕸️</div>
+            <h3>No scrape jobs yet</h3>
+            <p>Create your first scraping job to get started.</p>
+            <a href="{{ url_for('jobs.new_job') }}" class="btn btn-primary">Create Your First Job</a>
+        </div>
+        {% endif %}
+    </div>
+</div>
+{% endblock %}

app/templates/pages/error.html

@@ -0,0 +1,30 @@
+{% extends "base.html" %}
+{% block title %}{{ code }} — WebScraper.pro{% endblock %}
+
+{% block content %}
+<div class="error-page">
+    <div class="error-content">
+        <div class="error-code">{{ code }}</div>
+        <h1 class="error-title">{{ message }}</h1>
+        <p class="error-description">
+            {% if code == 404 %}
+                The page you're looking for doesn't exist or has been moved.
+            {% elif code == 429 %}
+                You've made too many requests. Please wait a moment and try again.
+            {% elif code == 500 %}
+                Something went wrong on our end. Our team has been notified.
+            {% else %}
+                An unexpected error occurred. Please try again.
+            {% endif %}
+        </p>
+        <div class="error-actions">
+            <a href="{{ url_for('main.index') }}" class="btn btn-primary">← Back to Dashboard</a>
+            <a href="{{ url_for('jobs.list_view') }}" class="btn btn-secondary">View Jobs</a>
+        </div>
+    </div>
+    <div class="error-visual">
+        <div class="error-spider">🕷️</div>
+        <div class="error-web"></div>
+    </div>
+</div>
+{% endblock %}

app/templates/pages/job_detail.html

@@ -0,0 +1,262 @@
+{% extends "base.html" %}
+{% block title %}Job #{{ job.id }} — WebScraper.pro{% endblock %}
+
+{% block content %}
+<div class="page-header">
+    <div class="page-header-content">
+        <h1 class="page-title">{{ job.name }}</h1>
+        <p class="page-subtitle">Job #{{ job.id }} — {{ job.url }}</p>
+    </div>
+    <div class="page-actions">
+        {% if job.status in ('pending', 'running') %}
+        <form method="post" action="{{ url_for('jobs.cancel', job_id=job.id) }}" class="inline-form">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+            <button type="submit" class="btn btn-warning" id="btn-cancel">⏸️ Cancel</button>
+        </form>
+        {% endif %}
+        {% if job.status in ('completed', 'failed', 'cancelled') %}
+        <form method="post" action="{{ url_for('jobs.rerun', job_id=job.id) }}" class="inline-form">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+            <button type="submit" class="btn btn-secondary" id="btn-rerun">🔄 Re-run</button>
+        </form>
+        {% endif %}
+        <form method="post" action="{{ url_for('jobs.delete', job_id=job.id) }}" class="inline-form" onsubmit="return confirm('Permanently delete this job and all results?')">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+            <button type="submit" class="btn btn-danger" id="btn-delete">🗑️ Delete</button>
+        </form>
+    </div>
+</div>
+
+<!-- Status Bar -->
+<div class="status-bar status-bar-{{ job.status }}" id="job-status-bar" data-job-id="{{ job.id }}" data-status="{{ job.status }}">
+    <div class="status-bar-inner">
+        <span class="status-badge status-{{ job.status }} status-lg">
+            <span class="status-dot"></span>
+            {{ job.status | upper }}
+        </span>
+        <div class="status-meta">
+            <span>📦 <strong>{{ job.total_items }}</strong> items</span>
+            <span>📄 <strong>{{ job.pages_scraped }}</strong> pages</span>
+            {% if job.error_count %}<span>❌ <strong>{{ job.error_count }}</strong> errors</span>{% endif %}
+            {% if job.duration_seconds %}<span>⏱️ <strong>{{ "%.2f"|format(job.duration_seconds) }}</strong>s</span>{% endif %}
+        </div>
+    </div>
+    {% if job.status in ('pending', 'running') %}
+    <div class="progress-bar">
+        <div class="progress-bar-fill progress-animate"></div>
+    </div>
+    {% endif %}
+</div>
+
+<!-- Job Info -->
+<div class="detail-grid">
+    <div class="card" id="job-config-card">
+        <div class="card-header">
+            <h2 class="card-title">⚙️ Configuration</h2>
+        </div>
+        <div class="card-body">
+            <div class="detail-list">
+                <div class="detail-item">
+                    <span class="detail-key">URL</span>
+                    <a href="{{ job.url }}" target="_blank" rel="noopener" class="detail-value link">{{ job.url }}</a>
+                </div>
+                <div class="detail-item">
+                    <span class="detail-key">Scrape Type</span>
+                    <span class="detail-value"><span class="badge badge-type">{{ job.scrape_type }}</span></span>
+                </div>
+                <div class="detail-item">
+                    <span class="detail-key">Extraction</span>
+                    <span class="detail-value">{{ job.extraction_type }}</span>
+                </div>
+                {% if job.css_selector %}
+                <div class="detail-item">
+                    <span class="detail-key">CSS Selector</span>
+                    <code class="detail-value code">{{ job.css_selector }}</code>
+                </div>
+                {% endif %}
+                {% if job.xpath_selector %}
+                <div class="detail-item">
+                    <span class="detail-key">XPath</span>
+                    <code class="detail-value code">{{ job.xpath_selector }}</code>
+                </div>
+                {% endif %}
+                {% if job.html_tag %}
+                <div class="detail-item">
+                    <span class="detail-key">HTML Tag</span>
+                    <code class="detail-value code">{{ job.html_tag }}</code>
+                </div>
+                {% endif %}
+                <div class="detail-item">
+                    <span class="detail-key">Pagination</span>
+                    <span class="detail-value">{{ 'Yes' if job.follow_pagination else 'No' }} (max {{ job.max_pages }} pages)</span>
+                </div>
+                <div class="detail-item">
+                    <span class="detail-key">Delay</span>
+                    <span class="detail-value">{{ job.delay_seconds }}s</span>
+                </div>
+                <div class="detail-item">
+                    <span class="detail-key">Created</span>
+                    <span class="detail-value">{{ job.created_at.strftime('%Y-%m-%d %H:%M:%S') if job.created_at else '—' }}</span>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <!-- Export -->
+    <div class="card" id="export-card">
+        <div class="card-header">
+            <h2 class="card-title">📤 Export Results</h2>
+        </div>
+        <div class="card-body">
+            {% if job.total_items > 0 %}
+            <div class="export-buttons">
+                <a href="{{ url_for('jobs.export', job_id=job.id, fmt='json') }}" class="btn btn-export" id="export-json">
+                    <span class="export-icon">{ }</span>
+                    <span>JSON</span>
+                </a>
+                <a href="{{ url_for('jobs.export', job_id=job.id, fmt='csv') }}" class="btn btn-export" id="export-csv">
+                    <span class="export-icon">📊</span>
+                    <span>CSV</span>
+                </a>
+                <a href="{{ url_for('jobs.export', job_id=job.id, fmt='excel') }}" class="btn btn-export" id="export-excel">
+                    <span class="export-icon">📗</span>
+                    <span>Excel</span>
+                </a>
+            </div>
+            {% if exports %}
+            <div class="export-history">
+                <h4>Previous Exports</h4>
+                {% for exp in exports %}
+                <div class="export-item">
+                    <span class="badge badge-type">{{ exp.format }}</span>
+                    <span>{{ exp.filename }}</span>
+                    <span class="text-muted">{{ exp.row_count }} rows</span>
+                </div>
+                {% endfor %}
+            </div>
+            {% endif %}
+            {% else %}
+            <div class="empty-state-sm">
+                <p class="text-muted">No results to export yet.</p>
+            </div>
+            {% endif %}
+        </div>
+    </div>
+</div>
+
+<!-- Results -->
+<div class="card" id="results-card">
+    <div class="card-header">
+        <h2 class="card-title">📦 Scraped Results <span class="count-badge">{{ results.total }}</span></h2>
+    </div>
+    <div class="card-body">
+        {% if results.items %}
+        <div class="table-responsive">
+            <table class="table table-compact" id="results-table">
+                <thead>
+                    <tr>
+                        <th>#</th>
+                        <th>Page</th>
+                        <th>Type</th>
+                        <th>Content</th>
+                        <th>URL</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for r in results.items %}
+                    <tr>
+                        <td>{{ r.item_index }}</td>
+                        <td>{{ r.page_num }}</td>
+                        <td><span class="badge badge-muted">{{ r.content_type }}</span></td>
+                        <td class="content-cell">
+                            <div class="content-preview">{{ r.content[:300] if r.content else '—' }}{% if r.content and r.content|length > 300 %}…{% endif %}</div>
+                        </td>
+                        <td class="text-truncate text-muted" style="max-width:180px;">{{ r.page_url }}</td>
+                    </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+        </div>
+        {% if results.pages > 1 %}
+        <div class="pagination">
+            {% if results.has_prev %}
+            <a href="{{ url_for('jobs.detail', job_id=job.id, rpage=results.prev_num) }}" class="pagination-link">← Prev</a>
+            {% endif %}
+            {% for p in results.iter_pages(left_edge=1, right_edge=1, left_current=2, right_current=2) %}
+                {% if p %}
+                    <a href="{{ url_for('jobs.detail', job_id=job.id, rpage=p) }}" class="pagination-link {% if p == results.page %}active{% endif %}">{{ p }}</a>
+                {% else %}
+                    <span class="pagination-ellipsis">…</span>
+                {% endif %}
+            {% endfor %}
+            {% if results.has_next %}
+            <a href="{{ url_for('jobs.detail', job_id=job.id, rpage=results.next_num) }}" class="pagination-link">Next →</a>
+            {% endif %}
+        </div>
+        {% endif %}
+        {% else %}
+        <div class="empty-state-sm">
+            <p class="text-muted">No results scraped yet.</p>
+        </div>
+        {% endif %}
+    </div>
+</div>
+
+<!-- Logs -->
+<div class="card" id="logs-card">
+    <div class="card-header collapsible" onclick="toggleSection('logs-body')">
+        <h2 class="card-title">📋 Job Logs <span class="count-badge">{{ logs.total }}</span></h2>
+        <svg class="collapse-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><polyline points="6 9 12 15 18 9"/></svg>
+    </div>
+    <div class="card-body" id="logs-body">
+        {% if logs.items %}
+        <div class="logs-container">
+            {% for log in logs.items %}
+            <div class="log-entry log-{{ log.level | lower }}">
+                <span class="log-time">{{ log.created_at.strftime('%H:%M:%S') if log.created_at else '' }}</span>
+                <span class="log-level">{{ log.level }}</span>
+                <span class="log-message">{{ log.message }}</span>
+            </div>
+            {% endfor %}
+        </div>
+        {% else %}
+        <p class="text-muted">No logs recorded.</p>
+        {% endif %}
+    </div>
+</div>
+{% endblock %}
+
+{% block extra_scripts %}
+{% if job.status in ('pending', 'running') %}
+<script>
+    // Poll for status updates
+    (function() {
+        const jobId = {{ job.id }};
+        const statusBar = document.getElementById('job-status-bar');
+        const interval = setInterval(async () => {
+            try {
+                const res = await fetch(`/jobs/api/jobs/${jobId}/status`);
+                const data = await res.json();
+                if (data.status !== '{{ job.status }}') {
+                    window.location.reload();
+                }
+                // Update live counts
+                const meta = statusBar.querySelector('.status-meta');
+                if (meta) {
+                    meta.innerHTML = `
+                        <span>📦 <strong>${data.total_items}</strong> items</span>
+                        <span>📄 <strong>${data.pages_scraped}</strong> pages</span>
+                        ${data.error_count ? `<span>❌ <strong>${data.error_count}</strong> errors</span>` : ''}
+                    `;
+                }
+            } catch(e) { /* silently continue */ }
+        }, 2000);
+
+        // Stop polling when job completes
+        if (!['pending', 'running'].includes('{{ job.status }}')) {
+            clearInterval(interval);
+        }
+    })();
+</script>
+{% endif %}
+{% endblock %}

app/templates/pages/jobs.html

@@ -0,0 +1,122 @@
+{% extends "base.html" %}
+{% block title %}Jobs — WebScraper.pro{% endblock %}
+
+{% block content %}
+<div class="page-header">
+    <div class="page-header-content">
+        <h1 class="page-title">Scrape Jobs</h1>
+        <p class="page-subtitle">Manage and monitor all your scraping tasks</p>
+    </div>
+    <a href="{{ url_for('jobs.new_job') }}" class="btn btn-primary btn-glow">
+        <svg class="btn-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><circle cx="12" cy="12" r="10"/><line x1="12" y1="8" x2="12" y2="16"/><line x1="8" y1="12" x2="16" y2="12"/></svg>
+        New Scrape Job
+    </a>
+</div>
+
+<!-- Filters -->
+<div class="card filter-bar" id="job-filters">
+    <form method="get" class="filter-form">
+        <div class="filter-group">
+            <label class="filter-label">Status</label>
+            <select name="status" class="form-select" id="filter-status">
+                <option value="">All Statuses</option>
+                <option value="pending" {% if status_filter == 'pending' %}selected{% endif %}>Pending</option>
+                <option value="running" {% if status_filter == 'running' %}selected{% endif %}>Running</option>
+                <option value="completed" {% if status_filter == 'completed' %}selected{% endif %}>Completed</option>
+                <option value="failed" {% if status_filter == 'failed' %}selected{% endif %}>Failed</option>
+                <option value="cancelled" {% if status_filter == 'cancelled' %}selected{% endif %}>Cancelled</option>
+            </select>
+        </div>
+        <div class="filter-group filter-search">
+            <label class="filter-label">Search</label>
+            <input type="text" name="search" class="form-input" placeholder="Search by name or URL..." value="{{ search or '' }}" id="filter-search">
+        </div>
+        <button type="submit" class="btn btn-secondary" id="filter-submit">Filter</button>
+        {% if status_filter or search %}
+        <a href="{{ url_for('jobs.list_view') }}" class="btn btn-ghost" id="filter-clear">Clear</a>
+        {% endif %}
+    </form>
+</div>
+
+<!-- Jobs Table -->
+<div class="card" id="jobs-list-card">
+    <div class="card-body">
+        {% if pagination.items %}
+        <div class="table-responsive">
+            <table class="table" id="jobs-table">
+                <thead>
+                    <tr>
+                        <th>ID</th>
+                        <th>Name</th>
+                        <th>URL</th>
+                        <th>Type</th>
+                        <th>Status</th>
+                        <th>Items</th>
+                        <th>Duration</th>
+                        <th>Created</th>
+                        <th>Actions</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for job in pagination.items %}
+                    <tr class="table-row-hover">
+                        <td><span class="badge badge-muted">#{{ job.id }}</span></td>
+                        <td>
+                            <a href="{{ url_for('jobs.detail', job_id=job.id) }}" class="table-link">{{ job.name }}</a>
+                        </td>
+                        <td class="text-truncate text-muted" style="max-width: 220px;" title="{{ job.url }}">{{ job.url }}</td>
+                        <td><span class="badge badge-type">{{ job.scrape_type }}</span></td>
+                        <td>
+                            <span class="status-badge status-{{ job.status }}">
+                                <span class="status-dot"></span>
+                                {{ job.status | capitalize }}
+                            </span>
+                        </td>
+                        <td>{{ job.total_items }}</td>
+                        <td>{% if job.duration_seconds %}{{ "%.1f"|format(job.duration_seconds) }}s{% else %}—{% endif %}</td>
+                        <td class="text-muted">{{ job.created_at.strftime('%Y-%m-%d %H:%M') if job.created_at else '—' }}</td>
+                        <td>
+                            <div class="action-group">
+                                <a href="{{ url_for('jobs.detail', job_id=job.id) }}" class="btn btn-icon-sm" title="View Details">👁️</a>
+                                <form method="post" action="{{ url_for('jobs.delete', job_id=job.id) }}" class="inline-form" onsubmit="return confirm('Delete job #{{ job.id }}?')">
+                                    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+                                    <button type="submit" class="btn btn-icon-sm btn-danger-ghost" title="Delete">🗑️</button>
+                                </form>
+                            </div>
+                        </td>
+                    </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+        </div>
+
+        <!-- Pagination -->
+        {% if pagination.pages > 1 %}
+        <div class="pagination" id="jobs-pagination">
+            {% if pagination.has_prev %}
+            <a href="{{ url_for('jobs.list_view', page=pagination.prev_num, status=status_filter, search=search) }}" class="pagination-link">← Prev</a>
+            {% endif %}
+            {% for p in pagination.iter_pages(left_edge=1, right_edge=1, left_current=2, right_current=2) %}
+                {% if p %}
+                    <a href="{{ url_for('jobs.list_view', page=p, status=status_filter, search=search) }}" class="pagination-link {% if p == pagination.page %}active{% endif %}">{{ p }}</a>
+                {% else %}
+                    <span class="pagination-ellipsis">…</span>
+                {% endif %}
+            {% endfor %}
+            {% if pagination.has_next %}
+            <a href="{{ url_for('jobs.list_view', page=pagination.next_num, status=status_filter, search=search) }}" class="pagination-link">Next →</a>
+            {% endif %}
+        </div>
+        {% endif %}
+
+        {% else %}
+        <div class="empty-state">
+            <div class="empty-icon">🔍</div>
+            <h3>No jobs found</h3>
+            <p>{% if status_filter or search %}Try adjusting your filters.{% else %}Create your first scraping job to get started.{% endif %}</p>
+            <a href="{{ url_for('jobs.new_job') }}" class="btn btn-primary">Create New Job</a>
+        </div>
+        {% endif %}
+    </div>
+</div>
+{% endblock %}

app/templates/pages/new_job.html

@@ -0,0 +1,181 @@
+{% extends "base.html" %}
+{% block title %}New Scrape Job — WebScraper.pro{% endblock %}
+
+{% block content %}
+<div class="page-header">
+    <div class="page-header-content">
+        <h1 class="page-title">Create Scrape Job</h1>
+        <p class="page-subtitle">Configure and launch a new web scraping task</p>
+    </div>
+</div>
+
+<form method="post" action="{{ url_for('jobs.new_job') }}" id="new-job-form" class="job-form">
+    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+
+    <!-- Basic Config -->
+    <div class="card form-section" id="section-basic">
+        <div class="card-header">
+            <h2 class="card-title">🎯 Basic Configuration</h2>
+        </div>
+        <div class="card-body">
+            <div class="form-grid">
+                <div class="form-group form-full">
+                    <label for="name" class="form-label">Job Name</label>
+                    <input type="text" name="name" id="name" class="form-input {% if errors.get('name') %}form-error{% endif %}" placeholder="e.g. Product Prices - Amazon" value="{{ form_data.get('name', '') }}">
+                    {% if errors.get('name') %}<span class="form-error-text">{{ errors.name }}</span>{% endif %}
+                </div>
+
+                <div class="form-group form-full">
+                    <label for="url" class="form-label">Target URL <span class="required">*</span></label>
+                    <input type="url" name="url" id="url" class="form-input {% if errors.get('url') %}form-error{% endif %}" placeholder="https://example.com/page" value="{{ form_data.get('url', '') }}" required>
+                    {% if errors.get('url') %}<span class="form-error-text">{{ errors.url }}</span>{% endif %}
+                </div>
+
+                <div class="form-group">
+                    <label for="scrape_type" class="form-label">Scrape Type</label>
+                    <select name="scrape_type" id="scrape_type" class="form-select">
+                        <option value="static" {% if form_data.get('scrape_type') == 'static' %}selected{% endif %}>Static (Requests + BS4)</option>
+                        <option value="dynamic" {% if form_data.get('scrape_type') == 'dynamic' %}selected{% endif %}>Dynamic (Playwright)</option>
+                    </select>
+                    <span class="form-hint">Use Dynamic for JS-rendered pages</span>
+                </div>
+
+                <div class="form-group">
+                    <label for="extraction_type" class="form-label">Extraction Type</label>
+                    <select name="extraction_type" id="extraction_type" class="form-select {% if errors.get('extraction_type') %}form-error{% endif %}">
+                        <option value="text" {% if form_data.get('extraction_type') == 'text' %}selected{% endif %}>Text Content</option>
+                        <option value="links" {% if form_data.get('extraction_type') == 'links' %}selected{% endif %}>Links (URLs)</option>
+                        <option value="images" {% if form_data.get('extraction_type') == 'images' %}selected{% endif %}>Image URLs</option>
+                        <option value="attributes" {% if form_data.get('extraction_type') == 'attributes' %}selected{% endif %}>HTML Attributes</option>
+                        <option value="table" {% if form_data.get('extraction_type') == 'table' %}selected{% endif %}>Table Data</option>
+                        <option value="json_ld" {% if form_data.get('extraction_type') == 'json_ld' %}selected{% endif %}>JSON-LD Schema</option>
+                        <option value="full_html" {% if form_data.get('extraction_type') == 'full_html' %}selected{% endif %}>Full HTML</option>
+                    </select>
+                    {% if errors.get('extraction_type') %}<span class="form-error-text">{{ errors.extraction_type }}</span>{% endif %}
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <!-- Selectors -->
+    <div class="card form-section" id="section-selectors">
+        <div class="card-header">
+            <h2 class="card-title">🔎 Selectors</h2>
+            <span class="card-subtitle">Define what to extract (leave blank for all elements)</span>
+        </div>
+        <div class="card-body">
+            <div class="form-grid">
+                <div class="form-group">
+                    <label for="css_selector" class="form-label">CSS Selector</label>
+                    <input type="text" name="css_selector" id="css_selector" class="form-input {% if errors.get('css_selector') %}form-error{% endif %}" placeholder="e.g. .product-card h2" value="{{ form_data.get('css_selector', '') }}">
+                    {% if errors.get('css_selector') %}<span class="form-error-text">{{ errors.css_selector }}</span>{% endif %}
+                </div>
+
+                <div class="form-group">
+                    <label for="xpath_selector" class="form-label">XPath Selector</label>
+                    <input type="text" name="xpath_selector" id="xpath_selector" class="form-input {% if errors.get('xpath_selector') %}form-error{% endif %}" placeholder="e.g. //div[@class='item']/h2" value="{{ form_data.get('xpath_selector', '') }}">
+                    {% if errors.get('xpath_selector') %}<span class="form-error-text">{{ errors.xpath_selector }}</span>{% endif %}
+                </div>
+
+                <div class="form-group">
+                    <label for="html_tag" class="form-label">HTML Tag</label>
+                    <input type="text" name="html_tag" id="html_tag" class="form-input" placeholder="e.g. h2, p, div" value="{{ form_data.get('html_tag', '') }}">
+                </div>
+
+                <div class="form-group">
+                    <label for="attribute_name" class="form-label">Attribute Name</label>
+                    <input type="text" name="attribute_name" id="attribute_name" class="form-input" placeholder="e.g. href, src, data-id" value="{{ form_data.get('attribute_name', '') }}">
+                    <span class="form-hint">Used when extraction type is "Attributes"</span>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <!-- Advanced Options -->
+    <div class="card form-section" id="section-advanced">
+        <div class="card-header collapsible" onclick="toggleSection('advanced-body')">
+            <h2 class="card-title">⚙️ Advanced Options</h2>
+            <svg class="collapse-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><polyline points="6 9 12 15 18 9"/></svg>
+        </div>
+        <div class="card-body" id="advanced-body">
+            <div class="form-grid">
+                <div class="form-group">
+                    <label for="delay_seconds" class="form-label">Delay Between Requests (s)</label>
+                    <input type="number" name="delay_seconds" id="delay_seconds" class="form-input {% if errors.get('delay_seconds') %}form-error{% endif %}" value="{{ form_data.get('delay_seconds', '1.0') }}" step="0.1" min="0" max="60">
+                    {% if errors.get('delay_seconds') %}<span class="form-error-text">{{ errors.delay_seconds }}</span>{% endif %}
+                </div>
+
+                <div class="form-group">
+                    <label for="timeout_seconds" class="form-label">Timeout (s)</label>
+                    <input type="number" name="timeout_seconds" id="timeout_seconds" class="form-input" value="{{ form_data.get('timeout_seconds', '30') }}" min="5" max="120">
+                </div>
+
+                <div class="form-group">
+                    <label for="max_retries" class="form-label">Max Retries</label>
+                    <input type="number" name="max_retries" id="max_retries" class="form-input" value="{{ form_data.get('max_retries', '3') }}" min="0" max="10">
+                </div>
+
+                <div class="form-group">
+                    <label for="max_pages" class="form-label">Max Pages</label>
+                    <input type="number" name="max_pages" id="max_pages" class="form-input {% if errors.get('max_pages') %}form-error{% endif %}" value="{{ form_data.get('max_pages', '1') }}" min="1" max="200">
+                    {% if errors.get('max_pages') %}<span class="form-error-text">{{ errors.max_pages }}</span>{% endif %}
+                </div>
+
+                <div class="form-group">
+                    <label for="scroll_count" class="form-label">Scroll Count</label>
+                    <input type="number" name="scroll_count" id="scroll_count" class="form-input" value="{{ form_data.get('scroll_count', '3') }}" min="1" max="50">
+                    <span class="form-hint">For infinite scroll pages</span>
+                </div>
+
+                <div class="form-group">
+                    <label for="user_agent" class="form-label">Custom User Agent</label>
+                    <input type="text" name="user_agent" id="user_agent" class="form-input" placeholder="Leave blank for random UA" value="{{ form_data.get('user_agent', '') }}">
+                </div>
+
+                <div class="form-group form-full">
+                    <label for="custom_headers" class="form-label">Custom Headers (JSON)</label>
+                    <textarea name="custom_headers" id="custom_headers" class="form-textarea" rows="3" placeholder='{"Accept": "text/html", "Cookie": "session=abc123"}'>{{ form_data.get('custom_headers', '') }}</textarea>
+                </div>
+            </div>
+
+            <!-- Checkboxes -->
+            <div class="form-checkboxes">
+                <label class="checkbox-label">
+                    <input type="checkbox" name="follow_pagination" {% if form_data.get('follow_pagination') %}checked{% endif %}>
+                    <span class="checkbox-custom"></span>
+                    Follow Pagination
+                </label>
+                <label class="checkbox-label">
+                    <input type="checkbox" name="infinite_scroll" {% if form_data.get('infinite_scroll') %}checked{% endif %}>
+                    <span class="checkbox-custom"></span>
+                    Infinite Scroll
+                </label>
+                <label class="checkbox-label">
+                    <input type="checkbox" name="check_robots_txt" {% if form_data.get('check_robots_txt', True) %}checked{% endif %}>
+                    <span class="checkbox-custom"></span>
+                    Respect robots.txt
+                </label>
+                <label class="checkbox-label">
+                    <input type="checkbox" name="deduplicate" {% if form_data.get('deduplicate', True) %}checked{% endif %}>
+                    <span class="checkbox-custom"></span>
+                    Deduplicate Results
+                </label>
+                <label class="checkbox-label">
+                    <input type="checkbox" name="download_images" {% if form_data.get('download_images') %}checked{% endif %}>
+                    <span class="checkbox-custom"></span>
+                    Download Images
+                </label>
+            </div>
+        </div>
+    </div>
+
+    <!-- Submit -->
+    <div class="form-actions">
+        <a href="{{ url_for('jobs.list_view') }}" class="btn btn-ghost">Cancel</a>
+        <button type="submit" class="btn btn-primary btn-glow btn-lg" id="submit-job">
+            <svg class="btn-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><polygon points="5 3 19 12 5 21 5 3"/></svg>
+            Launch Scrape Job
+        </button>
+    </div>
+</form>
+{% endblock %}

app/utils/__init__.py

@@ -0,0 +1,4 @@
+from .validators import validate_url, validate_job_data, sanitize_string
+from .logging_config import configure_logging
+
+__all__ = ["validate_url", "validate_job_data", "sanitize_string", "configure_logging"]

app/utils/logging_config.py

@@ -0,0 +1,60 @@
+"""
+Structured logging configuration.
+Sets up file + console handlers with proper formatting.
+"""
+from __future__ import annotations
+
+import logging
+import logging.handlers
+import sys
+from pathlib import Path
+
+
+def configure_logging(log_level: str = "INFO", log_dir: str = "logs") -> None:
+    """Configure root logger with rotating file and console handlers."""
+    log_path = Path(log_dir)
+    log_path.mkdir(parents=True, exist_ok=True)
+
+    level = getattr(logging, log_level.upper(), logging.INFO)
+    fmt = logging.Formatter(
+        "%(asctime)s [%(levelname)-8s] %(name)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    # Root logger
+    root = logging.getLogger()
+    root.setLevel(level)
+    root.handlers.clear()
+
+    # Console handler
+    console = logging.StreamHandler(sys.stdout)
+    console.setFormatter(fmt)
+    console.setLevel(level)
+    root.addHandler(console)
+
+    # Rotating file handler
+    file_handler = logging.handlers.RotatingFileHandler(
+        log_path / "app.log",
+        maxBytes=10 * 1024 * 1024,  # 10 MB
+        backupCount=5,
+        encoding="utf-8",
+    )
+    file_handler.setFormatter(fmt)
+    file_handler.setLevel(level)
+    root.addHandler(file_handler)
+
+    # Error-only file
+    error_handler = logging.handlers.RotatingFileHandler(
+        log_path / "errors.log",
+        maxBytes=5 * 1024 * 1024,
+        backupCount=3,
+        encoding="utf-8",
+    )
+    error_handler.setFormatter(fmt)
+    error_handler.setLevel(logging.ERROR)
+    root.addHandler(error_handler)
+
+    # Suppress noisy libs
+    logging.getLogger("urllib3").setLevel(logging.WARNING)
+    logging.getLogger("requests").setLevel(logging.WARNING)
+    logging.getLogger("playwright").setLevel(logging.WARNING)

app/utils/validators.py

@@ -0,0 +1,113 @@
+"""
+Input validation and sanitization utilities.
+All user-supplied strings pass through here before entering business logic.
+"""
+from __future__ import annotations
+
+import re
+from typing import Any, Optional
+from urllib.parse import urlparse
+
+import bleach
+
+
+# Allowed HTML tags for any rich-text fields (none for our use case)
+ALLOWED_TAGS: list[str] = []
+ALLOWED_ATTRS: dict = {}
+
+# Valid CSS selector pattern (permissive but blocks script injection)
+_CSS_UNSAFE_PATTERN = re.compile(r"[<>\"']|javascript:", re.IGNORECASE)
+_XPATH_UNSAFE_PATTERN = re.compile(r"[<>]|javascript:", re.IGNORECASE)
+
+
+def sanitize_string(value: Any, max_length: int = 500) -> str:
+    """Strip HTML tags and trim whitespace."""
+    if not value:
+        return ""
+    cleaned = bleach.clean(str(value), tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRS, strip=True)
+    return cleaned[:max_length].strip()
+
+
+def validate_url(url: str) -> tuple[bool, str]:
+    """Validate URL format. Returns (is_valid, error_message)."""
+    if not url:
+        return False, "URL is required."
+    url = url.strip()
+    try:
+        parsed = urlparse(url)
+        if parsed.scheme not in ("http", "https"):
+            return False, "URL must use http or https."
+        if not parsed.netloc:
+            return False, "URL must include a valid domain."
+        # Block localhost/private IPs (SSRF prevention)
+        host = parsed.hostname or ""
+        blocked = ["localhost", "127.0.0.1", "0.0.0.0", "::1"]
+        if host in blocked or host.startswith("192.168.") or host.startswith("10."):
+            return False, "Requests to internal addresses are not allowed."
+    except Exception:
+        return False, "Invalid URL format."
+    return True, ""
+
+
+def validate_css_selector(selector: Optional[str]) -> tuple[bool, str]:
+    if not selector:
+        return True, ""
+    if _CSS_UNSAFE_PATTERN.search(selector):
+        return False, "CSS selector contains unsafe characters."
+    return True, ""
+
+
+def validate_xpath(xpath: Optional[str]) -> tuple[bool, str]:
+    if not xpath:
+        return True, ""
+    if _XPATH_UNSAFE_PATTERN.search(xpath):
+        return False, "XPath contains unsafe characters."
+    return True, ""
+
+
+def validate_job_data(data: dict) -> tuple[bool, dict]:
+    """
+    Validate all fields for a scrape job.
+    Returns (is_valid, error_dict).
+    """
+    errors: dict[str, str] = {}
+
+    url = data.get("url", "")
+    valid, msg = validate_url(url)
+    if not valid:
+        errors["url"] = msg
+
+    css = data.get("css_selector")
+    valid, msg = validate_css_selector(css)
+    if not valid:
+        errors["css_selector"] = msg
+
+    xpath = data.get("xpath_selector")
+    valid, msg = validate_xpath(xpath)
+    if not valid:
+        errors["xpath_selector"] = msg
+
+    extraction_type = data.get("extraction_type", "text")
+    valid_extractions = ("text", "images", "links", "attributes", "table", "json_ld", "full_html")
+    if extraction_type not in valid_extractions:
+        errors["extraction_type"] = f"Must be one of: {', '.join(valid_extractions)}"
+
+    scrape_type = data.get("scrape_type", "static")
+    if scrape_type not in ("static", "dynamic"):
+        errors["scrape_type"] = "Must be 'static' or 'dynamic'."
+
+    try:
+        max_pages = int(data.get("max_pages", 1))
+        if not 1 <= max_pages <= 200:
+            errors["max_pages"] = "max_pages must be between 1 and 200."
+    except (TypeError, ValueError):
+        errors["max_pages"] = "max_pages must be an integer."
+
+    try:
+        delay = float(data.get("delay_seconds", 1.0))
+        if not 0 <= delay <= 60:
+            errors["delay_seconds"] = "delay_seconds must be between 0 and 60."
+    except (TypeError, ValueError):
+        errors["delay_seconds"] = "delay_seconds must be a number."
+
+    return len(errors) == 0, errors

database/scraper.db

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78dfc516cfcade3e0ab261153191e2958ef3ac2ec9589c8f8ac1606a414dc293
+size 114688

env.example

@@ -0,0 +1,30 @@
+# Application
+FLASK_ENV=development
+SECRET_KEY=dev-secret-key-change-in-production-use-32-chars
+DEBUG=True
+
+# Database
+DATABASE_URL=sqlite:///database/scraper.db
+
+# Security
+WTF_CSRF_ENABLED=True
+SESSION_COOKIE_SECURE=False
+SESSION_COOKIE_HTTPONLY=True
+SESSION_COOKIE_SAMESITE=Lax
+
+# Rate Limiting
+RATELIMIT_DEFAULT=100 per hour
+RATELIMIT_STORAGE_URL=memory://
+
+# Scraping
+MAX_CONCURRENT_JOBS=5
+REQUEST_TIMEOUT=30
+MAX_RETRIES=3
+DEFAULT_DELAY=1.0
+
+# Exports
+EXPORT_DIR=exports
+
+# Logging
+LOG_LEVEL=DEBUG
+LOG_DIR=logs

logs/app.log

@@ -0,0 +1,262 @@
+2026-05-17 14:44:10 [INFO    ] app: WebScraper Platform started [development]
+2026-05-17 14:44:10 [INFO    ] werkzeug: WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
+ * Running on http://127.0.0.1:5000
+2026-05-17 14:44:10 [INFO    ] werkzeug: Press CTRL+C to quit
+2026-05-17 14:44:10 [INFO    ] werkzeug:  * Restarting with watchdog (windowsapi)
+2026-05-17 14:44:11 [INFO    ] app: WebScraper Platform started [development]
+2026-05-17 14:44:11 [WARNING ] werkzeug:  * Debugger is active!
+2026-05-17 14:44:11 [INFO    ] werkzeug:  * Debugger PIN: 590-942-950
+2026-05-17 14:44:32 [DEBUG   ] app.middleware.security: GET / -> 200 (94.00ms)
+2026-05-17 14:44:32 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:44:32] "GET / HTTP/1.1" 200 -
+2026-05-17 14:44:32 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 200 (0.00ms)
+2026-05-17 14:44:32 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:44:32] "GET /static/css/style.css HTTP/1.1" 200 -
+2026-05-17 14:44:32 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 200 (0.00ms)
+2026-05-17 14:44:32 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:44:32] "GET /static/js/app.js HTTP/1.1" 200 -
+2026-05-17 14:44:32 [DEBUG   ] app.middleware.security: GET /favicon.ico -> 404 (0.00ms)
+2026-05-17 14:44:32 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:44:32] "GET /favicon.ico HTTP/1.1" 404 -
+2026-05-17 14:44:44 [DEBUG   ] app.middleware.security: GET /jobs/new -> 200 (94.00ms)
+2026-05-17 14:44:44 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:44:44] "GET /jobs/new HTTP/1.1" 200 -
+2026-05-17 14:44:44 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:44:44 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:44:44] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:44:44 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:44:44 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:44:44] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\__init__ (1).py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\__init__ (2).py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\__init__ (3).py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\__init__ (4).py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\__init__ (5).py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\__init__ (6).py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\__init__ (7).py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\__init__.py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\engine.py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\models.py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\job_service.py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\export_service.py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\security.py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\main.py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\jobs.py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\validators.py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\logging_config.py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\settings.py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\AppData\\Local\\Programs\\Python\\Python310\\Lib\\site-packages\\pygments\\plugin.py', reloading
+2026-05-17 14:45:18 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\AppData\\Local\\Programs\\Python\\Python310\\Lib\\site-packages\\dns\\rdtypes\\IN\\A.py', reloading
+2026-05-17 14:45:19 [INFO    ] werkzeug:  * Restarting with watchdog (windowsapi)
+2026-05-17 14:45:20 [INFO    ] app: WebScraper Platform started [development]
+2026-05-17 14:45:20 [WARNING ] werkzeug:  * Debugger is active!
+2026-05-17 14:45:20 [INFO    ] werkzeug:  * Debugger PIN: 590-942-950
+2026-05-17 14:46:31 [DEBUG   ] app.middleware.security: POST /jobs/new -> 302 (62.00ms)
+2026-05-17 14:46:31 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:46:31] "POST /jobs/new HTTP/1.1" 302 -
+2026-05-17 14:46:31 [DEBUG   ] app.middleware.security: GET /jobs/1 -> 200 (78.00ms)
+2026-05-17 14:46:31 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:46:31] "GET /jobs/1 HTTP/1.1" 200 -
+2026-05-17 14:46:31 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:46:31 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:46:31 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:46:31] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:46:31 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:46:31] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:46:34 [DEBUG   ] app.middleware.security: GET /jobs/api/jobs/1/status -> 200 (0.00ms)
+2026-05-17 14:46:34 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:46:34] "GET /jobs/api/jobs/1/status HTTP/1.1" 200 -
+2026-05-17 14:46:36 [DEBUG   ] app.middleware.security: GET /jobs/api/jobs/1/status -> 200 (0.00ms)
+2026-05-17 14:46:36 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:46:36] "GET /jobs/api/jobs/1/status HTTP/1.1" 200 -
+2026-05-17 14:46:38 [DEBUG   ] app.middleware.security: GET /jobs/api/jobs/1/status -> 200 (0.00ms)
+2026-05-17 14:46:38 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:46:38] "GET /jobs/api/jobs/1/status HTTP/1.1" 200 -
+2026-05-17 14:46:38 [DEBUG   ] app.middleware.security: GET /jobs/1 -> 200 (0.00ms)
+2026-05-17 14:46:38 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:46:38] "GET /jobs/1 HTTP/1.1" 200 -
+2026-05-17 14:46:38 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:46:38 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:46:38] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:46:38 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:46:38 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:46:38] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:47:34 [DEBUG   ] app.middleware.security: GET / -> 200 (62.00ms)
+2026-05-17 14:47:34 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:47:34] "GET / HTTP/1.1" 200 -
+2026-05-17 14:47:34 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:47:34 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:47:34] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:47:34 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:47:34 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:47:34] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:47:55 [DEBUG   ] app.middleware.security: GET /jobs/1 -> 200 (16.00ms)
+2026-05-17 14:47:55 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:47:55] "GET /jobs/1 HTTP/1.1" 200 -
+2026-05-17 14:47:55 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:47:55 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:47:55] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:47:55 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:47:55 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:47:55] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:48:14 [DEBUG   ] app.middleware.security: GET /jobs/new -> 200 (31.00ms)
+2026-05-17 14:48:14 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:14] "GET /jobs/new HTTP/1.1" 200 -
+2026-05-17 14:48:14 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:48:14 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:14] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:48:14 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:48:15 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:15] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:48:38 [DEBUG   ] app.middleware.security: POST /jobs/new -> 302 (31.00ms)
+2026-05-17 14:48:38 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:38] "POST /jobs/new HTTP/1.1" 302 -
+2026-05-17 14:48:38 [DEBUG   ] app.middleware.security: GET /jobs/2 -> 200 (79.00ms)
+2026-05-17 14:48:38 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:38] "GET /jobs/2 HTTP/1.1" 200 -
+2026-05-17 14:48:39 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:48:39 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:39] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:48:39 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:48:39 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:39] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:48:41 [DEBUG   ] app.middleware.security: GET /jobs/api/jobs/2/status -> 200 (15.00ms)
+2026-05-17 14:48:41 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:41] "GET /jobs/api/jobs/2/status HTTP/1.1" 200 -
+2026-05-17 14:48:43 [DEBUG   ] app.middleware.security: GET /jobs/api/jobs/2/status -> 200 (16.00ms)
+2026-05-17 14:48:43 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:43] "GET /jobs/api/jobs/2/status HTTP/1.1" 200 -
+2026-05-17 14:48:43 [DEBUG   ] app.middleware.security: GET /jobs/2 -> 200 (47.00ms)
+2026-05-17 14:48:43 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:43] "GET /jobs/2 HTTP/1.1" 200 -
+2026-05-17 14:48:43 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:48:43 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:43] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:48:43 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:48:43 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:43] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:48:53 [DEBUG   ] app.middleware.security: GET /jobs/new -> 200 (0.00ms)
+2026-05-17 14:48:53 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:53] "GET /jobs/new HTTP/1.1" 200 -
+2026-05-17 14:48:53 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:48:53 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:53] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:48:53 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:48:53 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:48:53] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:49:14 [DEBUG   ] app.middleware.security: POST /jobs/new -> 302 (32.00ms)
+2026-05-17 14:49:14 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:49:14] "POST /jobs/new HTTP/1.1" 302 -
+2026-05-17 14:49:14 [DEBUG   ] app.middleware.security: GET /jobs/3 -> 200 (47.00ms)
+2026-05-17 14:49:14 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:49:14] "GET /jobs/3 HTTP/1.1" 200 -
+2026-05-17 14:49:14 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:49:14 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:49:14] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:49:14 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:49:14 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:49:14] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:49:16 [DEBUG   ] app.middleware.security: GET /jobs/api/jobs/3/status -> 200 (0.00ms)
+2026-05-17 14:49:16 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:49:16] "GET /jobs/api/jobs/3/status HTTP/1.1" 200 -
+2026-05-17 14:49:16 [DEBUG   ] app.middleware.security: GET /jobs/3 -> 200 (47.00ms)
+2026-05-17 14:49:16 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:49:16] "GET /jobs/3 HTTP/1.1" 200 -
+2026-05-17 14:49:16 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (15.00ms)
+2026-05-17 14:49:16 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:49:16] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:49:16 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:49:16 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:49:16] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:50:47 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\app\\scrapers\\engine.py', reloading
+2026-05-17 14:50:47 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\Desktop\\webscraping\\app\\scrapers\\engine.py', reloading
+2026-05-17 14:50:47 [INFO    ] werkzeug:  * Restarting with watchdog (windowsapi)
+2026-05-17 14:50:48 [INFO    ] app: WebScraper Platform started [development]
+2026-05-17 14:50:48 [WARNING ] werkzeug:  * Debugger is active!
+2026-05-17 14:50:48 [INFO    ] werkzeug:  * Debugger PIN: 590-942-950
+2026-05-17 14:51:04 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\AppData\\Local\\Programs\\Python\\Python310\\Lib\\site-packages\\brotli.py', reloading
+2026-05-17 14:51:04 [INFO    ] werkzeug:  * Detected change in 'C:\\Users\\princ\\AppData\\Local\\Programs\\Python\\Python310\\Lib\\site-packages\\brotli.py', reloading
+2026-05-17 14:51:05 [INFO    ] werkzeug:  * Restarting with watchdog (windowsapi)
+2026-05-17 14:51:06 [INFO    ] app: WebScraper Platform started [development]
+2026-05-17 14:51:06 [WARNING ] werkzeug:  * Debugger is active!
+2026-05-17 14:51:06 [INFO    ] werkzeug:  * Debugger PIN: 590-942-950
+2026-05-17 14:52:00 [INFO    ] app: WebScraper Platform started [development]
+2026-05-17 14:52:00 [INFO    ] werkzeug: WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
+ * Running on http://127.0.0.1:5000
+2026-05-17 14:52:00 [INFO    ] werkzeug: Press CTRL+C to quit
+2026-05-17 14:52:00 [INFO    ] werkzeug:  * Restarting with watchdog (windowsapi)
+2026-05-17 14:52:01 [INFO    ] app: WebScraper Platform started [development]
+2026-05-17 14:52:01 [WARNING ] werkzeug:  * Debugger is active!
+2026-05-17 14:52:01 [INFO    ] werkzeug:  * Debugger PIN: 590-942-950
+2026-05-17 14:52:12 [DEBUG   ] app.middleware.security: GET /jobs/new -> 200 (31.00ms)
+2026-05-17 14:52:12 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:52:12] "GET /jobs/new HTTP/1.1" 200 -
+2026-05-17 14:52:12 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:52:12 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:52:12] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:52:12 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:52:12 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:52:12] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:53:26 [DEBUG   ] app.middleware.security: POST /jobs/new -> 302 (171.00ms)
+2026-05-17 14:53:26 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:53:26] "POST /jobs/new HTTP/1.1" 302 -
+2026-05-17 14:53:27 [DEBUG   ] app.middleware.security: GET /jobs/4 -> 200 (250.00ms)
+2026-05-17 14:53:27 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:53:27] "GET /jobs/4 HTTP/1.1" 200 -
+2026-05-17 14:53:27 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (16.00ms)
+2026-05-17 14:53:27 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:53:27] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:53:27 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:53:27 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:53:27] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:53:29 [DEBUG   ] app.middleware.security: GET /jobs/api/jobs/4/status -> 200 (15.00ms)
+2026-05-17 14:53:29 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:53:29] "GET /jobs/api/jobs/4/status HTTP/1.1" 200 -
+2026-05-17 14:53:29 [DEBUG   ] app.middleware.security: GET /jobs/4 -> 200 (15.00ms)
+2026-05-17 14:53:29 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:53:29] "GET /jobs/4 HTTP/1.1" 200 -
+2026-05-17 14:53:29 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:53:29 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:53:29] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:53:29 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (16.00ms)
+2026-05-17 14:53:29 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:53:29] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:53:31 [DEBUG   ] app.middleware.security: GET /jobs/api/jobs/4/status -> 200 (0.00ms)
+2026-05-17 14:53:31 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:53:31] "GET /jobs/api/jobs/4/status HTTP/1.1" 200 -
+2026-05-17 14:53:33 [DEBUG   ] app.middleware.security: GET /jobs/api/jobs/4/status -> 200 (15.00ms)
+2026-05-17 14:53:33 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:53:33] "GET /jobs/api/jobs/4/status HTTP/1.1" 200 -
+2026-05-17 14:53:33 [DEBUG   ] app.middleware.security: GET /jobs/4 -> 200 (16.00ms)
+2026-05-17 14:53:33 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:53:33] "GET /jobs/4 HTTP/1.1" 200 -
+2026-05-17 14:53:33 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (15.00ms)
+2026-05-17 14:53:33 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:53:33] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:53:33 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:53:33 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:53:33] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:57:40 [INFO    ] app: WebScraper Platform started [development]
+2026-05-17 14:57:40 [INFO    ] werkzeug: WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
+ * Running on http://127.0.0.1:5000
+2026-05-17 14:57:40 [INFO    ] werkzeug: Press CTRL+C to quit
+2026-05-17 14:57:40 [INFO    ] werkzeug:  * Restarting with watchdog (windowsapi)
+2026-05-17 14:57:41 [INFO    ] app: WebScraper Platform started [development]
+2026-05-17 14:57:41 [WARNING ] werkzeug:  * Debugger is active!
+2026-05-17 14:57:41 [INFO    ] werkzeug:  * Debugger PIN: 590-942-950
+2026-05-17 14:57:53 [DEBUG   ] app.middleware.security: GET / -> 200 (63.00ms)
+2026-05-17 14:57:53 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:57:53] "GET / HTTP/1.1" 200 -
+2026-05-17 14:57:53 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 200 (0.00ms)
+2026-05-17 14:57:53 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:57:53] "GET /static/css/style.css HTTP/1.1" 200 -
+2026-05-17 14:57:53 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 200 (0.00ms)
+2026-05-17 14:57:53 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:57:53] "GET /static/js/app.js HTTP/1.1" 200 -
+2026-05-17 14:57:54 [DEBUG   ] app.middleware.security: GET /favicon.ico -> 404 (15.00ms)
+2026-05-17 14:57:54 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:57:54] "GET /favicon.ico HTTP/1.1" 404 -
+2026-05-17 14:58:06 [DEBUG   ] app.middleware.security: GET /jobs/ -> 200 (47.00ms)
+2026-05-17 14:58:06 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:06] "GET /jobs/ HTTP/1.1" 200 -
+2026-05-17 14:58:06 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:58:06 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:06] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:58:06 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:58:06 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:06] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:58:10 [DEBUG   ] app.middleware.security: GET /jobs/new -> 200 (16.00ms)
+2026-05-17 14:58:10 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:10] "GET /jobs/new HTTP/1.1" 200 -
+2026-05-17 14:58:10 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:58:10 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:10] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:58:10 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:58:10 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:10] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:58:27 [DEBUG   ] app.middleware.security: GET / -> 200 (15.00ms)
+2026-05-17 14:58:27 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:27] "GET / HTTP/1.1" 200 -
+2026-05-17 14:58:27 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:58:27 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:58:27 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:27] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:58:27 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:27] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:58:32 [DEBUG   ] app.middleware.security: GET /jobs/ -> 200 (16.00ms)
+2026-05-17 14:58:32 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:32] "GET /jobs/ HTTP/1.1" 200 -
+2026-05-17 14:58:32 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:58:32 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:58:32 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:32] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:58:32 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:32] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:58:34 [DEBUG   ] app.middleware.security: GET /jobs/new -> 200 (0.00ms)
+2026-05-17 14:58:34 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:34] "GET /jobs/new HTTP/1.1" 200 -
+2026-05-17 14:58:34 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:58:34 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:34] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:58:34 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:58:34 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:34] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:58:36 [DEBUG   ] app.middleware.security: GET / -> 200 (0.00ms)
+2026-05-17 14:58:36 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:36] "GET / HTTP/1.1" 200 -
+2026-05-17 14:58:36 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:58:36 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:36] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:58:36 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:58:36 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:36] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:58:38 [DEBUG   ] app.middleware.security: GET /jobs/new -> 200 (0.00ms)
+2026-05-17 14:58:38 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:38] "GET /jobs/new HTTP/1.1" 200 -
+2026-05-17 14:58:38 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:58:38 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:38] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:58:38 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:58:38 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:38] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:58:49 [DEBUG   ] app.middleware.security: GET /jobs/new -> 200 (0.00ms)
+2026-05-17 14:58:49 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:49] "GET /jobs/new HTTP/1.1" 200 -
+2026-05-17 14:58:49 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:58:49 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:58:49 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:49] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:58:49 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:58:49] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:59:09 [DEBUG   ] app.middleware.security: POST /jobs/new -> 302 (31.00ms)
+2026-05-17 14:59:09 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:59:09] "POST /jobs/new HTTP/1.1" 302 -
+2026-05-17 14:59:09 [DEBUG   ] app.middleware.security: GET /jobs/5 -> 200 (78.00ms)
+2026-05-17 14:59:09 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:59:09] "GET /jobs/5 HTTP/1.1" 200 -
+2026-05-17 14:59:09 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:59:09 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:59:09] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:59:09 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:59:09 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:59:09] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:59:11 [DEBUG   ] app.middleware.security: GET /jobs/api/jobs/5/status -> 200 (16.00ms)
+2026-05-17 14:59:11 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:59:11] "GET /jobs/api/jobs/5/status HTTP/1.1" 200 -
+2026-05-17 14:59:11 [DEBUG   ] app.middleware.security: GET /jobs/5 -> 200 (15.00ms)
+2026-05-17 14:59:11 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:59:11] "GET /jobs/5 HTTP/1.1" 200 -
+2026-05-17 14:59:11 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (16.00ms)
+2026-05-17 14:59:11 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:59:11] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:59:11 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:59:11 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:59:11] "GET /static/js/app.js HTTP/1.1" 304 -
+2026-05-17 14:59:42 [DEBUG   ] app.middleware.security: GET / -> 200 (16.00ms)
+2026-05-17 14:59:42 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:59:42] "GET / HTTP/1.1" 200 -
+2026-05-17 14:59:42 [DEBUG   ] app.middleware.security: GET /static/css/style.css -> 304 (0.00ms)
+2026-05-17 14:59:42 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:59:42] "GET /static/css/style.css HTTP/1.1" 304 -
+2026-05-17 14:59:42 [DEBUG   ] app.middleware.security: GET /static/js/app.js -> 304 (0.00ms)
+2026-05-17 14:59:42 [INFO    ] werkzeug: 127.0.0.1 - - [17/May/2026 14:59:42] "GET /static/js/app.js HTTP/1.1" 304 -