crawler/__init__.py

from .config import (
    MAX_SHARD_ROWS,
    MAX_SHARDS,
    NORMAL_TOTAL_WORKERS,
    SUPER_TOTAL_WORKERS,
    CrawlerConfig,
    compute_worker_split,
)
from .engine import AsyncCrawler

__all__ = [
    "AsyncCrawler",
    "CrawlerConfig",
    "MAX_SHARD_ROWS",
    "MAX_SHARDS",
    "NORMAL_TOTAL_WORKERS",
    "SUPER_TOTAL_WORKERS",
    "compute_worker_split",
]