crawler.py

import asyncio
import logging
from scraper.async_crawler import AsyncCrawler

logging.basicConfig(
  level=logging.INFO,
  format='%(asctime)s - %(levelname)s - %(message)s'
)

async def main():
  try:
    start_url = "https://shopify.dev/docs/apps/build/flow"
    crawler = AsyncCrawler(start_url)
    await crawler.run()
  except Exception as e:
    logging.error(f"Crawler failed: {str(e)}")
    raise

if __name__ == "__main__":
  asyncio.run(main())