(feat): add script to fetch data from website

scripts/ingest.py

@@ -0,0 +1,85 @@
+import asyncio
+import os
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+
+async def main():
+    urls = [
+        "https://www.locus.com.np/",
+        "https://www.locus.com.np/about-us",
+        "https://www.locus.com.np/events",
+        "https://www.locus.com.np/zerone",
+        "https://www.locus.com.np/sponsors",
+        "https://www.locus.com.np/blogs",
+        "https://www.locus.com.np/teams",
+        "https://www.locus.com.np/contact-us",
+        "https://www.locus.com.np/event/HARDWARE-FELLOWSHIP",
+        "https://www.locus.com.np/past-locus",
+        "https://www.locus.com.np/event/CYBER%20SHIELD",
+        "https://www.locus.com.np/event/GIRLS%20LOCUS%20CUP",
+        "https://www.locus.com.np/event/HACK-A-WEEK",
+        "https://www.locus.com.np/event/CODE-JAM",
+        "https://www.locus.com.np/event/GIRLS-TO-CODE",
+        "https://www.locus.com.np/event/15%20DAYS%20OF%20LEARNING",
+        "https://www.locus.com.np/event/DATAVERSE",
+        "https://www.locus.com.np/event/Walkathon",
+        "https://www.locus.com.np/event/Flashmob",
+        "https://www.locus.com.np/event/Energy%20Hackathon",
+        "https://www.locus.com.np/event/LOCUS%20Exhibition",
+        "https://www.locus.com.np/event/Dronacharya%202026",
+        "https://www.locus.com.np/event/RoboWarz%202026",
+        "https://www.locus.com.np/event/RoboSoccer%202026",
+        "https://www.locus.com.np/event/RoboPop%202026",
+        "https://www.locus.com.np/event/Robo%20Line%20Dash%202026",
+        # Important External Links
+        "https://medium.com/zerone-magazine/tagged/blogging-competition",
+        "https://medium.com/zerone-magazine/tagged/technical-competition",
+        "https://drive.google.com/file/u/2/d/1S99bX6EzDScZ2Is4MLLpNWGO4suq-fwj/view?usp=sharing",
+        "https://drive.google.com/file/d/1RRV3RALnP9BzUy1vq5KzS0HeH2XGH6HP/view?usp=sharing",
+        "https://drive.google.com/drive/folders/14UPc7jaazIt-E7RBRF6IJ9LmFrwzTRjc",
+        "https://drive.google.com/file/d/1pJMHmXZUcOCYBG5xt2Do5OMD9xEB7yD5/view",
+        "https://drive.google.com/file/d/1mm78B6Hc3oLw3IrT1R6fMyI9C8di5ga5/view",
+        "https://drive.google.com/file/d/1wn-QI5akpAgEA2vs8P9jeJDVxRbfNhPm/view?usp=drive_link",
+        "https://drive.google.com/file/d/1X-QENlLrFw0GM8NIlmTP4hsKYESjeRbh/view?usp=drive_link",
+        "https://drive.google.com/file/d/1b8MlmHNM_0GeYJhxAajb8YC22euZwyZQ/view?usp=drive_link",
+        "https://drive.google.com/file/d/14UPRPobB6AXR7YmS1570rtIHgWdVpyDY/view"
+    ]
+    
+    os.makedirs("data", exist_ok=True)
+    
+    browser_config = BrowserConfig(
+        headless=True,
+        extra_args=["--disable-gpu", "--disable-software-rasterizer", "--no-sandbox"]
+    )
+    
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        wait_for="css:body", 
+        delay_before_return_html=10.0, 
+        page_timeout=120000 
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        for url in urls:
+            print(f"Crawling {url}...")
+            try:
+                result = await crawler.arun(url=url, config=run_config)
+                
+                if result.success:
+                    # Create a safe filename from the URL
+                    filename = url.replace("https://", "").replace("www.", "").replace("/", "_").strip("_")
+                    if not filename or filename == "locus.com.np":
+                        filename = "index"
+                    else:
+                        filename = filename.replace("locus.com.np_", "")
+                    
+                    filepath = f"data/{filename}.md"
+                    with open(filepath, "w", encoding="utf-8") as f:
+                        f.write(result.markdown)
+                    print(f"Saved to {filepath}")
+                else:
+                    print(f"Failed to crawl {url}: {result.error_message}")
+            except Exception as e:
+                print(f"Exception during crawl of {url}: {str(e)}")
+
+if __name__ == "__main__":
+    asyncio.run(main())