| import json, asyncio, uuid, httpx |
| from datetime import datetime |
| from fastapi import FastAPI, Query |
| from fastapi.middleware.cors import CORSMiddleware |
| from fastapi.responses import HTMLResponse, JSONResponse |
| from typing import Optional |
|
|
| from crawler import crawl_url |
|
|
| app = FastAPI(title="Juskeo Crawler", version="3.0.0") |
| app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]) |
|
|
| tasks = {} |
|
|
| @app.get("/", response_class=HTMLResponse) |
| async def root( |
| url: Optional[str] = Query(None), |
| webhook: Optional[str] = Query(None), |
| username: Optional[str] = Query(None), |
| ): |
| if url and webhook and username: |
| task_id = str(uuid.uuid4())[:8] |
| tasks[task_id] = { |
| "status": "queued", "url": url, "progress": 0, |
| "pages_crawled": 0, "username": username, "webhook": webhook, |
| } |
| asyncio.create_task(run_crawl(task_id, url, webhook, username)) |
| return HTMLResponse(f""" |
| <!DOCTYPE html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"> |
| <title>Crawling…</title> |
| <style> |
| *{{margin:0;padding:0;box-sizing:border-box}} |
| body{{font-family:sans-serif;background:#FFFDF5;min-height:100vh;display:flex;align-items:center;justify-content:center}} |
| .card{{background:#fff;border:4px solid #000;box-shadow:8px 8px 0 #000;padding:2.5rem;max-width:480px;width:90%;text-align:center}} |
| h1{{font-size:1.5rem;margin-bottom:.5rem}} |
| p{{color:#666;font-size:.875rem;margin-bottom:2rem}} |
| .spinner{{width:48px;height:48px;border:4px solid #000;border-top-color:#CCFF00;border-radius:50%;animation:spin .8s linear infinite;margin:0 auto 1.5rem}} |
| @keyframes spin{{to{{transform:rotate(360deg)}}}} |
| .bar{{width:100%;height:12px;background:#eee;border:2px solid #000;margin:1rem 0;overflow:hidden}} |
| .bar-fill{{height:100%;background:#CCFF00;width:0%;transition:width .5s}} |
| .info{{font-size:.75rem;color:#999;margin-top:.5rem}} |
| a{{display:inline-block;margin-top:1.5rem;padding:.5rem 1.5rem;border:2px solid #000;color:#000;text-decoration:none;font-weight:700;font-size:.875rem}} |
| </style></head><body> |
| <div class="card"> |
| <div class="spinner"></div> |
| <h1>Deep Crawling…</h1> |
| <p id="url">{url}</p> |
| <div class="bar"><div class="bar-fill" id="progress"></div></div> |
| <p class="info"><span id="pages">0</span> pages crawled</p> |
| <a href="/" id="doneLink" style="display:none">View Results</a> |
| </div> |
| <script> |
| let t="{task_id}"; |
| async function poll(){{ |
| try{{ |
| let r=await fetch("/status?task_id="+t); |
| let d=await r.json(); |
| document.getElementById("progress").style.width=d.progress+"%"; |
| document.getElementById("pages").textContent=d.pages_crawled; |
| if(d.status==="completed"){{document.getElementById("doneLink").style.display="inline-block";document.querySelector(".spinner").style.display="none";document.querySelector("h1").textContent="Complete!";}} |
| else if(d.status!=="error") setTimeout(poll,2000); |
| }}catch(e){{setTimeout(poll,3000);}} |
| }} |
| setTimeout(poll,2000); |
| </script></body></html>""") |
|
|
| return JSONResponse({"status": "idle", "message": "Juskeo Crawler ready"}) |
|
|
| @app.get("/status") |
| async def status(task_id: str = Query(...)): |
| t = tasks.get(task_id) |
| if not t: |
| return JSONResponse({"status": "error", "message": "Task not found"}, 404) |
| return { |
| "status": t["status"], |
| "progress": t["progress"], |
| "pages_crawled": t["pages_crawled"], |
| "error": t.get("error"), |
| } |
|
|
| @app.get("/result") |
| async def result(task_id: str = Query(...)): |
| t = tasks.get(task_id) |
| if not t: |
| return JSONResponse({"status": "error", "message": "Task not found"}, 404) |
| if t["status"] not in ("completed", "error"): |
| return JSONResponse({"status": "pending", "message": "Crawl still running"}) |
| return t.get("result", {"status": "no_result"}) |
|
|
| async def run_crawl(task_id, url, webhook, username): |
| t = tasks[task_id] |
| t["status"] = "running" |
| t["started_at"] = datetime.utcnow().isoformat() |
|
|
| def progress(pct, pages): |
| t["progress"] = pct |
| t["pages_crawled"] = pages |
|
|
| try: |
| result = await crawl_url(url, progress_callback=progress) |
| t["status"] = "completed" |
| t["progress"] = 100 |
| t["completed_at"] = datetime.utcnow().isoformat() |
| t["result"] = result |
|
|
| |
| payload = {"username": username, "result": result} |
| async with httpx.AsyncClient(timeout=30) as client: |
| try: |
| await client.post(webhook, json=payload) |
| except Exception: |
| pass |
| except Exception as e: |
| t["status"] = "error" |
| t["error"] = str(e) |
|
|
