File size: 5,066 Bytes
7f1ccfb 02544a0 7f1ccfb 02544a0 7f1ccfb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | import json, asyncio, uuid, httpx
from datetime import datetime
from fastapi import FastAPI, Query
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import HTMLResponse, JSONResponse
from typing import Optional
from crawler import crawl_url
app = FastAPI(title="Juskeo Crawler", version="3.0.0")
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
tasks = {}
@app.get("/", response_class=HTMLResponse)
async def root(
url: Optional[str] = Query(None),
webhook: Optional[str] = Query(None),
username: Optional[str] = Query(None),
):
if url and webhook and username:
task_id = str(uuid.uuid4())[:8]
tasks[task_id] = {
"status": "queued", "url": url, "progress": 0,
"pages_crawled": 0, "username": username, "webhook": webhook,
}
asyncio.create_task(run_crawl(task_id, url, webhook, username))
return HTMLResponse(f"""
<!DOCTYPE html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Crawling…</title>
<style>
*{{margin:0;padding:0;box-sizing:border-box}}
body{{font-family:sans-serif;background:#FFFDF5;min-height:100vh;display:flex;align-items:center;justify-content:center}}
.card{{background:#fff;border:4px solid #000;box-shadow:8px 8px 0 #000;padding:2.5rem;max-width:480px;width:90%;text-align:center}}
h1{{font-size:1.5rem;margin-bottom:.5rem}}
p{{color:#666;font-size:.875rem;margin-bottom:2rem}}
.spinner{{width:48px;height:48px;border:4px solid #000;border-top-color:#CCFF00;border-radius:50%;animation:spin .8s linear infinite;margin:0 auto 1.5rem}}
@keyframes spin{{to{{transform:rotate(360deg)}}}}
.bar{{width:100%;height:12px;background:#eee;border:2px solid #000;margin:1rem 0;overflow:hidden}}
.bar-fill{{height:100%;background:#CCFF00;width:0%;transition:width .5s}}
.info{{font-size:.75rem;color:#999;margin-top:.5rem}}
a{{display:inline-block;margin-top:1.5rem;padding:.5rem 1.5rem;border:2px solid #000;color:#000;text-decoration:none;font-weight:700;font-size:.875rem}}
</style></head><body>
<div class="card">
<div class="spinner"></div>
<h1>Deep Crawling…</h1>
<p id="url">{url}</p>
<div class="bar"><div class="bar-fill" id="progress"></div></div>
<p class="info"><span id="pages">0</span> pages crawled</p>
<a href="/" id="doneLink" style="display:none">View Results</a>
</div>
<script>
let t="{task_id}";
async function poll(){{
try{{
let r=await fetch("/status?task_id="+t);
let d=await r.json();
document.getElementById("progress").style.width=d.progress+"%";
document.getElementById("pages").textContent=d.pages_crawled;
if(d.status==="completed"){{document.getElementById("doneLink").style.display="inline-block";document.querySelector(".spinner").style.display="none";document.querySelector("h1").textContent="Complete!";}}
else if(d.status!=="error") setTimeout(poll,2000);
}}catch(e){{setTimeout(poll,3000);}}
}}
setTimeout(poll,2000);
</script></body></html>""")
return JSONResponse({"status": "idle", "message": "Juskeo Crawler ready"})
@app.get("/status")
async def status(task_id: str = Query(...)):
t = tasks.get(task_id)
if not t:
return JSONResponse({"status": "error", "message": "Task not found"}, 404)
return {
"status": t["status"],
"progress": t["progress"],
"pages_crawled": t["pages_crawled"],
"error": t.get("error"),
}
@app.get("/result")
async def result(task_id: str = Query(...)):
t = tasks.get(task_id)
if not t:
return JSONResponse({"status": "error", "message": "Task not found"}, 404)
if t["status"] not in ("completed", "error"):
return JSONResponse({"status": "pending", "message": "Crawl still running"})
return t.get("result", {"status": "no_result"})
async def run_crawl(task_id, url, webhook, username):
t = tasks[task_id]
t["status"] = "running"
t["started_at"] = datetime.utcnow().isoformat()
def progress(pct, pages):
t["progress"] = pct
t["pages_crawled"] = pages
try:
result = await crawl_url(url, progress_callback=progress)
t["status"] = "completed"
t["progress"] = 100
t["completed_at"] = datetime.utcnow().isoformat()
t["result"] = result
# Send results to webhook
payload = {"username": username, "result": result}
async with httpx.AsyncClient(timeout=30) as client:
try:
await client.post(webhook, json=payload)
except Exception:
pass # webhook fire-and-forget
except Exception as e:
t["status"] = "error"
t["error"] = str(e)
|