import json, asyncio, uuid, httpx
from datetime import datetime
from fastapi import FastAPI, Query
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import HTMLResponse, JSONResponse
from typing import Optional
from crawler import crawl_url
app = FastAPI(title="Juskeo Crawler", version="3.0.0")
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
tasks = {}
@app.get("/", response_class=HTMLResponse)
async def root(
url: Optional[str] = Query(None),
webhook: Optional[str] = Query(None),
username: Optional[str] = Query(None),
):
if url and webhook and username:
task_id = str(uuid.uuid4())[:8]
tasks[task_id] = {
"status": "queued", "url": url, "progress": 0,
"pages_crawled": 0, "username": username, "webhook": webhook,
}
asyncio.create_task(run_crawl(task_id, url, webhook, username))
return HTMLResponse(f"""
Crawling…
""")
return JSONResponse({"status": "idle", "message": "Juskeo Crawler ready"})
@app.get("/status")
async def status(task_id: str = Query(...)):
t = tasks.get(task_id)
if not t:
return JSONResponse({"status": "error", "message": "Task not found"}, 404)
return {
"status": t["status"],
"progress": t["progress"],
"pages_crawled": t["pages_crawled"],
"error": t.get("error"),
}
@app.get("/result")
async def result(task_id: str = Query(...)):
t = tasks.get(task_id)
if not t:
return JSONResponse({"status": "error", "message": "Task not found"}, 404)
if t["status"] not in ("completed", "error"):
return JSONResponse({"status": "pending", "message": "Crawl still running"})
return t.get("result", {"status": "no_result"})
async def run_crawl(task_id, url, webhook, username):
t = tasks[task_id]
t["status"] = "running"
t["started_at"] = datetime.utcnow().isoformat()
def progress(pct, pages):
t["progress"] = pct
t["pages_crawled"] = pages
try:
result = await crawl_url(url, progress_callback=progress)
t["status"] = "completed"
t["progress"] = 100
t["completed_at"] = datetime.utcnow().isoformat()
t["result"] = result
# Send results to webhook
payload = {"username": username, "result": result}
async with httpx.AsyncClient(timeout=30) as client:
try:
await client.post(webhook, json=payload)
except Exception:
pass # webhook fire-and-forget
except Exception as e:
t["status"] = "error"
t["error"] = str(e)