crawler_fastapi.py

from fastapi import FastAPI, HTTPException, BackgroundTasks
from pydantic import HttpUrl, BaseModel
from typing import List, Optional
import requests
import time
from dotenv import load_dotenv
import os

load_dotenv()

CLOUDFLARE_ACCOUNT_ID = os.getenv("CLOUDFLARE_ACCOUNT_ID")
CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN")

app = FastAPI(title="Cloudflare Crawl Proxy for Startups")


BASE_URL = f"https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/browser-rendering/crawl"

headers = {
    "Authorization": f"Bearer {CLOUDFLARE_TOKEN}",
    "Content-Type": "application/json"
}

# Modèle de données pour l'entrée
class CrawlRequest(BaseModel):
    url: HttpUrl
    limit: Optional[int] = 3
    render: Optional[bool] = True
    formats: Optional[List[str]] = ["markdown"]

# --- ENDPOINTS ---

@app.post("/launch-crawl")
async def launch_crawl(request: CrawlRequest):
    """Lance la mission de crawl et retourne le Job ID immédiatement."""
    payload = {
        "url": str(request.url),
        "render": request.render,
        "limit": request.limit,
        "formats": request.formats
    }
    
    response = requests.post(BASE_URL, headers=headers, json=payload)
    
    if response.status_code != 200:
        raise HTTPException(status_code=response.status_code, detail=response.json())
    
    return response.json()

@app.get("/results/{job_id}")
async def get_results(job_id: str):
    """Récupère les résultats du crawl via le Job ID."""
    result_url = f"{BASE_URL}/{job_id}"
    response = requests.get(result_url, headers=headers)
    
    if response.status_code != 200:
        raise HTTPException(status_code=response.status_code, detail="Job non trouvé ou erreur Cloudflare")
    
    data = response.json()
    
    # On vérifie si les données sont prêtes
    if data.get("result") is None:
        return {"status": "pending", "message": "Le scan est encore en cours de traitement..."}
    
    return data

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=8000)