fix: scrape public Trustpilot page instead of blocked API

app.py

@@ -1,7 +1,9 @@
-from fastapi import FastAPI, Response
+from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 import httpx
-import os
+import json
+import asyncio
+from datetime import datetime, timedelta
 
 app = FastAPI()
 
@@ -12,31 +14,55 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-API_KEY = os.environ.get("TRUSTPILOT_API_KEY")
-BUSINESS_UNIT_ID = os.environ.get("TRUSTPILOT_BUSINESS_UNIT_ID")
-BASE_URL = "https://api.trustpilot.com/v1"
+TRUSTPILOT_URL = "https://www.trustpilot.com/review/collectionsdarchitectes.fr"
+USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
 
+cache = {"data": None, "expires": None}
+
+
+async def fetch_reviews():
+    now = datetime.utcnow()
+    if cache["data"] and cache["expires"] and now < cache["expires"]:
+        return cache["data"]
 
-@app.get("/business")
-async def get_business():
     async with httpx.AsyncClient() as client:
-        r = await client.get(
-            f"{BASE_URL}/business-units/{BUSINESS_UNIT_ID}",
-            params={"apikey": API_KEY},
-        )
-    return Response(content=r.content, media_type="application/json", status_code=r.status_code)
+        r = await client.get(TRUSTPILOT_URL, headers={"User-Agent": USER_AGENT}, follow_redirects=True)
+
+    html = r.text
+    marker = '__NEXT_DATA__'
+    start = html.find(marker)
+    if start == -1:
+        return cache.get("data") or {"error": "Could not parse Trustpilot page"}
+
+    start = html.find('>', start) + 1
+    end = html.find('</script>', start)
+    raw = html[start:end]
+
+    data = json.loads(raw)
+    props = data.get("props", {}).get("pageProps", {})
+    biz = props.get("businessUnit", {})
+    reviews = props.get("reviews", [])
+
+    result = {
+        "score": biz.get("trustScore"),
+        "total": biz.get("numberOfReviews"),
+        "reviews": [
+            {
+                "stars": r.get("rating"),
+                "title": r.get("title"),
+                "text": r.get("text"),
+                "author": r.get("consumer", {}).get("displayName"),
+                "date": r.get("dates", {}).get("publishedDate"),
+            }
+            for r in reviews
+        ],
+    }
+
+    cache["data"] = result
+    cache["expires"] = now + timedelta(hours=1)
+    return result
 
 
 @app.get("/reviews")
-async def get_reviews(perPage: int = 6, stars: str = "4,5"):
-    async with httpx.AsyncClient() as client:
-        r = await client.get(
-            f"{BASE_URL}/business-units/{BUSINESS_UNIT_ID}/reviews",
-            params={
-                "apikey": API_KEY,
-                "perPage": perPage,
-                "orderBy": "createdat.desc",
-                "stars": stars,
-            },
-        )
-    return Response(content=r.content, media_type="application/json", status_code=r.status_code)
+async def get_reviews():
+    return await fetch_reviews()