import html
import re
import sys
from urllib.parse import urlencode
from APIs.api_types import GamespotType
BASE = "https://www.gamespot.com"
class ChallengeError(RuntimeError):
"""Raised when Cloudflare returns a bot-challenge instead of JSON."""
def _looks_like_challenge(status, body):
return status in (403, 503) or "Just a moment" in (body or "")[:1000]
class _CffiFetcher:
def __init__(self):
from curl_cffi import requests as creq
self.session = creq.Session(impersonate="chrome")
self.session.headers.update({"Accept": "application/json", "Referer": f"{BASE}/"})
def get_json(self, url, **params):
r = self.session.get(url, params=params, timeout=30)
if _looks_like_challenge(r.status_code, r.text):
raise ChallengeError(f"Cloudflare challenge (HTTP {r.status_code}) on {url}")
r.raise_for_status()
return r.json()
def close(self):
try:
self.session.close()
except Exception:
pass
class _BrowserFetcher:
def __init__(self):
from playwright.sync_api import sync_playwright
self._pw = sync_playwright().start()
self.browser = self._pw.chromium.launch(headless=True)
self.page = self.browser.new_page()
self.page.goto(f"{BASE}/", wait_until="domcontentloaded", timeout=60000)
self._wait_past_challenge()
def _wait_past_challenge(self):
for _ in range(30):
try:
if "Just a moment" not in self.page.title():
return
except Exception:
pass
self.page.wait_for_timeout(1000)
def get_json(self, url, **params):
import json
if params:
sep = "&" if "?" in url else "?"
url = f"{url}{sep}{urlencode(params)}"
result = self.page.evaluate(
"""async (u) => {
const r = await fetch(u, {headers: {'Accept': 'application/json'}});
return {status: r.status, body: await r.text()};
}""",
url,
)
status, body = result["status"], result["body"]
if _looks_like_challenge(status, body):
raise ChallengeError(f"Cloudflare still blocking (HTTP {status}).")
if status != 200:
raise RuntimeError(f"HTTP {status} on {url}\n{body[:200]}")
return json.loads(body)
def close(self):
try:
self.browser.close()
self._pw.stop()
except Exception:
pass
class Gamespot:
def __init__(self):
self._fetcher = self._make_fetcher()
def _make_fetcher(self):
try:
return _CffiFetcher()
except ImportError:
pass
try:
return _BrowserFetcher()
except ImportError:
raise ImportError(
"Neither curl_cffi nor Playwright is installed.\n"
" pip install curl_cffi\n"
" # or: pip install playwright && playwright install chromium"
)
def close(self):
try:
self._fetcher.close()
except Exception:
pass
def _get_json(self, url, **params):
try:
return self._fetcher.get_json(url, **params)
except ChallengeError:
if isinstance(self._fetcher, _BrowserFetcher):
raise
print("curl_cffi got a Cloudflare challenge; falling back to Playwright.", file=sys.stderr)
self._fetcher.close()
try:
self._fetcher = _BrowserFetcher()
except ImportError:
raise ImportError(
"Cloudflare blocked and Playwright isn't installed.\n"
" pip install playwright && playwright install chromium"
)
return self._fetcher.get_json(url, **params)
@staticmethod
def _clean(html_text):
return html.unescape(re.sub(r"<[^>]+>", "", html_text or "")).strip()
def _find_games(self, name, max_n):
# Ask for more results than needed so filtering to subtype='games' still yields max_n
results = self._get_json(
f"{BASE}/wp-json/wp/v2/search",
search=name,
per_page=max(10, max_n),
)
return [r for r in results if r.get("subtype") == "games"][:max_n]
def _fetch_game(self, search_item):
return self._get_json(search_item["_links"]["self"][0]["href"])
def _resolve_terms(self, game):
out = {}
for link in game.get("_links", {}).get("wp:term", []):
tax = link.get("taxonomy")
if tax == "author":
continue
try:
terms = self._get_json(link["href"], per_page=100)
out[tax] = [t["name"] for t in terms if t.get("name")]
except (ChallengeError, RuntimeError):
names = []
for tid in game.get(tax, []):
try:
t = self._get_json(f"{BASE}/wp-json/wp/v2/{tax}/{tid}")
if t.get("name"):
names.append(t["name"])
except RuntimeError:
pass
out[tax] = names
return out
def _fetch_media(self, game_id):
try:
items = self._get_json(
f"{BASE}/wp-json/wp/v2/media", parent=game_id, per_page=100
)
return [m["source_url"] for m in items if m.get("source_url")]
except RuntimeError:
return []
def search(self, game_name: str, max_n: int = 1) -> list[GamespotType]:
hits = self._find_games(game_name, max_n)
results = []
for hit in hits:
try:
game = self._fetch_game(hit)
taxonomies = self._resolve_terms(game)
media_urls = self._fetch_media(game["id"])
acf = game.get("acf") or {}
# Fall back to featured_images thumbnail when media endpoint returns nothing
if not media_urls:
fi = game.get("featured_images") or {}
if fi.get("2x"):
media_urls = [fi["2x"]]
# Franchise names + ESRB content descriptors as keywords
keywords = (
taxonomies.get("game_franchise", [])
+ taxonomies.get("esrb_descriptors", [])
)
results.append(GamespotType(
name=self._clean(game.get("title", {}).get("rendered", "")),
release=acf.get("gamepost_release_date", ""),
description=self._clean(game.get("content", {}).get("rendered", "")),
genres=taxonomies.get("genre", []),
platforms=taxonomies.get("platform", []),
themes=taxonomies.get("theme", []),
cover_url=media_urls,
keywords=keywords,
))
except Exception as e:
print(f"Gamespot: skipping '{hit.get('title')}': {e}", file=sys.stderr)
return results