fix(ingest): add User-Agent and headers to httpx requests to fix Gallica 403

Gallica (BnF) rejects automated requests without a valid User-Agent.
Added _HEADERS to iiif_fetcher.py (image download) and _MANIFEST_HEADERS to
ingest.py (_fetch_json_manifest) with:
- User-Agent: Mozilla/5.0 (compatible; ScriptoriumAI/1.0; ...)
- Accept: appropriate MIME types for each request type
- Referer: https://gallica.bnf.fr/

Updated test_image_pipeline.py to assert the headers are passed in the mock call.

https://claude.ai/code/session_018woyEHc8HG2th7V4ewJ4Kg

backend/app/api/v1/ingest.py

@@ -112,10 +112,20 @@ async def _create_page(
     return page
 
 
+_MANIFEST_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (compatible; ScriptoriumAI/1.0; "
+        "+https://huggingface.co/spaces/Ma-Ri-Ba-Ku/scriptorium-ai)"
+    ),
+    "Accept": "application/ld+json,application/json,*/*",
+    "Referer": "https://gallica.bnf.fr/",
+}
+
+
 async def _fetch_json_manifest(url: str) -> dict:
     """Télécharge un manifest IIIF. Fonction isolée pour faciliter les tests."""
     async with httpx.AsyncClient() as client:
-        resp = await client.get(url, follow_redirects=True, timeout=30.0)
+        resp = await client.get(url, headers=_MANIFEST_HEADERS, follow_redirects=True, timeout=30.0)
         resp.raise_for_status()
         return resp.json()
 

backend/app/services/ingest/iiif_fetcher.py

@@ -11,6 +11,15 @@ logger = logging.getLogger(__name__)
 
 _DEFAULT_TIMEOUT = 60.0  # secondes — les images IIIF haute résolution peuvent être lourdes
 
+_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (compatible; ScriptoriumAI/1.0; "
+        "+https://huggingface.co/spaces/Ma-Ri-Ba-Ku/scriptorium-ai)"
+    ),
+    "Accept": "image/jpeg,image/png,image/*,*/*",
+    "Referer": "https://gallica.bnf.fr/",
+}
+
 
 def fetch_iiif_image(url: str, timeout: float = _DEFAULT_TIMEOUT) -> bytes:
     """Télécharge une image depuis une URL IIIF complète.
@@ -28,7 +37,7 @@ def fetch_iiif_image(url: str, timeout: float = _DEFAULT_TIMEOUT) -> bytes:
         httpx.RequestError: pour toute autre erreur réseau.
     """
     logger.info("Fetching IIIF image", extra={"url": url})
-    response = httpx.get(url, follow_redirects=True, timeout=timeout)
+    response = httpx.get(url, headers=_HEADERS, follow_redirects=True, timeout=timeout)
     response.raise_for_status()
     logger.info(
         "IIIF image fetched",

backend/tests/test_image_pipeline.py

@@ -272,6 +272,14 @@ def test_fetch_iiif_image_success():
     assert result == fake_bytes
     mock_get.assert_called_once_with(
         "https://example.com/image.jpg",
+        headers={
+            "User-Agent": (
+                "Mozilla/5.0 (compatible; ScriptoriumAI/1.0; "
+                "+https://huggingface.co/spaces/Ma-Ri-Ba-Ku/scriptorium-ai)"
+            ),
+            "Accept": "image/jpeg,image/png,image/*,*/*",
+            "Referer": "https://gallica.bnf.fr/",
+        },
         follow_redirects=True,
         timeout=60.0,
     )