maribakulj commited on
Commit
76f5338
·
unverified ·
2 Parent(s): 0532e968e5d22a

Merge pull request #2 from maribakulj/claude/review-and-plan-r20qn

Browse files
backend/app/schemas/image.py ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Schéma Pydantic pour les métadonnées du dérivé image produit par le pipeline.
3
+ """
4
+ # 2. third-party
5
+ from pydantic import BaseModel
6
+
7
+
8
+ class ImageDerivativeInfo(BaseModel):
9
+ """Résultat de la normalisation d'une image : dimensions originales et chemins des dérivés."""
10
+
11
+ original_url: str
12
+ original_width: int
13
+ original_height: int
14
+ derivative_path: str
15
+ derivative_width: int
16
+ derivative_height: int
17
+ thumbnail_path: str
18
+ thumbnail_width: int
19
+ thumbnail_height: int
backend/app/schemas/model_config.py ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Schémas Pydantic pour la configuration et la découverte des modèles IA.
3
+ """
4
+ # 1. stdlib
5
+ from datetime import datetime
6
+ from enum import Enum
7
+ from typing import Any
8
+
9
+ # 2. third-party
10
+ from pydantic import BaseModel, ConfigDict, Field
11
+
12
+
13
+ class ProviderType(str, Enum):
14
+ GOOGLE_AI_STUDIO = "google_ai_studio"
15
+ VERTEX_API_KEY = "vertex_api_key"
16
+ VERTEX_SERVICE_ACCOUNT = "vertex_service_account"
17
+
18
+
19
+ class ModelInfo(BaseModel):
20
+ """Décrit un modèle IA disponible chez un provider."""
21
+
22
+ model_config = ConfigDict(frozen=True)
23
+
24
+ model_id: str
25
+ display_name: str
26
+ provider: ProviderType
27
+ supports_vision: bool
28
+ input_token_limit: int | None = None
29
+ output_token_limit: int | None = None
30
+
31
+
32
+ class ModelConfig(BaseModel):
33
+ """Configuration du modèle sélectionné pour un corpus (CLAUDE.md §9)."""
34
+
35
+ corpus_id: str
36
+ selected_model_id: str
37
+ selected_model_display_name: str
38
+ provider: ProviderType
39
+ supports_vision: bool
40
+ last_fetched_at: datetime
41
+ available_models: list[dict[str, Any]] # cache sérialisé des ModelInfo
backend/app/services/ai/__init__.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Services AI — providers Google AI et registre de modèles.
3
+ """
4
+ from app.services.ai.model_registry import build_model_config, list_all_models
5
+ from app.services.ai.provider_google_ai import GoogleAIProvider
6
+ from app.services.ai.provider_vertex_key import VertexAPIKeyProvider
7
+ from app.services.ai.provider_vertex_sa import VertexServiceAccountProvider
8
+
9
+ __all__ = [
10
+ "GoogleAIProvider",
11
+ "VertexAPIKeyProvider",
12
+ "VertexServiceAccountProvider",
13
+ "list_all_models",
14
+ "build_model_config",
15
+ ]
backend/app/services/ai/base.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Interface abstraite commune à tous les providers Google AI.
3
+ """
4
+ # 1. stdlib
5
+ from abc import ABC, abstractmethod
6
+ from typing import Any
7
+
8
+ # 2. local
9
+ from app.schemas.model_config import ModelInfo, ProviderType
10
+
11
+
12
+ def is_vision_model(model: Any) -> bool:
13
+ """Détermine si un modèle supporte les entrées image.
14
+
15
+ Les modèles Gemini sont tous multimodaux ; les modèles texte-only (ex :
16
+ embedding, AQA) ne contiennent pas 'gemini' dans leur identifiant.
17
+ """
18
+ name = (getattr(model, "name", "") or "").lower()
19
+ display = (getattr(model, "display_name", "") or "").lower()
20
+ return "gemini" in name or "vision" in name or "vision" in display
21
+
22
+
23
+ class AIProvider(ABC):
24
+ """Interface commune à tous les providers Google AI."""
25
+
26
+ @property
27
+ @abstractmethod
28
+ def provider_type(self) -> ProviderType: ...
29
+
30
+ @abstractmethod
31
+ def is_configured(self) -> bool:
32
+ """Retourne True si les credentials nécessaires sont présents en environnement."""
33
+ ...
34
+
35
+ @abstractmethod
36
+ def list_models(self) -> list[ModelInfo]:
37
+ """Liste les modèles filtrés (generateContent présent dans les méthodes supportées).
38
+
39
+ Lève RuntimeError si le provider n'est pas configuré.
40
+ Propage les exceptions réseau/API sans les masquer.
41
+ """
42
+ ...
backend/app/services/ai/model_registry.py ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Registre agrégé des modèles disponibles tous providers confondus.
3
+ """
4
+ # 1. stdlib
5
+ import logging
6
+ from datetime import datetime, timezone
7
+
8
+ # 2. local
9
+ from app.schemas.model_config import ModelConfig, ModelInfo
10
+ from app.services.ai.base import AIProvider
11
+ from app.services.ai.provider_google_ai import GoogleAIProvider
12
+ from app.services.ai.provider_vertex_key import VertexAPIKeyProvider
13
+ from app.services.ai.provider_vertex_sa import VertexServiceAccountProvider
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+
18
+ def _build_providers() -> list[AIProvider]:
19
+ return [
20
+ GoogleAIProvider(),
21
+ VertexAPIKeyProvider(),
22
+ VertexServiceAccountProvider(),
23
+ ]
24
+
25
+
26
+ def list_all_models() -> list[ModelInfo]:
27
+ """Interroge tous les providers configurés et retourne la liste agrégée.
28
+
29
+ - Un provider non configuré (credentials absentes) est silencieusement ignoré.
30
+ - Un provider défaillant (clé invalide, erreur réseau) logue un warning et est ignoré.
31
+ """
32
+ result: list[ModelInfo] = []
33
+
34
+ for provider in _build_providers():
35
+ if not provider.is_configured():
36
+ logger.debug(
37
+ "Provider non configuré, ignoré",
38
+ extra={"provider": provider.provider_type},
39
+ )
40
+ continue
41
+
42
+ try:
43
+ models = provider.list_models()
44
+ result.extend(models)
45
+ logger.info(
46
+ "Provider interrogé avec succès",
47
+ extra={"provider": provider.provider_type, "count": len(models)},
48
+ )
49
+ except Exception as exc:
50
+ logger.warning(
51
+ "Provider inaccessible",
52
+ extra={"provider": provider.provider_type, "error": str(exc)},
53
+ )
54
+
55
+ return result
56
+
57
+
58
+ def build_model_config(corpus_id: str, selected_model_id: str) -> ModelConfig:
59
+ """Construit un ModelConfig à partir d'un model_id sélectionné.
60
+
61
+ Lève ValueError si le modèle n'est pas dans la liste des disponibles.
62
+ """
63
+ models = list_all_models()
64
+ model_map = {m.model_id: m for m in models}
65
+
66
+ if selected_model_id not in model_map:
67
+ available = sorted(model_map.keys())
68
+ raise ValueError(
69
+ f"Modèle '{selected_model_id}' non disponible. "
70
+ f"Modèles disponibles : {available}"
71
+ )
72
+
73
+ selected = model_map[selected_model_id]
74
+ return ModelConfig(
75
+ corpus_id=corpus_id,
76
+ selected_model_id=selected.model_id,
77
+ selected_model_display_name=selected.display_name,
78
+ provider=selected.provider,
79
+ supports_vision=selected.supports_vision,
80
+ last_fetched_at=datetime.now(tz=timezone.utc),
81
+ available_models=[m.model_dump() for m in models],
82
+ )
backend/app/services/ai/provider_google_ai.py ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Provider Google AI Studio — authentification via GOOGLE_AI_STUDIO_API_KEY.
3
+ """
4
+ # 1. stdlib
5
+ import logging
6
+ import os
7
+
8
+ # 2. third-party
9
+ from google import genai
10
+
11
+ # 3. local
12
+ from app.schemas.model_config import ModelInfo, ProviderType
13
+ from app.services.ai.base import AIProvider, is_vision_model
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+ _ENV_KEY = "GOOGLE_AI_STUDIO_API_KEY"
18
+
19
+
20
+ class GoogleAIProvider(AIProvider):
21
+ """Provider Google AI Studio (clé API GOOGLE_AI_STUDIO_API_KEY)."""
22
+
23
+ @property
24
+ def provider_type(self) -> ProviderType:
25
+ return ProviderType.GOOGLE_AI_STUDIO
26
+
27
+ def is_configured(self) -> bool:
28
+ return bool(os.environ.get(_ENV_KEY))
29
+
30
+ def list_models(self) -> list[ModelInfo]:
31
+ if not self.is_configured():
32
+ raise RuntimeError(f"Variable d'environnement manquante : {_ENV_KEY}")
33
+
34
+ client = genai.Client(api_key=os.environ[_ENV_KEY])
35
+ result: list[ModelInfo] = []
36
+
37
+ for model in client.models.list():
38
+ methods = getattr(model, "supported_generation_methods", []) or []
39
+ if "generateContent" not in methods:
40
+ continue
41
+
42
+ result.append(ModelInfo(
43
+ model_id=model.name,
44
+ display_name=getattr(model, "display_name", model.name),
45
+ provider=self.provider_type,
46
+ supports_vision=is_vision_model(model),
47
+ input_token_limit=getattr(model, "input_token_limit", None),
48
+ output_token_limit=getattr(model, "output_token_limit", None),
49
+ ))
50
+
51
+ logger.info(
52
+ "Google AI Studio models fetched",
53
+ extra={"provider": self.provider_type, "count": len(result)},
54
+ )
55
+ return result
backend/app/services/ai/provider_vertex_key.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Provider Vertex AI — authentification via clé API GCP (VERTEX_API_KEY).
3
+ """
4
+ # 1. stdlib
5
+ import logging
6
+ import os
7
+
8
+ # 2. third-party
9
+ from google import genai
10
+
11
+ # 3. local
12
+ from app.schemas.model_config import ModelInfo, ProviderType
13
+ from app.services.ai.base import AIProvider, is_vision_model
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+ _ENV_KEY = "VERTEX_API_KEY"
18
+
19
+
20
+ class VertexAPIKeyProvider(AIProvider):
21
+ """Provider Vertex AI via clé API GCP (VERTEX_API_KEY).
22
+
23
+ Utilise le SDK google-genai avec la clé GCP. La clé doit être autorisée
24
+ sur l'API Generative Language dans la console GCP.
25
+ """
26
+
27
+ @property
28
+ def provider_type(self) -> ProviderType:
29
+ return ProviderType.VERTEX_API_KEY
30
+
31
+ def is_configured(self) -> bool:
32
+ return bool(os.environ.get(_ENV_KEY))
33
+
34
+ def list_models(self) -> list[ModelInfo]:
35
+ if not self.is_configured():
36
+ raise RuntimeError(f"Variable d'environnement manquante : {_ENV_KEY}")
37
+
38
+ client = genai.Client(api_key=os.environ[_ENV_KEY])
39
+ result: list[ModelInfo] = []
40
+
41
+ for model in client.models.list():
42
+ methods = getattr(model, "supported_generation_methods", []) or []
43
+ if "generateContent" not in methods:
44
+ continue
45
+
46
+ result.append(ModelInfo(
47
+ model_id=model.name,
48
+ display_name=getattr(model, "display_name", model.name),
49
+ provider=self.provider_type,
50
+ supports_vision=is_vision_model(model),
51
+ input_token_limit=getattr(model, "input_token_limit", None),
52
+ output_token_limit=getattr(model, "output_token_limit", None),
53
+ ))
54
+
55
+ logger.info(
56
+ "Vertex API key models fetched",
57
+ extra={"provider": self.provider_type, "count": len(result)},
58
+ )
59
+ return result
backend/app/services/ai/provider_vertex_sa.py ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Provider Vertex AI — authentification via compte de service JSON (VERTEX_SERVICE_ACCOUNT_JSON).
3
+ """
4
+ # 1. stdlib
5
+ import json
6
+ import logging
7
+ import os
8
+
9
+ # 2. third-party
10
+ from google import genai
11
+ from google.oauth2 import service_account
12
+
13
+ # 3. local
14
+ from app.schemas.model_config import ModelInfo, ProviderType
15
+ from app.services.ai.base import AIProvider, is_vision_model
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ _ENV_KEY = "VERTEX_SERVICE_ACCOUNT_JSON"
20
+ _VERTEX_SCOPES = ["https://www.googleapis.com/auth/cloud-platform"]
21
+ _DEFAULT_LOCATION = "us-central1"
22
+
23
+
24
+ class VertexServiceAccountProvider(AIProvider):
25
+ """Provider Vertex AI via compte de service JSON (VERTEX_SERVICE_ACCOUNT_JSON).
26
+
27
+ Le JSON complet du compte de service est lu depuis la variable d'environnement.
28
+ Le project_id est extrait du JSON ; la localisation par défaut est us-central1.
29
+ """
30
+
31
+ @property
32
+ def provider_type(self) -> ProviderType:
33
+ return ProviderType.VERTEX_SERVICE_ACCOUNT
34
+
35
+ def is_configured(self) -> bool:
36
+ return bool(os.environ.get(_ENV_KEY))
37
+
38
+ def list_models(self) -> list[ModelInfo]:
39
+ if not self.is_configured():
40
+ raise RuntimeError(f"Variable d'environnement manquante : {_ENV_KEY}")
41
+
42
+ sa_json_str = os.environ[_ENV_KEY]
43
+ try:
44
+ sa_info = json.loads(sa_json_str)
45
+ except json.JSONDecodeError as exc:
46
+ raise ValueError(
47
+ f"{_ENV_KEY} : JSON invalide — {exc}"
48
+ ) from exc
49
+
50
+ project_id: str | None = sa_info.get("project_id")
51
+ if not project_id:
52
+ raise ValueError(
53
+ f"{_ENV_KEY} : champ 'project_id' manquant dans le JSON"
54
+ )
55
+
56
+ credentials = service_account.Credentials.from_service_account_info(
57
+ sa_info,
58
+ scopes=_VERTEX_SCOPES,
59
+ )
60
+ client = genai.Client(
61
+ vertexai=True,
62
+ project=project_id,
63
+ location=_DEFAULT_LOCATION,
64
+ credentials=credentials,
65
+ )
66
+ result: list[ModelInfo] = []
67
+
68
+ for model in client.models.list():
69
+ methods = getattr(model, "supported_generation_methods", []) or []
70
+ if "generateContent" not in methods:
71
+ continue
72
+
73
+ result.append(ModelInfo(
74
+ model_id=model.name,
75
+ display_name=getattr(model, "display_name", model.name),
76
+ provider=self.provider_type,
77
+ supports_vision=is_vision_model(model),
78
+ input_token_limit=getattr(model, "input_token_limit", None),
79
+ output_token_limit=getattr(model, "output_token_limit", None),
80
+ ))
81
+
82
+ logger.info(
83
+ "Vertex service account models fetched",
84
+ extra={
85
+ "provider": self.provider_type,
86
+ "project": project_id,
87
+ "count": len(result),
88
+ },
89
+ )
90
+ return result
backend/app/services/image/__init__.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ """
2
+ Services image — normalisation et production des dérivés JPEG pour le pipeline IA.
3
+ """
4
+ from app.services.image.normalizer import create_derivatives, fetch_and_normalize
5
+
6
+ __all__ = ["create_derivatives", "fetch_and_normalize"]
backend/app/services/image/normalizer.py ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Normalisation d'images : dérivé JPEG 1500px max + thumbnail pour le pipeline IA.
3
+ """
4
+ # 1. stdlib
5
+ import io
6
+ import logging
7
+ from pathlib import Path
8
+
9
+ # 2. third-party
10
+ from PIL import Image
11
+
12
+ # 3. local
13
+ from app.schemas.image import ImageDerivativeInfo
14
+ from app.services.ingest.iiif_fetcher import fetch_iiif_image
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+ # Constantes de normalisation
19
+ _MAX_DERIVATIVE_PX = 1500 # grand côté max du dérivé envoyé à l'IA
20
+ _MAX_THUMBNAIL_PX = 256 # grand côté max du thumbnail
21
+ _DERIVATIVE_QUALITY = 90 # qualité JPEG dérivé
22
+ _THUMBNAIL_QUALITY = 75 # qualité JPEG thumbnail
23
+
24
+
25
+ def _resize_to_max(image: Image.Image, max_size: int) -> Image.Image:
26
+ """Redimensionne l'image pour que son grand côté vaille max_size.
27
+
28
+ Si l'image est déjà plus petite ou égale, retourne une copie sans upscaling.
29
+ Le ratio d'aspect est préservé. Utilise LANCZOS pour la qualité.
30
+ """
31
+ w, h = image.size
32
+ if max(w, h) <= max_size:
33
+ return image.copy()
34
+ if w >= h:
35
+ new_w = max_size
36
+ new_h = max(1, round(h * max_size / w))
37
+ else:
38
+ new_h = max_size
39
+ new_w = max(1, round(w * max_size / h))
40
+ return image.resize((new_w, new_h), Image.Resampling.LANCZOS)
41
+
42
+
43
+ def create_derivatives(
44
+ source_bytes: bytes,
45
+ original_url: str,
46
+ corpus_slug: str,
47
+ folio_label: str,
48
+ base_data_dir: Path = Path("data"),
49
+ ) -> ImageDerivativeInfo:
50
+ """Produit un dérivé JPEG (1500px max) et un thumbnail depuis des bytes image.
51
+
52
+ Structure de sortie (CLAUDE.md §3) :
53
+ {base_data_dir}/corpora/{corpus_slug}/derivatives/{folio_label}.jpg
54
+ {base_data_dir}/corpora/{corpus_slug}/derivatives/{folio_label}_thumb.jpg
55
+
56
+ Args:
57
+ source_bytes: contenu brut de l'image source (JPEG, PNG, TIFF, etc.).
58
+ original_url: URL d'origine (conservée dans le schéma de sortie).
59
+ corpus_slug: identifiant du corpus (ex. "beatus-lat8878").
60
+ folio_label: identifiant du folio (ex. "0013r").
61
+ base_data_dir: racine du dossier data (défaut : Path("data")).
62
+
63
+ Returns:
64
+ ImageDerivativeInfo avec dimensions et chemins des fichiers produits.
65
+
66
+ Raises:
67
+ PIL.UnidentifiedImageError: si les bytes ne sont pas une image valide.
68
+ OSError: si l'écriture sur disque échoue.
69
+ """
70
+ image = Image.open(io.BytesIO(source_bytes))
71
+
72
+ # Convertir en RGB pour garantir un JPEG valide (PNG RGBA, palette, etc.)
73
+ if image.mode != "RGB":
74
+ image = image.convert("RGB")
75
+
76
+ original_width, original_height = image.size
77
+ logger.info(
78
+ "Image ouverte",
79
+ extra={
80
+ "corpus": corpus_slug,
81
+ "folio": folio_label,
82
+ "original_size": f"{original_width}x{original_height}",
83
+ },
84
+ )
85
+
86
+ # Dossier de sortie
87
+ derivatives_dir = base_data_dir / "corpora" / corpus_slug / "derivatives"
88
+ derivatives_dir.mkdir(parents=True, exist_ok=True)
89
+
90
+ # Dérivé IA : grand côté <= 1500px
91
+ deriv_image = _resize_to_max(image, _MAX_DERIVATIVE_PX)
92
+ derivative_width, derivative_height = deriv_image.size
93
+ derivative_path = derivatives_dir / f"{folio_label}.jpg"
94
+ deriv_image.save(derivative_path, format="JPEG", quality=_DERIVATIVE_QUALITY)
95
+
96
+ # Thumbnail : grand côté <= 256px
97
+ thumb_image = _resize_to_max(image, _MAX_THUMBNAIL_PX)
98
+ thumbnail_width, thumbnail_height = thumb_image.size
99
+ thumbnail_path = derivatives_dir / f"{folio_label}_thumb.jpg"
100
+ thumb_image.save(thumbnail_path, format="JPEG", quality=_THUMBNAIL_QUALITY)
101
+
102
+ logger.info(
103
+ "Dérivés produits",
104
+ extra={
105
+ "corpus": corpus_slug,
106
+ "folio": folio_label,
107
+ "derivative": f"{derivative_width}x{derivative_height}",
108
+ "thumbnail": f"{thumbnail_width}x{thumbnail_height}",
109
+ },
110
+ )
111
+
112
+ return ImageDerivativeInfo(
113
+ original_url=original_url,
114
+ original_width=original_width,
115
+ original_height=original_height,
116
+ derivative_path=str(derivative_path),
117
+ derivative_width=derivative_width,
118
+ derivative_height=derivative_height,
119
+ thumbnail_path=str(thumbnail_path),
120
+ thumbnail_width=thumbnail_width,
121
+ thumbnail_height=thumbnail_height,
122
+ )
123
+
124
+
125
+ def fetch_and_normalize(
126
+ url: str,
127
+ corpus_slug: str,
128
+ folio_label: str,
129
+ base_data_dir: Path = Path("data"),
130
+ ) -> ImageDerivativeInfo:
131
+ """Point d'entrée principal : télécharge depuis une URL IIIF et produit les dérivés.
132
+
133
+ Chaîne fetch_iiif_image() → create_derivatives().
134
+
135
+ Args:
136
+ url: URL complète de l'image IIIF.
137
+ corpus_slug: identifiant du corpus.
138
+ folio_label: identifiant du folio.
139
+ base_data_dir: racine du dossier data.
140
+
141
+ Returns:
142
+ ImageDerivativeInfo rempli.
143
+ """
144
+ source_bytes = fetch_iiif_image(url)
145
+ return create_derivatives(source_bytes, url, corpus_slug, folio_label, base_data_dir)
backend/app/services/ingest/__init__.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ """
2
+ Services d'ingestion — téléchargement de sources (IIIF, fichiers locaux).
3
+ """
4
+ from app.services.ingest.iiif_fetcher import fetch_iiif_image
5
+
6
+ __all__ = ["fetch_iiif_image"]
backend/app/services/ingest/iiif_fetcher.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Téléchargement d'images depuis des URLs IIIF via httpx.
3
+ """
4
+ # 1. stdlib
5
+ import logging
6
+
7
+ # 2. third-party
8
+ import httpx
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+ _DEFAULT_TIMEOUT = 60.0 # secondes — les images IIIF haute résolution peuvent être lourdes
13
+
14
+
15
+ def fetch_iiif_image(url: str, timeout: float = _DEFAULT_TIMEOUT) -> bytes:
16
+ """Télécharge une image depuis une URL IIIF complète.
17
+
18
+ Args:
19
+ url: URL complète de l'image (ex. https://.../full/max/0/default.jpg).
20
+ timeout: délai maximal en secondes (défaut : 60 s).
21
+
22
+ Returns:
23
+ Contenu brut de l'image en bytes.
24
+
25
+ Raises:
26
+ httpx.HTTPStatusError: si le serveur retourne un code 4xx ou 5xx.
27
+ httpx.TimeoutException: si la requête dépasse le délai.
28
+ httpx.RequestError: pour toute autre erreur réseau.
29
+ """
30
+ logger.info("Fetching IIIF image", extra={"url": url})
31
+ response = httpx.get(url, follow_redirects=True, timeout=timeout)
32
+ response.raise_for_status()
33
+ logger.info(
34
+ "IIIF image fetched",
35
+ extra={"url": url, "size_bytes": len(response.content)},
36
+ )
37
+ return response.content
backend/pyproject.toml CHANGED
@@ -13,7 +13,9 @@ dependencies = [
13
  "pydantic>=2.7",
14
  "sqlalchemy>=2.0",
15
  "aiosqlite>=0.20",
16
- "google-generativeai>=0.3",
 
 
17
  "lxml>=5.2",
18
  "Pillow>=10.3",
19
  ]
 
13
  "pydantic>=2.7",
14
  "sqlalchemy>=2.0",
15
  "aiosqlite>=0.20",
16
+ "google-genai>=1.0",
17
+ "google-auth>=2.0",
18
+ "httpx>=0.27",
19
  "lxml>=5.2",
20
  "Pillow>=10.3",
21
  ]
backend/tests/test_ai_providers.py ADDED
@@ -0,0 +1,471 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Tests des providers Google AI et du registre de modèles.
3
+ Aucun appel réseau réel — tous les clients SDK sont mockés.
4
+ """
5
+ # 1. stdlib
6
+ import json
7
+ from datetime import datetime, timezone
8
+ from unittest.mock import MagicMock, patch
9
+
10
+ # 2. third-party
11
+ import pytest
12
+ from pydantic import ValidationError
13
+
14
+ # 3. local
15
+ from app.schemas.model_config import ModelConfig, ModelInfo, ProviderType
16
+ from app.services.ai.base import is_vision_model
17
+ from app.services.ai.model_registry import build_model_config, list_all_models
18
+ from app.services.ai.provider_google_ai import GoogleAIProvider
19
+ from app.services.ai.provider_vertex_key import VertexAPIKeyProvider
20
+ from app.services.ai.provider_vertex_sa import VertexServiceAccountProvider
21
+
22
+ # ---------------------------------------------------------------------------
23
+ # Données de test partagées
24
+ # ---------------------------------------------------------------------------
25
+
26
+ FAKE_SA_JSON = {
27
+ "type": "service_account",
28
+ "project_id": "test-project-123",
29
+ "private_key_id": "key-abc",
30
+ "private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEA\n-----END RSA PRIVATE KEY-----\n",
31
+ "client_email": "test-sa@test-project-123.iam.gserviceaccount.com",
32
+ "client_id": "123456789",
33
+ "auth_uri": "https://accounts.google.com/o/oauth2/auth",
34
+ "token_uri": "https://oauth2.googleapis.com/token",
35
+ }
36
+
37
+
38
+ def _make_mock_model(
39
+ name: str = "models/gemini-1.5-pro",
40
+ display_name: str = "Gemini 1.5 Pro",
41
+ methods: list[str] | None = None,
42
+ input_token_limit: int = 1_000_000,
43
+ output_token_limit: int = 8192,
44
+ ) -> MagicMock:
45
+ """Construit un objet modèle factice imitant google.genai.types.Model."""
46
+ m = MagicMock()
47
+ m.name = name
48
+ m.display_name = display_name
49
+ m.supported_generation_methods = methods if methods is not None else ["generateContent"]
50
+ m.input_token_limit = input_token_limit
51
+ m.output_token_limit = output_token_limit
52
+ return m
53
+
54
+
55
+ # ---------------------------------------------------------------------------
56
+ # Tests — ModelInfo (schéma)
57
+ # ---------------------------------------------------------------------------
58
+
59
+ def test_model_info_valid():
60
+ info = ModelInfo(
61
+ model_id="models/gemini-1.5-pro",
62
+ display_name="Gemini 1.5 Pro",
63
+ provider=ProviderType.GOOGLE_AI_STUDIO,
64
+ supports_vision=True,
65
+ input_token_limit=1_000_000,
66
+ output_token_limit=8192,
67
+ )
68
+ assert info.model_id == "models/gemini-1.5-pro"
69
+ assert info.supports_vision is True
70
+
71
+
72
+ def test_model_info_is_frozen():
73
+ info = ModelInfo(
74
+ model_id="models/gemini-1.5-pro",
75
+ display_name="Gemini 1.5 Pro",
76
+ provider=ProviderType.GOOGLE_AI_STUDIO,
77
+ supports_vision=True,
78
+ )
79
+ with pytest.raises((TypeError, ValidationError)):
80
+ info.model_id = "changed" # type: ignore[misc]
81
+
82
+
83
+ def test_model_info_optional_token_limits():
84
+ info = ModelInfo(
85
+ model_id="models/gemini-2.0-flash",
86
+ display_name="Gemini 2.0 Flash",
87
+ provider=ProviderType.VERTEX_SERVICE_ACCOUNT,
88
+ supports_vision=True,
89
+ )
90
+ assert info.input_token_limit is None
91
+ assert info.output_token_limit is None
92
+
93
+
94
+ def test_model_info_all_provider_types():
95
+ for ptype in ProviderType:
96
+ info = ModelInfo(
97
+ model_id=f"models/test-{ptype.value}",
98
+ display_name="Test",
99
+ provider=ptype,
100
+ supports_vision=False,
101
+ )
102
+ assert info.provider == ptype
103
+
104
+
105
+ # ---------------------------------------------------------------------------
106
+ # Tests — ModelConfig (schéma)
107
+ # ---------------------------------------------------------------------------
108
+
109
+ def test_model_config_valid():
110
+ cfg = ModelConfig(
111
+ corpus_id="corpus-001",
112
+ selected_model_id="models/gemini-1.5-pro",
113
+ selected_model_display_name="Gemini 1.5 Pro",
114
+ provider=ProviderType.GOOGLE_AI_STUDIO,
115
+ supports_vision=True,
116
+ last_fetched_at=datetime(2026, 3, 17, tzinfo=timezone.utc),
117
+ available_models=[],
118
+ )
119
+ assert cfg.corpus_id == "corpus-001"
120
+ assert cfg.supports_vision is True
121
+
122
+
123
+ def test_model_config_missing_required_field():
124
+ with pytest.raises(ValidationError):
125
+ ModelConfig.model_validate({"corpus_id": "x"})
126
+
127
+
128
+ # ---------------------------------------------------------------------------
129
+ # Tests — is_vision_model helper
130
+ # ---------------------------------------------------------------------------
131
+
132
+ def test_is_vision_model_gemini():
133
+ m = MagicMock()
134
+ m.name = "models/gemini-1.5-pro"
135
+ m.display_name = "Gemini 1.5 Pro"
136
+ assert is_vision_model(m) is True
137
+
138
+
139
+ def test_is_vision_model_vision_in_name():
140
+ m = MagicMock()
141
+ m.name = "models/some-vision-model"
142
+ m.display_name = "Some Model"
143
+ assert is_vision_model(m) is True
144
+
145
+
146
+ def test_is_vision_model_vision_in_display():
147
+ m = MagicMock()
148
+ m.name = "models/some-model"
149
+ m.display_name = "Some Vision Model"
150
+ assert is_vision_model(m) is True
151
+
152
+
153
+ def test_is_vision_model_text_only():
154
+ m = MagicMock()
155
+ m.name = "models/text-embedding-004"
156
+ m.display_name = "Text Embedding"
157
+ assert is_vision_model(m) is False
158
+
159
+
160
+ # ---------------------------------------------------------------------------
161
+ # Tests — GoogleAIProvider
162
+ # ---------------------------------------------------------------------------
163
+
164
+ def test_google_ai_provider_not_configured(monkeypatch):
165
+ monkeypatch.delenv("GOOGLE_AI_STUDIO_API_KEY", raising=False)
166
+ provider = GoogleAIProvider()
167
+ assert provider.is_configured() is False
168
+
169
+
170
+ def test_google_ai_provider_configured(monkeypatch):
171
+ monkeypatch.setenv("GOOGLE_AI_STUDIO_API_KEY", "fake-key")
172
+ provider = GoogleAIProvider()
173
+ assert provider.is_configured() is True
174
+
175
+
176
+ def test_google_ai_provider_type():
177
+ assert GoogleAIProvider().provider_type == ProviderType.GOOGLE_AI_STUDIO
178
+
179
+
180
+ def test_google_ai_provider_list_models_not_configured(monkeypatch):
181
+ monkeypatch.delenv("GOOGLE_AI_STUDIO_API_KEY", raising=False)
182
+ with pytest.raises(RuntimeError, match="GOOGLE_AI_STUDIO_API_KEY"):
183
+ GoogleAIProvider().list_models()
184
+
185
+
186
+ def test_google_ai_provider_list_models_success(monkeypatch):
187
+ monkeypatch.setenv("GOOGLE_AI_STUDIO_API_KEY", "fake-key")
188
+ mock_model = _make_mock_model()
189
+
190
+ with patch("app.services.ai.provider_google_ai.genai.Client") as MockClient:
191
+ MockClient.return_value.models.list.return_value = [mock_model]
192
+ models = GoogleAIProvider().list_models()
193
+
194
+ assert len(models) == 1
195
+ assert models[0].model_id == "models/gemini-1.5-pro"
196
+ assert models[0].provider == ProviderType.GOOGLE_AI_STUDIO
197
+ assert models[0].supports_vision is True
198
+ MockClient.assert_called_once_with(api_key="fake-key")
199
+
200
+
201
+ def test_google_ai_provider_filters_non_generate_content(monkeypatch):
202
+ monkeypatch.setenv("GOOGLE_AI_STUDIO_API_KEY", "fake-key")
203
+ embedding = _make_mock_model(
204
+ name="models/text-embedding-004",
205
+ display_name="Text Embedding",
206
+ methods=["embedContent"],
207
+ )
208
+ gemini = _make_mock_model()
209
+
210
+ with patch("app.services.ai.provider_google_ai.genai.Client") as MockClient:
211
+ MockClient.return_value.models.list.return_value = [embedding, gemini]
212
+ models = GoogleAIProvider().list_models()
213
+
214
+ assert len(models) == 1
215
+ assert models[0].model_id == "models/gemini-1.5-pro"
216
+
217
+
218
+ def test_google_ai_provider_empty_list(monkeypatch):
219
+ monkeypatch.setenv("GOOGLE_AI_STUDIO_API_KEY", "fake-key")
220
+ with patch("app.services.ai.provider_google_ai.genai.Client") as MockClient:
221
+ MockClient.return_value.models.list.return_value = []
222
+ models = GoogleAIProvider().list_models()
223
+ assert models == []
224
+
225
+
226
+ # ---------------------------------------------------------------------------
227
+ # Tests — VertexAPIKeyProvider
228
+ # ---------------------------------------------------------------------------
229
+
230
+ def test_vertex_key_provider_not_configured(monkeypatch):
231
+ monkeypatch.delenv("VERTEX_API_KEY", raising=False)
232
+ assert VertexAPIKeyProvider().is_configured() is False
233
+
234
+
235
+ def test_vertex_key_provider_configured(monkeypatch):
236
+ monkeypatch.setenv("VERTEX_API_KEY", "fake-vertex-key")
237
+ assert VertexAPIKeyProvider().is_configured() is True
238
+
239
+
240
+ def test_vertex_key_provider_type():
241
+ assert VertexAPIKeyProvider().provider_type == ProviderType.VERTEX_API_KEY
242
+
243
+
244
+ def test_vertex_key_provider_list_models_not_configured(monkeypatch):
245
+ monkeypatch.delenv("VERTEX_API_KEY", raising=False)
246
+ with pytest.raises(RuntimeError, match="VERTEX_API_KEY"):
247
+ VertexAPIKeyProvider().list_models()
248
+
249
+
250
+ def test_vertex_key_provider_list_models_success(monkeypatch):
251
+ monkeypatch.setenv("VERTEX_API_KEY", "fake-vertex-key")
252
+ mock_model = _make_mock_model(
253
+ name="models/gemini-2.0-flash",
254
+ display_name="Gemini 2.0 Flash",
255
+ )
256
+
257
+ with patch("app.services.ai.provider_vertex_key.genai.Client") as MockClient:
258
+ MockClient.return_value.models.list.return_value = [mock_model]
259
+ models = VertexAPIKeyProvider().list_models()
260
+
261
+ assert len(models) == 1
262
+ assert models[0].model_id == "models/gemini-2.0-flash"
263
+ assert models[0].provider == ProviderType.VERTEX_API_KEY
264
+ MockClient.assert_called_once_with(api_key="fake-vertex-key")
265
+
266
+
267
+ # ---------------------------------------------------------------------------
268
+ # Tests — VertexServiceAccountProvider
269
+ # ---------------------------------------------------------------------------
270
+
271
+ def test_vertex_sa_provider_not_configured(monkeypatch):
272
+ monkeypatch.delenv("VERTEX_SERVICE_ACCOUNT_JSON", raising=False)
273
+ assert VertexServiceAccountProvider().is_configured() is False
274
+
275
+
276
+ def test_vertex_sa_provider_configured(monkeypatch):
277
+ monkeypatch.setenv("VERTEX_SERVICE_ACCOUNT_JSON", json.dumps(FAKE_SA_JSON))
278
+ assert VertexServiceAccountProvider().is_configured() is True
279
+
280
+
281
+ def test_vertex_sa_provider_type():
282
+ assert VertexServiceAccountProvider().provider_type == ProviderType.VERTEX_SERVICE_ACCOUNT
283
+
284
+
285
+ def test_vertex_sa_provider_list_models_not_configured(monkeypatch):
286
+ monkeypatch.delenv("VERTEX_SERVICE_ACCOUNT_JSON", raising=False)
287
+ with pytest.raises(RuntimeError, match="VERTEX_SERVICE_ACCOUNT_JSON"):
288
+ VertexServiceAccountProvider().list_models()
289
+
290
+
291
+ def test_vertex_sa_provider_invalid_json(monkeypatch):
292
+ monkeypatch.setenv("VERTEX_SERVICE_ACCOUNT_JSON", "not-valid-json{{{")
293
+ with pytest.raises(ValueError, match="JSON invalide"):
294
+ VertexServiceAccountProvider().list_models()
295
+
296
+
297
+ def test_vertex_sa_provider_missing_project_id(monkeypatch):
298
+ sa_no_project = {k: v for k, v in FAKE_SA_JSON.items() if k != "project_id"}
299
+ monkeypatch.setenv("VERTEX_SERVICE_ACCOUNT_JSON", json.dumps(sa_no_project))
300
+ with pytest.raises(ValueError, match="project_id"):
301
+ VertexServiceAccountProvider().list_models()
302
+
303
+
304
+ def test_vertex_sa_provider_list_models_success(monkeypatch):
305
+ monkeypatch.setenv("VERTEX_SERVICE_ACCOUNT_JSON", json.dumps(FAKE_SA_JSON))
306
+ mock_model = _make_mock_model(
307
+ name="models/gemini-1.5-pro-002",
308
+ display_name="Gemini 1.5 Pro 002",
309
+ )
310
+ mock_credentials = MagicMock()
311
+
312
+ with patch(
313
+ "app.services.ai.provider_vertex_sa.service_account.Credentials.from_service_account_info",
314
+ return_value=mock_credentials,
315
+ ) as mock_creds_factory:
316
+ with patch("app.services.ai.provider_vertex_sa.genai.Client") as MockClient:
317
+ MockClient.return_value.models.list.return_value = [mock_model]
318
+ models = VertexServiceAccountProvider().list_models()
319
+
320
+ assert len(models) == 1
321
+ assert models[0].model_id == "models/gemini-1.5-pro-002"
322
+ assert models[0].provider == ProviderType.VERTEX_SERVICE_ACCOUNT
323
+ mock_creds_factory.assert_called_once_with(
324
+ FAKE_SA_JSON,
325
+ scopes=["https://www.googleapis.com/auth/cloud-platform"],
326
+ )
327
+ MockClient.assert_called_once_with(
328
+ vertexai=True,
329
+ project="test-project-123",
330
+ location="us-central1",
331
+ credentials=mock_credentials,
332
+ )
333
+
334
+
335
+ def test_vertex_sa_provider_filters_non_generate_content(monkeypatch):
336
+ monkeypatch.setenv("VERTEX_SERVICE_ACCOUNT_JSON", json.dumps(FAKE_SA_JSON))
337
+ embedding = _make_mock_model(
338
+ name="models/textembedding-gecko",
339
+ display_name="Text Embedding Gecko",
340
+ methods=["embedContent"],
341
+ )
342
+
343
+ with patch(
344
+ "app.services.ai.provider_vertex_sa.service_account.Credentials.from_service_account_info",
345
+ return_value=MagicMock(),
346
+ ):
347
+ with patch("app.services.ai.provider_vertex_sa.genai.Client") as MockClient:
348
+ MockClient.return_value.models.list.return_value = [embedding]
349
+ models = VertexServiceAccountProvider().list_models()
350
+
351
+ assert models == []
352
+
353
+
354
+ # ---------------------------------------------------------------------------
355
+ # Tests — model_registry.list_all_models
356
+ # ---------------------------------------------------------------------------
357
+
358
+ def test_list_all_models_no_providers_configured(monkeypatch):
359
+ monkeypatch.delenv("GOOGLE_AI_STUDIO_API_KEY", raising=False)
360
+ monkeypatch.delenv("VERTEX_API_KEY", raising=False)
361
+ monkeypatch.delenv("VERTEX_SERVICE_ACCOUNT_JSON", raising=False)
362
+ result = list_all_models()
363
+ assert result == []
364
+
365
+
366
+ def test_list_all_models_one_provider(monkeypatch):
367
+ monkeypatch.setenv("GOOGLE_AI_STUDIO_API_KEY", "fake-key")
368
+ monkeypatch.delenv("VERTEX_API_KEY", raising=False)
369
+ monkeypatch.delenv("VERTEX_SERVICE_ACCOUNT_JSON", raising=False)
370
+ mock_model = _make_mock_model()
371
+
372
+ with patch("app.services.ai.provider_google_ai.genai.Client") as MockClient:
373
+ MockClient.return_value.models.list.return_value = [mock_model]
374
+ result = list_all_models()
375
+
376
+ assert len(result) == 1
377
+ assert result[0].provider == ProviderType.GOOGLE_AI_STUDIO
378
+
379
+
380
+ def test_list_all_models_aggregates_multiple_providers(monkeypatch):
381
+ # Note : provider_google_ai et provider_vertex_key partagent le même objet
382
+ # google.genai (import module). On patch au niveau des méthodes pour éviter
383
+ # que le second patch.object("...genai.Client") écrase le premier.
384
+ monkeypatch.setenv("GOOGLE_AI_STUDIO_API_KEY", "fake-key-ai")
385
+ monkeypatch.setenv("VERTEX_API_KEY", "fake-key-vertex")
386
+ monkeypatch.delenv("VERTEX_SERVICE_ACCOUNT_JSON", raising=False)
387
+
388
+ models_ai = [ModelInfo(
389
+ model_id="models/gemini-1.5-pro",
390
+ display_name="Gemini 1.5 Pro",
391
+ provider=ProviderType.GOOGLE_AI_STUDIO,
392
+ supports_vision=True,
393
+ )]
394
+ models_vertex = [ModelInfo(
395
+ model_id="models/gemini-2.0-flash",
396
+ display_name="Gemini 2.0 Flash",
397
+ provider=ProviderType.VERTEX_API_KEY,
398
+ supports_vision=True,
399
+ )]
400
+
401
+ with patch.object(GoogleAIProvider, "list_models", return_value=models_ai):
402
+ with patch.object(VertexAPIKeyProvider, "list_models", return_value=models_vertex):
403
+ result = list_all_models()
404
+
405
+ assert len(result) == 2
406
+ providers = {m.provider for m in result}
407
+ assert ProviderType.GOOGLE_AI_STUDIO in providers
408
+ assert ProviderType.VERTEX_API_KEY in providers
409
+
410
+
411
+ def test_list_all_models_failing_provider_is_skipped(monkeypatch):
412
+ monkeypatch.setenv("GOOGLE_AI_STUDIO_API_KEY", "bad-key")
413
+ monkeypatch.setenv("VERTEX_API_KEY", "good-key")
414
+ monkeypatch.delenv("VERTEX_SERVICE_ACCOUNT_JSON", raising=False)
415
+
416
+ models_vertex = [ModelInfo(
417
+ model_id="models/gemini-2.0-flash",
418
+ display_name="Gemini 2.0 Flash",
419
+ provider=ProviderType.VERTEX_API_KEY,
420
+ supports_vision=True,
421
+ )]
422
+
423
+ with patch.object(GoogleAIProvider, "list_models", side_effect=Exception("API key invalid")):
424
+ with patch.object(VertexAPIKeyProvider, "list_models", return_value=models_vertex):
425
+ result = list_all_models()
426
+
427
+ assert len(result) == 1
428
+ assert result[0].provider == ProviderType.VERTEX_API_KEY
429
+
430
+
431
+ # ---------------------------------------------------------------------------
432
+ # Tests — model_registry.build_model_config
433
+ # ---------------------------------------------------------------------------
434
+
435
+ def test_build_model_config_valid(monkeypatch):
436
+ monkeypatch.setenv("GOOGLE_AI_STUDIO_API_KEY", "fake-key")
437
+ monkeypatch.delenv("VERTEX_API_KEY", raising=False)
438
+ monkeypatch.delenv("VERTEX_SERVICE_ACCOUNT_JSON", raising=False)
439
+ mock_model = _make_mock_model()
440
+
441
+ with patch("app.services.ai.provider_google_ai.genai.Client") as MockClient:
442
+ MockClient.return_value.models.list.return_value = [mock_model]
443
+ cfg = build_model_config("corpus-001", "models/gemini-1.5-pro")
444
+
445
+ assert cfg.corpus_id == "corpus-001"
446
+ assert cfg.selected_model_id == "models/gemini-1.5-pro"
447
+ assert cfg.selected_model_display_name == "Gemini 1.5 Pro"
448
+ assert cfg.provider == ProviderType.GOOGLE_AI_STUDIO
449
+ assert cfg.supports_vision is True
450
+ assert len(cfg.available_models) == 1
451
+ assert isinstance(cfg.available_models[0], dict)
452
+
453
+
454
+ def test_build_model_config_unknown_model(monkeypatch):
455
+ monkeypatch.setenv("GOOGLE_AI_STUDIO_API_KEY", "fake-key")
456
+ monkeypatch.delenv("VERTEX_API_KEY", raising=False)
457
+ monkeypatch.delenv("VERTEX_SERVICE_ACCOUNT_JSON", raising=False)
458
+ mock_model = _make_mock_model()
459
+
460
+ with patch("app.services.ai.provider_google_ai.genai.Client") as MockClient:
461
+ MockClient.return_value.models.list.return_value = [mock_model]
462
+ with pytest.raises(ValueError, match="non disponible"):
463
+ build_model_config("corpus-001", "models/nonexistent-model")
464
+
465
+
466
+ def test_build_model_config_no_providers(monkeypatch):
467
+ monkeypatch.delenv("GOOGLE_AI_STUDIO_API_KEY", raising=False)
468
+ monkeypatch.delenv("VERTEX_API_KEY", raising=False)
469
+ monkeypatch.delenv("VERTEX_SERVICE_ACCOUNT_JSON", raising=False)
470
+ with pytest.raises(ValueError, match="non disponible"):
471
+ build_model_config("corpus-001", "models/gemini-1.5-pro")
backend/tests/test_image_pipeline.py ADDED
@@ -0,0 +1,393 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Tests du pipeline image : fetch IIIF + normalisation (dérivé + thumbnail).
3
+ Tests unitaires : httpx mocké, images créées en mémoire (Pillow).
4
+ Tests d'intégration : requêtes réseau réelles, activés via RUN_INTEGRATION_TESTS=1.
5
+ """
6
+ # 1. stdlib
7
+ import io
8
+ import os
9
+ from pathlib import Path
10
+ from unittest.mock import MagicMock, patch
11
+
12
+ # 2. third-party
13
+ import httpx
14
+ import pytest
15
+ from PIL import Image
16
+ from pydantic import ValidationError
17
+
18
+ # 3. local
19
+ from app.schemas.image import ImageDerivativeInfo
20
+ from app.services.image.normalizer import (
21
+ _MAX_DERIVATIVE_PX,
22
+ _MAX_THUMBNAIL_PX,
23
+ _resize_to_max,
24
+ create_derivatives,
25
+ fetch_and_normalize,
26
+ )
27
+ from app.services.ingest.iiif_fetcher import fetch_iiif_image
28
+
29
+ # ---------------------------------------------------------------------------
30
+ # Marqueur d'intégration — activé seulement si RUN_INTEGRATION_TESTS=1
31
+ # ---------------------------------------------------------------------------
32
+
33
+ integration = pytest.mark.skipif(
34
+ not os.environ.get("RUN_INTEGRATION_TESTS"),
35
+ reason="Tests réseau réels : définir RUN_INTEGRATION_TESTS=1 pour les activer",
36
+ )
37
+
38
+ # URLs IIIF des 3 manuscrits de test (BnF Gallica)
39
+ _URL_BEATUS_HI = (
40
+ "https://gallica.bnf.fr/iiif/ark:/12148/btv1b8432836p/f13/full/max/0/default.jpg"
41
+ )
42
+ _URL_BEATUS_LO = (
43
+ "https://gallica.bnf.fr/iiif/ark:/12148/btv1b8432836p/f13/full/600,/0/default.jpg"
44
+ )
45
+ _URL_GRANDES_CHRONIQUES = (
46
+ "https://gallica.bnf.fr/iiif/ark:/12148/btv1b8427295k/f3/full/max/0/default.jpg"
47
+ )
48
+
49
+
50
+ # ---------------------------------------------------------------------------
51
+ # Helpers de test
52
+ # ---------------------------------------------------------------------------
53
+
54
+ def _make_jpeg_bytes(width: int, height: int, color: tuple[int, int, int] = (200, 150, 100)) -> bytes:
55
+ """Crée un JPEG minimal en mémoire pour les tests unitaires."""
56
+ img = Image.new("RGB", (width, height), color=color)
57
+ buf = io.BytesIO()
58
+ img.save(buf, format="JPEG", quality=85)
59
+ return buf.getvalue()
60
+
61
+
62
+ def _make_png_rgba_bytes(width: int, height: int) -> bytes:
63
+ """Crée un PNG RGBA en mémoire (pour tester la conversion RGB)."""
64
+ img = Image.new("RGBA", (width, height), color=(100, 150, 200, 128))
65
+ buf = io.BytesIO()
66
+ img.save(buf, format="PNG")
67
+ return buf.getvalue()
68
+
69
+
70
+ # ---------------------------------------------------------------------------
71
+ # Tests — ImageDerivativeInfo (schéma)
72
+ # ---------------------------------------------------------------------------
73
+
74
+ def test_schema_valid():
75
+ info = ImageDerivativeInfo(
76
+ original_url="https://example.com/image.jpg",
77
+ original_width=3000,
78
+ original_height=4000,
79
+ derivative_path="/data/corpora/test/derivatives/0001r.jpg",
80
+ derivative_width=1125,
81
+ derivative_height=1500,
82
+ thumbnail_path="/data/corpora/test/derivatives/0001r_thumb.jpg",
83
+ thumbnail_width=192,
84
+ thumbnail_height=256,
85
+ )
86
+ assert info.original_width == 3000
87
+ assert info.derivative_width == 1125
88
+
89
+
90
+ def test_schema_missing_required_field():
91
+ with pytest.raises(ValidationError):
92
+ ImageDerivativeInfo.model_validate({"original_url": "https://x.com/img.jpg"})
93
+
94
+
95
+ def test_schema_all_fields_present():
96
+ fields = ImageDerivativeInfo.model_fields.keys()
97
+ expected = {
98
+ "original_url", "original_width", "original_height",
99
+ "derivative_path", "derivative_width", "derivative_height",
100
+ "thumbnail_path", "thumbnail_width", "thumbnail_height",
101
+ }
102
+ assert set(fields) == expected
103
+
104
+
105
+ # ---------------------------------------------------------------------------
106
+ # Tests — _resize_to_max
107
+ # ---------------------------------------------------------------------------
108
+
109
+ def test_resize_small_image_not_upscaled():
110
+ """Une image déjà petite ne doit pas être agrandie."""
111
+ img = Image.new("RGB", (800, 600))
112
+ result = _resize_to_max(img, _MAX_DERIVATIVE_PX)
113
+ assert result.size == (800, 600)
114
+
115
+
116
+ def test_resize_exact_max_not_changed():
117
+ """Une image dont le grand côté est exactement max_size n'est pas redimensionnée."""
118
+ img = Image.new("RGB", (1500, 1000))
119
+ result = _resize_to_max(img, _MAX_DERIVATIVE_PX)
120
+ assert result.size == (1500, 1000)
121
+
122
+
123
+ def test_resize_landscape_large():
124
+ """Paysage 3000x2000 → 1500x1000."""
125
+ img = Image.new("RGB", (3000, 2000))
126
+ result = _resize_to_max(img, 1500)
127
+ assert result.size == (1500, 1000)
128
+
129
+
130
+ def test_resize_portrait_large():
131
+ """Portrait 2000x3000 → 1000x1500."""
132
+ img = Image.new("RGB", (2000, 3000))
133
+ result = _resize_to_max(img, 1500)
134
+ assert result.size == (1000, 1500)
135
+
136
+
137
+ def test_resize_square_large():
138
+ """Carré 2000x2000 → 1500x1500."""
139
+ img = Image.new("RGB", (2000, 2000))
140
+ result = _resize_to_max(img, 1500)
141
+ assert result.size == (1500, 1500)
142
+
143
+
144
+ def test_resize_preserves_aspect_ratio():
145
+ """Le ratio d'aspect est préservé après resize."""
146
+ img = Image.new("RGB", (4000, 3000))
147
+ result = _resize_to_max(img, 1500)
148
+ w, h = result.size
149
+ assert w == 1500
150
+ assert abs(w / h - 4 / 3) < 0.01
151
+
152
+
153
+ def test_resize_returns_copy_when_no_resize_needed():
154
+ """Retourne une copie (pas la même instance) même sans resize."""
155
+ img = Image.new("RGB", (100, 100))
156
+ result = _resize_to_max(img, 1500)
157
+ assert result is not img
158
+
159
+
160
+ def test_resize_thumbnail_size():
161
+ """Vérification pour la taille thumbnail (256px)."""
162
+ img = Image.new("RGB", (1200, 800))
163
+ result = _resize_to_max(img, _MAX_THUMBNAIL_PX)
164
+ assert result.size[0] == 256
165
+ assert result.size[1] == 171 # round(800 * 256 / 1200) = round(170.67) = 171
166
+
167
+
168
+ # ---------------------------------------------------------------------------
169
+ # Tests — create_derivatives
170
+ # ---------------------------------------------------------------------------
171
+
172
+ def test_create_derivatives_large_landscape(tmp_path):
173
+ """Image 3000x2000 → dérivé 1500x1000, thumbnail 256x171."""
174
+ source = _make_jpeg_bytes(3000, 2000)
175
+ info = create_derivatives(source, "https://x.com/img.jpg", "test-corpus", "0001r", tmp_path)
176
+
177
+ assert info.original_width == 3000
178
+ assert info.original_height == 2000
179
+ assert info.derivative_width == 1500
180
+ assert info.derivative_height == 1000
181
+ assert info.thumbnail_width == 256
182
+ assert info.thumbnail_height == 171
183
+ assert info.original_url == "https://x.com/img.jpg"
184
+
185
+
186
+ def test_create_derivatives_small_image_not_upscaled(tmp_path):
187
+ """Image 600x900 (< 1500px) : dérivé conserve les dimensions originales."""
188
+ source = _make_jpeg_bytes(600, 900)
189
+ info = create_derivatives(source, "https://x.com/img.jpg", "test-corpus", "0001r", tmp_path)
190
+
191
+ assert info.derivative_width == 600
192
+ assert info.derivative_height == 900
193
+ assert info.original_width == 600
194
+ assert info.original_height == 900
195
+
196
+
197
+ def test_create_derivatives_files_exist(tmp_path):
198
+ """Les deux fichiers JPEG sont bien créés sur disque."""
199
+ source = _make_jpeg_bytes(2000, 3000)
200
+ info = create_derivatives(source, "https://x.com/img.jpg", "corpus-a", "f001r", tmp_path)
201
+
202
+ assert Path(info.derivative_path).exists()
203
+ assert Path(info.thumbnail_path).exists()
204
+
205
+
206
+ def test_create_derivatives_path_structure(tmp_path):
207
+ """Les chemins respectent la convention CLAUDE.md §3."""
208
+ source = _make_jpeg_bytes(1000, 1000)
209
+ info = create_derivatives(source, "https://x.com/img.jpg", "beatus-lat8878", "0013r", tmp_path)
210
+
211
+ expected_deriv = tmp_path / "corpora" / "beatus-lat8878" / "derivatives" / "0013r.jpg"
212
+ expected_thumb = tmp_path / "corpora" / "beatus-lat8878" / "derivatives" / "0013r_thumb.jpg"
213
+ assert info.derivative_path == str(expected_deriv)
214
+ assert info.thumbnail_path == str(expected_thumb)
215
+
216
+
217
+ def test_create_derivatives_output_is_jpeg(tmp_path):
218
+ """Les fichiers produits sont bien des JPEG valides."""
219
+ source = _make_jpeg_bytes(1000, 800)
220
+ info = create_derivatives(source, "https://x.com/img.jpg", "corpus-b", "f002r", tmp_path)
221
+
222
+ with Image.open(info.derivative_path) as img:
223
+ assert img.format == "JPEG"
224
+ with Image.open(info.thumbnail_path) as img:
225
+ assert img.format == "JPEG"
226
+
227
+
228
+ def test_create_derivatives_rgba_converted_to_rgb(tmp_path):
229
+ """Un PNG RGBA est converti en RGB sans erreur."""
230
+ source = _make_png_rgba_bytes(800, 1000)
231
+ info = create_derivatives(source, "https://x.com/img.png", "corpus-c", "f003r", tmp_path)
232
+
233
+ with Image.open(info.derivative_path) as img:
234
+ assert img.mode == "RGB"
235
+ assert info.original_width == 800
236
+ assert info.original_height == 1000
237
+
238
+
239
+ def test_create_derivatives_thumbnail_dimensions(tmp_path):
240
+ """Le thumbnail a bien son grand côté <= 256px."""
241
+ source = _make_jpeg_bytes(3000, 4000)
242
+ info = create_derivatives(source, "https://x.com/img.jpg", "corpus-d", "f004r", tmp_path)
243
+
244
+ assert max(info.thumbnail_width, info.thumbnail_height) == _MAX_THUMBNAIL_PX
245
+
246
+
247
+ def test_create_derivatives_creates_parent_dirs(tmp_path):
248
+ """Les dossiers intermédiaires sont créés automatiquement."""
249
+ source = _make_jpeg_bytes(500, 500)
250
+ new_slug = "nouveau-corpus-jamais-vu"
251
+ info = create_derivatives(source, "https://x.com/img.jpg", new_slug, "f001r", tmp_path)
252
+
253
+ assert Path(info.derivative_path).parent.exists()
254
+
255
+
256
+ # ---------------------------------------------------------------------------
257
+ # Tests — fetch_iiif_image
258
+ # ---------------------------------------------------------------------------
259
+
260
+ def test_fetch_iiif_image_success():
261
+ """Retourne les bytes de l'image si la requête réussit."""
262
+ fake_bytes = _make_jpeg_bytes(100, 100)
263
+
264
+ with patch("app.services.ingest.iiif_fetcher.httpx.get") as mock_get:
265
+ mock_response = MagicMock()
266
+ mock_response.content = fake_bytes
267
+ mock_response.raise_for_status.return_value = None
268
+ mock_get.return_value = mock_response
269
+
270
+ result = fetch_iiif_image("https://example.com/image.jpg")
271
+
272
+ assert result == fake_bytes
273
+ mock_get.assert_called_once_with(
274
+ "https://example.com/image.jpg",
275
+ follow_redirects=True,
276
+ timeout=60.0,
277
+ )
278
+
279
+
280
+ def test_fetch_iiif_image_http_error():
281
+ """Propage HTTPStatusError si le serveur répond 404."""
282
+ with patch("app.services.ingest.iiif_fetcher.httpx.get") as mock_get:
283
+ mock_response = MagicMock()
284
+ mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
285
+ "404 Not Found",
286
+ request=MagicMock(),
287
+ response=MagicMock(status_code=404),
288
+ )
289
+ mock_get.return_value = mock_response
290
+
291
+ with pytest.raises(httpx.HTTPStatusError):
292
+ fetch_iiif_image("https://example.com/missing.jpg")
293
+
294
+
295
+ def test_fetch_iiif_image_timeout():
296
+ """Propage TimeoutException si la requête dépasse le délai."""
297
+ with patch("app.services.ingest.iiif_fetcher.httpx.get") as mock_get:
298
+ mock_get.side_effect = httpx.TimeoutException("timed out")
299
+
300
+ with pytest.raises(httpx.TimeoutException):
301
+ fetch_iiif_image("https://example.com/slow.jpg", timeout=1.0)
302
+
303
+
304
+ def test_fetch_iiif_image_custom_timeout():
305
+ """Le timeout personnalisé est bien transmis à httpx.get."""
306
+ fake_bytes = _make_jpeg_bytes(50, 50)
307
+
308
+ with patch("app.services.ingest.iiif_fetcher.httpx.get") as mock_get:
309
+ mock_response = MagicMock()
310
+ mock_response.content = fake_bytes
311
+ mock_response.raise_for_status.return_value = None
312
+ mock_get.return_value = mock_response
313
+
314
+ fetch_iiif_image("https://example.com/img.jpg", timeout=120.0)
315
+
316
+ _, kwargs = mock_get.call_args
317
+ assert kwargs["timeout"] == 120.0
318
+
319
+
320
+ # ---------------------------------------------------------------------------
321
+ # Tests — fetch_and_normalize (end-to-end mocké)
322
+ # ---------------------------------------------------------------------------
323
+
324
+ def test_fetch_and_normalize_chains_correctly(tmp_path):
325
+ """fetch_and_normalize appelle fetch_iiif_image puis create_derivatives."""
326
+ fake_bytes = _make_jpeg_bytes(2000, 1500)
327
+
328
+ with patch("app.services.image.normalizer.fetch_iiif_image", return_value=fake_bytes) as mock_fetch:
329
+ info = fetch_and_normalize(
330
+ "https://example.com/ms/f001.jpg",
331
+ "corpus-test",
332
+ "0001r",
333
+ tmp_path,
334
+ )
335
+
336
+ mock_fetch.assert_called_once_with("https://example.com/ms/f001.jpg")
337
+ assert info.original_url == "https://example.com/ms/f001.jpg"
338
+ assert info.original_width == 2000
339
+ assert info.original_height == 1500
340
+ assert info.derivative_width == 1500
341
+ assert info.derivative_height == 1125
342
+ assert Path(info.derivative_path).exists()
343
+ assert Path(info.thumbnail_path).exists()
344
+
345
+
346
+ def test_fetch_and_normalize_propagates_http_error(tmp_path):
347
+ """Les erreurs HTTP de fetch_iiif_image sont propagées sans être masquées."""
348
+ with patch(
349
+ "app.services.image.normalizer.fetch_iiif_image",
350
+ side_effect=httpx.HTTPStatusError("403", request=MagicMock(), response=MagicMock()),
351
+ ):
352
+ with pytest.raises(httpx.HTTPStatusError):
353
+ fetch_and_normalize("https://example.com/img.jpg", "corpus", "f001", tmp_path)
354
+
355
+
356
+ # ---------------------------------------------------------------------------
357
+ # Tests d'intégration — URLs IIIF BnF réelles (skippés par défaut)
358
+ # ---------------------------------------------------------------------------
359
+
360
+ @integration
361
+ def test_integration_beatus_high_res(tmp_path):
362
+ """Beatus de Saint-Sever, BnF Latin 8878, f.13 — haute résolution."""
363
+ info = fetch_and_normalize(_URL_BEATUS_HI, "beatus-lat8878", "0013r", tmp_path)
364
+
365
+ assert info.original_width > 1500 or info.original_height > 1500
366
+ assert info.derivative_width <= _MAX_DERIVATIVE_PX
367
+ assert info.derivative_height <= _MAX_DERIVATIVE_PX
368
+ assert max(info.derivative_width, info.derivative_height) == _MAX_DERIVATIVE_PX
369
+ assert Path(info.derivative_path).exists()
370
+ assert Path(info.thumbnail_path).exists()
371
+
372
+
373
+ @integration
374
+ def test_integration_beatus_low_res(tmp_path):
375
+ """Beatus de Saint-Sever, BnF Latin 8878, f.13 — 600px (image déjà petite)."""
376
+ info = fetch_and_normalize(_URL_BEATUS_LO, "beatus-lat8878", "0013r-600", tmp_path)
377
+
378
+ # Image à 600px de large : pas d'upscaling, dérivé == original
379
+ assert info.derivative_width <= _MAX_DERIVATIVE_PX
380
+ assert info.derivative_height <= _MAX_DERIVATIVE_PX
381
+ assert max(info.derivative_width, info.derivative_height) <= 600
382
+ assert Path(info.derivative_path).exists()
383
+
384
+
385
+ @integration
386
+ def test_integration_grandes_chroniques(tmp_path):
387
+ """Grandes Chroniques de France, BnF Français 2813."""
388
+ info = fetch_and_normalize(_URL_GRANDES_CHRONIQUES, "grandes-chroniques", "f003", tmp_path)
389
+
390
+ assert info.derivative_width <= _MAX_DERIVATIVE_PX
391
+ assert info.derivative_height <= _MAX_DERIVATIVE_PX
392
+ assert Path(info.derivative_path).exists()
393
+ assert Path(info.thumbnail_path).exists()