Spaces:
Runtime error
Runtime error
Upload folder using huggingface_hub
Browse files- .dockerignore +38 -0
- .env.example +11 -0
- .gitignore +2 -0
- .python-version +1 -0
- Dockerfile +26 -0
- README.md +375 -5
- app/__init__.py +7 -0
- app/api/__init__.py +1 -0
- app/api/controllers.py +36 -0
- app/api/models.py +85 -0
- app/api/routes/__init__.py +1 -0
- app/api/routes/prediction.py +23 -0
- app/core/__init__.py +1 -0
- app/core/app.py +71 -0
- app/core/dependencies.py +17 -0
- app/core/logging.py +26 -0
- app/services/__init__.py +1 -0
- app/services/base.py +30 -0
- app/services/inference.py +86 -0
- main.py +6 -0
- requirements.in +22 -0
- requirements.txt +180 -0
- scripts/generate_test_datasets.py +411 -0
- scripts/model_download.bash +8 -0
- scripts/test_datasets.py +382 -0
- test_main.http +12 -0
.dockerignore
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__
|
| 2 |
+
*.pyc
|
| 3 |
+
*.pyo
|
| 4 |
+
*.pyd
|
| 5 |
+
.Python
|
| 6 |
+
*.so
|
| 7 |
+
*.egg
|
| 8 |
+
*.egg-info
|
| 9 |
+
dist
|
| 10 |
+
build
|
| 11 |
+
|
| 12 |
+
.venv
|
| 13 |
+
venv
|
| 14 |
+
ENV
|
| 15 |
+
env
|
| 16 |
+
|
| 17 |
+
.git
|
| 18 |
+
.gitignore
|
| 19 |
+
.idea
|
| 20 |
+
.vscode
|
| 21 |
+
.claude
|
| 22 |
+
|
| 23 |
+
*.md
|
| 24 |
+
README.md
|
| 25 |
+
Dockerfile
|
| 26 |
+
.dockerignore
|
| 27 |
+
|
| 28 |
+
test_*.http
|
| 29 |
+
test_results
|
| 30 |
+
scripts/test_datasets
|
| 31 |
+
|
| 32 |
+
.pytest_cache
|
| 33 |
+
.coverage
|
| 34 |
+
htmlcov
|
| 35 |
+
|
| 36 |
+
*.log
|
| 37 |
+
.DS_Store
|
| 38 |
+
.python-version
|
.env.example
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# App Configuration
|
| 2 |
+
APP_NAME="ML Inference Service"
|
| 3 |
+
APP_VERSION="0.1.0"
|
| 4 |
+
DEBUG=false
|
| 5 |
+
|
| 6 |
+
# Server
|
| 7 |
+
HOST="0.0.0.0"
|
| 8 |
+
PORT=8000
|
| 9 |
+
|
| 10 |
+
# Model
|
| 11 |
+
MODEL_NAME="microsoft/resnet-18"
|
.gitignore
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
models/
|
| 2 |
+
venv/
|
.python-version
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
3.12.11
|
Dockerfile
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.12-slim as builder
|
| 2 |
+
|
| 3 |
+
WORKDIR /build
|
| 4 |
+
|
| 5 |
+
COPY requirements.txt .
|
| 6 |
+
RUN pip install --no-cache-dir --user -r requirements.txt
|
| 7 |
+
|
| 8 |
+
FROM python:3.12-slim
|
| 9 |
+
|
| 10 |
+
WORKDIR /app
|
| 11 |
+
|
| 12 |
+
RUN useradd -m -u 1000 appuser
|
| 13 |
+
|
| 14 |
+
COPY --from=builder --chown=appuser:appuser /root/.local /home/appuser/.local
|
| 15 |
+
COPY --chown=appuser:appuser app ./app
|
| 16 |
+
COPY --chown=appuser:appuser models ./models
|
| 17 |
+
COPY --chown=appuser:appuser main.py .
|
| 18 |
+
|
| 19 |
+
USER appuser
|
| 20 |
+
|
| 21 |
+
ENV PATH=/home/appuser/.local/bin:$PATH \
|
| 22 |
+
PYTHONUNBUFFERED=1
|
| 23 |
+
|
| 24 |
+
EXPOSE 8000
|
| 25 |
+
|
| 26 |
+
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
|
README.md
CHANGED
|
@@ -1,10 +1,380 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: docker
|
| 7 |
pinned: false
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: SAFE Challenge Example Submission
|
| 3 |
+
emoji: 🔒
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: purple
|
| 6 |
sdk: docker
|
| 7 |
pinned: false
|
| 8 |
+
license: apache-2.0
|
| 9 |
---
|
| 10 |
|
| 11 |
+
---
|
| 12 |
+
license: apache-2.0
|
| 13 |
+
---
|
| 14 |
+
|
| 15 |
+
# ML Inference Service
|
| 16 |
+
|
| 17 |
+
FastAPI service for serving ML models over HTTP. Comes with ResNet-18 for image classification out of the box, but you can swap in any model you want.
|
| 18 |
+
|
| 19 |
+
## Quick Start
|
| 20 |
+
|
| 21 |
+
**Local development:**
|
| 22 |
+
```bash
|
| 23 |
+
# Install dependencies
|
| 24 |
+
python -m venv .venv
|
| 25 |
+
source .venv/bin/activate
|
| 26 |
+
pip install -r requirements.txt
|
| 27 |
+
|
| 28 |
+
# Download the example model
|
| 29 |
+
bash scripts/model_download.bash
|
| 30 |
+
|
| 31 |
+
# Run it
|
| 32 |
+
uvicorn main:app --reload
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
Server runs on `http://127.0.0.1:8000`. Check `/docs` for the interactive API documentation.
|
| 36 |
+
|
| 37 |
+
**Docker:**
|
| 38 |
+
```bash
|
| 39 |
+
# Build
|
| 40 |
+
docker build -t ml-inference-service:test .
|
| 41 |
+
|
| 42 |
+
# Run
|
| 43 |
+
docker run -d --name ml-inference-test -p 8000:8000 ml-inference-service:test
|
| 44 |
+
|
| 45 |
+
# Check logs
|
| 46 |
+
docker logs -f ml-inference-test
|
| 47 |
+
|
| 48 |
+
# Stop
|
| 49 |
+
docker stop ml-inference-test && docker rm ml-inference-test
|
| 50 |
+
```
|
| 51 |
+
|
| 52 |
+
## Testing the API
|
| 53 |
+
|
| 54 |
+
```bash
|
| 55 |
+
# Using curl
|
| 56 |
+
curl -X POST http://localhost:8000/predict \
|
| 57 |
+
-H "Content-Type: application/json" \
|
| 58 |
+
-d '{
|
| 59 |
+
"image": {
|
| 60 |
+
"mediaType": "image/jpeg",
|
| 61 |
+
"data": "<base64-encoded-image>"
|
| 62 |
+
}
|
| 63 |
+
}'
|
| 64 |
+
```
|
| 65 |
+
|
| 66 |
+
Example response:
|
| 67 |
+
```json
|
| 68 |
+
{
|
| 69 |
+
"prediction": "tiger cat",
|
| 70 |
+
"confidence": 0.394,
|
| 71 |
+
"predicted_label": 282,
|
| 72 |
+
"model": "microsoft/resnet-18",
|
| 73 |
+
"mediaType": "image/jpeg"
|
| 74 |
+
}
|
| 75 |
+
```
|
| 76 |
+
|
| 77 |
+
## Project Structure
|
| 78 |
+
|
| 79 |
+
```
|
| 80 |
+
ml-inference-service/
|
| 81 |
+
├── main.py # Entry point
|
| 82 |
+
├── app/
|
| 83 |
+
│ ├── core/
|
| 84 |
+
│ │ ├── app.py # App factory, config, DI, lifecycle
|
| 85 |
+
│ │ └── logging.py # Logging setup
|
| 86 |
+
│ ├── api/
|
| 87 |
+
│ │ ├── models.py # Request/response schemas
|
| 88 |
+
│ │ ├── controllers.py # Business logic
|
| 89 |
+
│ │ └── routes/
|
| 90 |
+
│ │ └── prediction.py # POST /predict
|
| 91 |
+
│ └── services/
|
| 92 |
+
│ ├── base.py # Abstract InferenceService class
|
| 93 |
+
│ └── inference.py # ResNet implementation
|
| 94 |
+
├── models/
|
| 95 |
+
│ └── microsoft/
|
| 96 |
+
│ └── resnet-18/ # Model weights and config
|
| 97 |
+
├── scripts/
|
| 98 |
+
│ ├── model_download.bash
|
| 99 |
+
│ ├── generate_test_datasets.py
|
| 100 |
+
│ └── test_datasets.py
|
| 101 |
+
├── Dockerfile # Multi-stage build
|
| 102 |
+
├── .env.example # Environment config template
|
| 103 |
+
└── requirements.txt
|
| 104 |
+
```
|
| 105 |
+
|
| 106 |
+
The key design decision here is that `app/core/app.py` consolidates everything—config, dependency injection, lifecycle, and the app factory. This avoids the mess of managing global state across multiple files.
|
| 107 |
+
|
| 108 |
+
## How to Plug In Your Own Model
|
| 109 |
+
|
| 110 |
+
The whole service is built around one abstract base class: `InferenceService`. Implement it for your model, and everything else just works.
|
| 111 |
+
|
| 112 |
+
### Step 1: Create Your Service Class
|
| 113 |
+
|
| 114 |
+
```python
|
| 115 |
+
# app/services/your_model_service.py
|
| 116 |
+
from app.services.base import InferenceService
|
| 117 |
+
from app.api.models import ImageRequest, PredictionResponse
|
| 118 |
+
import asyncio
|
| 119 |
+
|
| 120 |
+
class YourModelService(InferenceService[ImageRequest, PredictionResponse]):
|
| 121 |
+
def __init__(self, model_name: str):
|
| 122 |
+
self.model_name = model_name
|
| 123 |
+
self.model_path = f"models/{model_name}"
|
| 124 |
+
self.model = None
|
| 125 |
+
self._is_loaded = False
|
| 126 |
+
|
| 127 |
+
async def load_model(self) -> None:
|
| 128 |
+
"""Load your model here. Called once at startup."""
|
| 129 |
+
self.model = load_your_model(self.model_path)
|
| 130 |
+
self._is_loaded = True
|
| 131 |
+
|
| 132 |
+
async def predict(self, request: ImageRequest) -> PredictionResponse:
|
| 133 |
+
"""Run inference. Offload heavy work to thread pool."""
|
| 134 |
+
return await asyncio.to_thread(self._predict_sync, request)
|
| 135 |
+
|
| 136 |
+
def _predict_sync(self, request: ImageRequest) -> PredictionResponse:
|
| 137 |
+
"""Actual inference happens here."""
|
| 138 |
+
image = decode_base64_image(request.image.data)
|
| 139 |
+
result = self.model(image)
|
| 140 |
+
|
| 141 |
+
return PredictionResponse(
|
| 142 |
+
prediction=result.label,
|
| 143 |
+
confidence=result.confidence,
|
| 144 |
+
predicted_label=result.class_id,
|
| 145 |
+
model=self.model_name,
|
| 146 |
+
mediaType=request.image.mediaType
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
@property
|
| 150 |
+
def is_loaded(self) -> bool:
|
| 151 |
+
return self._is_loaded
|
| 152 |
+
```
|
| 153 |
+
|
| 154 |
+
**Important:** Use `asyncio.to_thread()` to run CPU-heavy inference in a background thread. This keeps the server responsive while your model is working.
|
| 155 |
+
|
| 156 |
+
### Step 2: Register Your Service
|
| 157 |
+
|
| 158 |
+
Open `app/core/app.py` and find the lifespan function:
|
| 159 |
+
|
| 160 |
+
```python
|
| 161 |
+
# Change this line:
|
| 162 |
+
service = ResNetInferenceService(model_name="microsoft/resnet-18")
|
| 163 |
+
|
| 164 |
+
# To this:
|
| 165 |
+
service = YourModelService(model_name="your-org/your-model")
|
| 166 |
+
```
|
| 167 |
+
|
| 168 |
+
That's it. The `/predict` endpoint now serves your model.
|
| 169 |
+
|
| 170 |
+
### Model Files
|
| 171 |
+
|
| 172 |
+
Put your model files under `models/` with the full org/model structure:
|
| 173 |
+
|
| 174 |
+
```
|
| 175 |
+
models/
|
| 176 |
+
└── your-org/
|
| 177 |
+
└── your-model/
|
| 178 |
+
├── config.json
|
| 179 |
+
├── weights.bin
|
| 180 |
+
└── (other files)
|
| 181 |
+
```
|
| 182 |
+
|
| 183 |
+
No renaming, no dropping the org prefix—it just mirrors the Hugging Face structure.
|
| 184 |
+
|
| 185 |
+
## Configuration
|
| 186 |
+
|
| 187 |
+
Settings are managed via environment variables or a `.env` file. See `.env.example` for all available options.
|
| 188 |
+
|
| 189 |
+
**Default values:**
|
| 190 |
+
- `APP_NAME`: "ML Inference Service"
|
| 191 |
+
- `APP_VERSION`: "0.1.0"
|
| 192 |
+
- `DEBUG`: false
|
| 193 |
+
- `HOST`: "0.0.0.0"
|
| 194 |
+
- `PORT`: 8000
|
| 195 |
+
- `MODEL_NAME`: "microsoft/resnet-18"
|
| 196 |
+
|
| 197 |
+
**To customize:**
|
| 198 |
+
```bash
|
| 199 |
+
# Copy the example
|
| 200 |
+
cp .env.example .env
|
| 201 |
+
|
| 202 |
+
# Edit values
|
| 203 |
+
vim .env
|
| 204 |
+
```
|
| 205 |
+
|
| 206 |
+
Or set environment variables directly:
|
| 207 |
+
```bash
|
| 208 |
+
export MODEL_NAME="google/vit-base-patch16-224"
|
| 209 |
+
uvicorn main:app --reload
|
| 210 |
+
```
|
| 211 |
+
|
| 212 |
+
## Deployment
|
| 213 |
+
|
| 214 |
+
**Development:**
|
| 215 |
+
```bash
|
| 216 |
+
uvicorn main:app --reload
|
| 217 |
+
```
|
| 218 |
+
|
| 219 |
+
**Production:**
|
| 220 |
+
```bash
|
| 221 |
+
gunicorn main:app -w 4 -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000
|
| 222 |
+
```
|
| 223 |
+
|
| 224 |
+
The service runs on CPU by default. For GPU inference, install CUDA-enabled PyTorch and modify your service to move tensors to the GPU device.
|
| 225 |
+
|
| 226 |
+
**Docker:**
|
| 227 |
+
- Multi-stage build keeps the image small
|
| 228 |
+
- Runs as non-root user (`appuser`)
|
| 229 |
+
- Python dependencies installed in user site-packages
|
| 230 |
+
- Model files baked into the image
|
| 231 |
+
|
| 232 |
+
## What Happens When You Start the Server
|
| 233 |
+
|
| 234 |
+
```
|
| 235 |
+
INFO: Starting ML Inference Service...
|
| 236 |
+
INFO: Initializing ResNet service: models/microsoft/resnet-18
|
| 237 |
+
INFO: Loading model from models/microsoft/resnet-18
|
| 238 |
+
INFO: Model loaded: 1000 classes
|
| 239 |
+
INFO: Startup completed successfully
|
| 240 |
+
INFO: Uvicorn running on http://0.0.0.0:8000
|
| 241 |
+
```
|
| 242 |
+
|
| 243 |
+
If you see "Model directory not found", check that your model files exist at the expected path with the full org/model structure.
|
| 244 |
+
|
| 245 |
+
## API Reference
|
| 246 |
+
|
| 247 |
+
**Endpoint:** `POST /predict`
|
| 248 |
+
|
| 249 |
+
**Request:**
|
| 250 |
+
```json
|
| 251 |
+
{
|
| 252 |
+
"image": {
|
| 253 |
+
"mediaType": "image/jpeg", // or "image/png"
|
| 254 |
+
"data": "<base64-encoded-image>"
|
| 255 |
+
}
|
| 256 |
+
}
|
| 257 |
+
```
|
| 258 |
+
|
| 259 |
+
**Response:**
|
| 260 |
+
```json
|
| 261 |
+
{
|
| 262 |
+
"prediction": "string", // Human-readable label
|
| 263 |
+
"confidence": 0.0, // Softmax probability
|
| 264 |
+
"predicted_label": 0, // Numeric class index
|
| 265 |
+
"model": "org/model-name", // Model identifier
|
| 266 |
+
"mediaType": "image/jpeg" // Echoed from request
|
| 267 |
+
}
|
| 268 |
+
```
|
| 269 |
+
|
| 270 |
+
**Docs:**
|
| 271 |
+
- Swagger UI: `http://localhost:8000/docs`
|
| 272 |
+
- ReDoc: `http://localhost:8000/redoc`
|
| 273 |
+
- OpenAPI JSON: `http://localhost:8000/openapi.json`
|
| 274 |
+
|
| 275 |
+
## PyArrow Test Datasets
|
| 276 |
+
|
| 277 |
+
We've included a test dataset system for validating your model. It generates 100 standardized test cases covering normal inputs, edge cases, performance benchmarks, and model comparisons.
|
| 278 |
+
|
| 279 |
+
### Generate Datasets
|
| 280 |
+
|
| 281 |
+
```bash
|
| 282 |
+
python scripts/generate_test_datasets.py
|
| 283 |
+
```
|
| 284 |
+
|
| 285 |
+
This creates:
|
| 286 |
+
- `scripts/test_datasets/*.parquet` - Test data (images, requests, expected responses)
|
| 287 |
+
- `scripts/test_datasets/*_metadata.json` - Human-readable descriptions
|
| 288 |
+
- `scripts/test_datasets/datasets_summary.json` - Overview of all datasets
|
| 289 |
+
|
| 290 |
+
### Run Tests
|
| 291 |
+
|
| 292 |
+
```bash
|
| 293 |
+
# Start your service first
|
| 294 |
+
uvicorn main:app --reload
|
| 295 |
+
|
| 296 |
+
# Quick test (5 samples per dataset)
|
| 297 |
+
python scripts/test_datasets.py --quick
|
| 298 |
+
|
| 299 |
+
# Full validation
|
| 300 |
+
python scripts/test_datasets.py
|
| 301 |
+
|
| 302 |
+
# Test specific category
|
| 303 |
+
python scripts/test_datasets.py --category edge_case
|
| 304 |
+
```
|
| 305 |
+
|
| 306 |
+
### Dataset Categories (25 datasets each)
|
| 307 |
+
|
| 308 |
+
**1. Standard Tests** (`standard_test_*.parquet`)
|
| 309 |
+
- Normal images: random patterns, shapes, gradients
|
| 310 |
+
- Common sizes: 224x224, 256x256, 299x299, 384x384
|
| 311 |
+
- Formats: JPEG, PNG
|
| 312 |
+
- Purpose: Baseline validation
|
| 313 |
+
|
| 314 |
+
**2. Edge Cases** (`edge_case_*.parquet`)
|
| 315 |
+
- Tiny images (32x32, 1x1)
|
| 316 |
+
- Huge images (2048x2048)
|
| 317 |
+
- Extreme aspect ratios (1000x50)
|
| 318 |
+
- Corrupted data, malformed requests
|
| 319 |
+
- Purpose: Test error handling
|
| 320 |
+
|
| 321 |
+
**3. Performance Benchmarks** (`performance_test_*.parquet`)
|
| 322 |
+
- Batch sizes: 1, 5, 10, 25, 50, 100 images
|
| 323 |
+
- Latency and throughput tracking
|
| 324 |
+
- Purpose: Performance profiling
|
| 325 |
+
|
| 326 |
+
**4. Model Comparisons** (`model_comparison_*.parquet`)
|
| 327 |
+
- Same inputs across different architectures
|
| 328 |
+
- Models: ResNet-18/50, ViT, ConvNext, Swin
|
| 329 |
+
- Purpose: Cross-model benchmarking
|
| 330 |
+
|
| 331 |
+
### Test Output
|
| 332 |
+
|
| 333 |
+
```
|
| 334 |
+
DATASET TESTING SUMMARY
|
| 335 |
+
============================================================
|
| 336 |
+
Datasets tested: 100
|
| 337 |
+
Successful datasets: 95
|
| 338 |
+
Failed datasets: 5
|
| 339 |
+
Total samples: 1,247
|
| 340 |
+
Overall success rate: 87.3%
|
| 341 |
+
Test duration: 45.2s
|
| 342 |
+
|
| 343 |
+
Performance:
|
| 344 |
+
Avg latency: 123.4ms
|
| 345 |
+
Median latency: 98.7ms
|
| 346 |
+
p95 latency: 342.1ms
|
| 347 |
+
Max latency: 2,341.0ms
|
| 348 |
+
Requests/sec: 27.6
|
| 349 |
+
|
| 350 |
+
Category breakdown:
|
| 351 |
+
standard: 25 datasets, 94.2% avg success
|
| 352 |
+
edge_case: 25 datasets, 76.8% avg success
|
| 353 |
+
performance: 25 datasets, 91.1% avg success
|
| 354 |
+
model_comparison: 25 datasets, 89.3% avg success
|
| 355 |
+
```
|
| 356 |
+
|
| 357 |
+
## Common Issues
|
| 358 |
+
|
| 359 |
+
**Port 8000 already in use:**
|
| 360 |
+
```bash
|
| 361 |
+
# Find what's using it
|
| 362 |
+
lsof -i :8000
|
| 363 |
+
|
| 364 |
+
# Or just use a different port
|
| 365 |
+
uvicorn main:app --port 8080
|
| 366 |
+
```
|
| 367 |
+
|
| 368 |
+
**Model not loading:**
|
| 369 |
+
- Check the path: models should be in `models/<org>/<model-name>/`
|
| 370 |
+
- Make sure you ran `bash scripts/model_download.bash`
|
| 371 |
+
- Check logs for the exact error
|
| 372 |
+
|
| 373 |
+
**Slow inference:**
|
| 374 |
+
- Inference runs on CPU by default
|
| 375 |
+
- For GPU: install CUDA PyTorch and modify service to use GPU device
|
| 376 |
+
- Consider using smaller models or quantization
|
| 377 |
+
|
| 378 |
+
## License
|
| 379 |
+
|
| 380 |
+
Apache 2.0
|
app/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ML Inference Service
|
| 3 |
+
|
| 4 |
+
A FastAPI-based web service for machine learning model inference.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
__version__ = "0.1.0"
|
app/api/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""API layer for the ML inference service."""
|
app/api/controllers.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""API controllers for request handling and validation."""
|
| 2 |
+
|
| 3 |
+
import asyncio
|
| 4 |
+
from fastapi import HTTPException
|
| 5 |
+
|
| 6 |
+
from app.core.logging import logger
|
| 7 |
+
from app.services.base import InferenceService
|
| 8 |
+
from app.api.models import ImageRequest, PredictionResponse
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class PredictionController:
|
| 12 |
+
"""Controller for prediction endpoints."""
|
| 13 |
+
|
| 14 |
+
@staticmethod
|
| 15 |
+
async def predict(
|
| 16 |
+
request: ImageRequest,
|
| 17 |
+
service: InferenceService
|
| 18 |
+
) -> PredictionResponse:
|
| 19 |
+
"""Run inference using the configured service."""
|
| 20 |
+
try:
|
| 21 |
+
if not service or not service.is_loaded:
|
| 22 |
+
raise HTTPException(503, "Service not available")
|
| 23 |
+
|
| 24 |
+
if not request.image.mediaType.startswith('image/'):
|
| 25 |
+
raise HTTPException(400, f"Invalid media type: {request.image.mediaType}")
|
| 26 |
+
|
| 27 |
+
return await asyncio.to_thread(service.predict, request)
|
| 28 |
+
|
| 29 |
+
except HTTPException:
|
| 30 |
+
raise
|
| 31 |
+
except ValueError as e:
|
| 32 |
+
logger.error(f"Invalid input: {e}")
|
| 33 |
+
raise HTTPException(400, str(e))
|
| 34 |
+
except Exception as e:
|
| 35 |
+
logger.error(f"Prediction failed: {e}")
|
| 36 |
+
raise HTTPException(500, "Internal server error")
|
app/api/models.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Pydantic models for request/response validation.
|
| 3 |
+
"""
|
| 4 |
+
import enum
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import pydantic
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class ImageData(pydantic.BaseModel):
|
| 11 |
+
"""Image data model for base64 encoded images."""
|
| 12 |
+
mediaType: str
|
| 13 |
+
data: str
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class ImageRequest(pydantic.BaseModel):
|
| 17 |
+
"""Request model for image classification."""
|
| 18 |
+
image: ImageData
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class Labels(enum.IntEnum):
|
| 22 |
+
Natural = 0
|
| 23 |
+
FullySynthesized = 1
|
| 24 |
+
LocallyEdited = 2
|
| 25 |
+
LocallySynthesized = 3
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class LocalizationMask(pydantic.BaseModel):
|
| 29 |
+
"""A bit mask indicating which pixels are manipulated / synthesized.
|
| 30 |
+
|
| 31 |
+
A bit value of ``1`` means that the model believes the corresponding pixel
|
| 32 |
+
has been edited or synthesized (i.e., its label would be non-zero).
|
| 33 |
+
A bit value of ``0`` means that the model believes the pixel is unaltered.
|
| 34 |
+
|
| 35 |
+
The mask ``.width`` and ``.height`` should be the same as the input image.
|
| 36 |
+
Extra bits at the end of ``.bitsRowMajor`` after the first
|
| 37 |
+
``width * height`` bits are **ignored**; for simplicity/efficiency,
|
| 38 |
+
you should encode your bit mask into a byte array and not worry if the
|
| 39 |
+
final byte isn't "full", then convert the byte array to base64.
|
| 40 |
+
"""
|
| 41 |
+
|
| 42 |
+
width: int = pydantic.Field(
|
| 43 |
+
description="The width of the mask."
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
height: int = pydantic.Field(
|
| 47 |
+
description="The height of the mask."
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
bitsRowMajor: str = pydantic.Field(
|
| 51 |
+
description="A base64 string encoding the bit mask in row-major order.",
|
| 52 |
+
# Canonical base64 encoding
|
| 53 |
+
# https://stackoverflow.com/a/64467300/3709935
|
| 54 |
+
pattern=r"^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/][AQgw]==|[A-Za-z0-9+/]{2}[AEIMQUYcgkosw048]=)?$",
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
class PredictionResponse(pydantic.BaseModel):
|
| 59 |
+
"""Response model for synthetic image classification results.
|
| 60 |
+
|
| 61 |
+
Detector models will be scored primarily on their ability to classify the
|
| 62 |
+
entire image into 1 of the 4 label categories::
|
| 63 |
+
|
| 64 |
+
0: (Natural) The image is natural / unaltered.
|
| 65 |
+
1: (FullySynthesized) The entire image was synthesized by e.g., a
|
| 66 |
+
generative image model.
|
| 67 |
+
2: (LocallyEdited) The image is a natural image where a portion has
|
| 68 |
+
been edited using traditional photo editing techniques such as
|
| 69 |
+
splicing.
|
| 70 |
+
3: (LocallySynthesized) The image is a natural image where a portion
|
| 71 |
+
has been replaced by synthesized content.
|
| 72 |
+
"""
|
| 73 |
+
|
| 74 |
+
logprobs: list[float] = pydantic.Field(
|
| 75 |
+
description="The log-probabilities for each of the 4 possible labels.",
|
| 76 |
+
min_length=4,
|
| 77 |
+
max_length=4,
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
localizationMask: Optional[LocalizationMask] = pydantic.Field(
|
| 81 |
+
description="A bit mask localizing predicted edits. Models that are"
|
| 82 |
+
" not capable of localization may omit this field. It may also be"
|
| 83 |
+
" omitted if the predicted label is ``0`` or ``1``, in which case the"
|
| 84 |
+
" mask will be assumed to be all 0's or all 1's, as appropriate."
|
| 85 |
+
)
|
app/api/routes/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""API route definitions."""
|
app/api/routes/prediction.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Prediction API routes."""
|
| 2 |
+
|
| 3 |
+
from fastapi import APIRouter, Depends
|
| 4 |
+
|
| 5 |
+
from app.api.controllers import PredictionController
|
| 6 |
+
from app.api.models import ImageRequest, PredictionResponse
|
| 7 |
+
from app.core.dependencies import get_inference_service
|
| 8 |
+
from app.services.base import InferenceService
|
| 9 |
+
|
| 10 |
+
router = APIRouter()
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@router.post("/predict", response_model=PredictionResponse)
|
| 14 |
+
async def predict(
|
| 15 |
+
request: ImageRequest,
|
| 16 |
+
service: InferenceService = Depends(get_inference_service)
|
| 17 |
+
):
|
| 18 |
+
"""
|
| 19 |
+
Run inference on base64-encoded image.
|
| 20 |
+
|
| 21 |
+
Returns prediction, confidence, predicted label, model name, and media type.
|
| 22 |
+
"""
|
| 23 |
+
return await PredictionController.predict(request, service)
|
app/core/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Core utilities and configurations."""
|
app/core/app.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FastAPI application factory and core infrastructure."""
|
| 2 |
+
|
| 3 |
+
import asyncio
|
| 4 |
+
import warnings
|
| 5 |
+
from contextlib import asynccontextmanager
|
| 6 |
+
from typing import AsyncGenerator, Optional
|
| 7 |
+
|
| 8 |
+
from fastapi import FastAPI
|
| 9 |
+
from pydantic import Field
|
| 10 |
+
from pydantic_settings import BaseSettings
|
| 11 |
+
|
| 12 |
+
from app.core.logging import logger
|
| 13 |
+
from app.core.dependencies import set_inference_service
|
| 14 |
+
from app.services.inference import ResNetInferenceService
|
| 15 |
+
from app.api.routes import prediction
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class Settings(BaseSettings):
|
| 19 |
+
"""Application settings. Override via environment variables or .env file."""
|
| 20 |
+
|
| 21 |
+
app_name: str = Field(default="ML Inference Service")
|
| 22 |
+
app_version: str = Field(default="0.1.0")
|
| 23 |
+
debug: bool = Field(default=False)
|
| 24 |
+
host: str = Field(default="0.0.0.0")
|
| 25 |
+
port: int = Field(default=8000)
|
| 26 |
+
|
| 27 |
+
class Config:
|
| 28 |
+
env_file = ".env"
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
settings = Settings()
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@asynccontextmanager
|
| 35 |
+
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
| 36 |
+
"""Application lifecycle: startup/shutdown."""
|
| 37 |
+
logger.info("Starting ML Inference Service...")
|
| 38 |
+
|
| 39 |
+
try:
|
| 40 |
+
with warnings.catch_warnings():
|
| 41 |
+
warnings.filterwarnings("ignore", category=FutureWarning)
|
| 42 |
+
|
| 43 |
+
# Replace ResNetInferenceService with your own implementation
|
| 44 |
+
service = ResNetInferenceService(model_name="microsoft/resnet-18")
|
| 45 |
+
await asyncio.to_thread(service.load_model)
|
| 46 |
+
set_inference_service(service)
|
| 47 |
+
|
| 48 |
+
logger.info("Startup completed successfully")
|
| 49 |
+
|
| 50 |
+
except Exception as e:
|
| 51 |
+
logger.error(f"Startup failed: {e}")
|
| 52 |
+
raise
|
| 53 |
+
|
| 54 |
+
yield
|
| 55 |
+
|
| 56 |
+
logger.info("Shutting down...")
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def create_app() -> FastAPI:
|
| 60 |
+
"""Create and configure FastAPI application."""
|
| 61 |
+
app = FastAPI(
|
| 62 |
+
title=settings.app_name,
|
| 63 |
+
description="ML inference service for image classification",
|
| 64 |
+
version=settings.app_version,
|
| 65 |
+
debug=settings.debug,
|
| 66 |
+
lifespan=lifespan
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
app.include_router(prediction.router)
|
| 70 |
+
|
| 71 |
+
return app
|
app/core/dependencies.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Dependency injection for services."""
|
| 2 |
+
|
| 3 |
+
from typing import Optional
|
| 4 |
+
from app.services.base import InferenceService
|
| 5 |
+
|
| 6 |
+
_inference_service: Optional[InferenceService] = None
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def get_inference_service() -> Optional[InferenceService]:
|
| 10 |
+
"""Get inference service for dependency injection."""
|
| 11 |
+
return _inference_service
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def set_inference_service(service: InferenceService) -> None:
|
| 15 |
+
"""Set inference service. Called internally during startup."""
|
| 16 |
+
global _inference_service
|
| 17 |
+
_inference_service = service
|
app/core/logging.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Logging configuration."""
|
| 2 |
+
|
| 3 |
+
import logging
|
| 4 |
+
import sys
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def setup_logging(logger_name: str = "ML Inference Service") -> logging.Logger:
|
| 8 |
+
"""Setup and configure logger."""
|
| 9 |
+
logger = logging.getLogger(logger_name)
|
| 10 |
+
|
| 11 |
+
if logger.handlers:
|
| 12 |
+
return logger
|
| 13 |
+
|
| 14 |
+
logger.setLevel(logging.INFO)
|
| 15 |
+
handler = logging.StreamHandler(sys.stdout)
|
| 16 |
+
handler.setLevel(logging.INFO)
|
| 17 |
+
formatter = logging.Formatter(
|
| 18 |
+
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
| 19 |
+
)
|
| 20 |
+
handler.setFormatter(formatter)
|
| 21 |
+
logger.addHandler(handler)
|
| 22 |
+
|
| 23 |
+
return logger
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
logger = setup_logging()
|
app/services/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Business logic services."""
|
app/services/base.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Abstract base class for ML inference services."""
|
| 2 |
+
|
| 3 |
+
from abc import ABC, abstractmethod
|
| 4 |
+
from typing import Generic, TypeVar
|
| 5 |
+
from pydantic import BaseModel
|
| 6 |
+
|
| 7 |
+
TRequest = TypeVar('TRequest', bound=BaseModel)
|
| 8 |
+
TResponse = TypeVar('TResponse', bound=BaseModel)
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class InferenceService(ABC, Generic[TRequest, TResponse]):
|
| 12 |
+
"""
|
| 13 |
+
Base class for inference services. Subclass this to integrate your model.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
@abstractmethod
|
| 17 |
+
def load_model(self) -> None:
|
| 18 |
+
"""Load model weights and processors. Called once at startup."""
|
| 19 |
+
pass
|
| 20 |
+
|
| 21 |
+
@abstractmethod
|
| 22 |
+
def predict(self, request: TRequest) -> TResponse:
|
| 23 |
+
"""Run inference and return typed response."""
|
| 24 |
+
pass
|
| 25 |
+
|
| 26 |
+
@property
|
| 27 |
+
@abstractmethod
|
| 28 |
+
def is_loaded(self) -> bool:
|
| 29 |
+
"""Check if model is loaded and ready."""
|
| 30 |
+
pass
|
app/services/inference.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ResNet inference service implementation."""
|
| 2 |
+
|
| 3 |
+
import base64
|
| 4 |
+
import os
|
| 5 |
+
import random
|
| 6 |
+
from io import BytesIO
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from PIL import Image
|
| 10 |
+
from transformers import AutoImageProcessor, ResNetForImageClassification # type: ignore[import-untyped]
|
| 11 |
+
|
| 12 |
+
from app.core.logging import logger
|
| 13 |
+
from app.services.base import InferenceService
|
| 14 |
+
from app.api.models import ImageRequest, Labels, LocalizationMask, PredictionResponse
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class ResNetInferenceService(InferenceService[ImageRequest, PredictionResponse]):
|
| 18 |
+
"""ResNet-18 inference service for image classification."""
|
| 19 |
+
|
| 20 |
+
def __init__(self, model_name: str = "microsoft/resnet-18"):
|
| 21 |
+
self.model_name = model_name
|
| 22 |
+
self.model = None
|
| 23 |
+
self.processor = None
|
| 24 |
+
self._is_loaded = False
|
| 25 |
+
self.model_path = os.path.join("models", model_name)
|
| 26 |
+
logger.info(f"Initializing ResNet service: {self.model_path}")
|
| 27 |
+
|
| 28 |
+
def load_model(self) -> None:
|
| 29 |
+
if self._is_loaded:
|
| 30 |
+
return
|
| 31 |
+
|
| 32 |
+
if not os.path.exists(self.model_path):
|
| 33 |
+
raise FileNotFoundError(f"Model not found: {self.model_path}")
|
| 34 |
+
|
| 35 |
+
config_path = os.path.join(self.model_path, "config.json")
|
| 36 |
+
if not os.path.exists(config_path):
|
| 37 |
+
raise FileNotFoundError(f"Config not found: {config_path}")
|
| 38 |
+
|
| 39 |
+
logger.info(f"Loading model from {self.model_path}")
|
| 40 |
+
|
| 41 |
+
import warnings
|
| 42 |
+
with warnings.catch_warnings():
|
| 43 |
+
warnings.filterwarnings("ignore", category=FutureWarning)
|
| 44 |
+
self.processor = AutoImageProcessor.from_pretrained(
|
| 45 |
+
self.model_path, local_files_only=True
|
| 46 |
+
)
|
| 47 |
+
self.model = ResNetForImageClassification.from_pretrained(
|
| 48 |
+
self.model_path, local_files_only=True
|
| 49 |
+
)
|
| 50 |
+
assert self.model is not None
|
| 51 |
+
|
| 52 |
+
self._is_loaded = True
|
| 53 |
+
logger.info(f"Model loaded: {len(self.model.config.id2label)} classes") # pyright: ignore
|
| 54 |
+
|
| 55 |
+
def predict(self, request: ImageRequest) -> PredictionResponse:
|
| 56 |
+
if not self.is_loaded:
|
| 57 |
+
raise RuntimeError("model is not loaded")
|
| 58 |
+
assert self.processor is not None
|
| 59 |
+
assert self.model is not None
|
| 60 |
+
|
| 61 |
+
image_data = base64.b64decode(request.image.data)
|
| 62 |
+
image = Image.open(BytesIO(image_data))
|
| 63 |
+
width, height = image.size
|
| 64 |
+
|
| 65 |
+
if image.mode != 'RGB':
|
| 66 |
+
image = image.convert('RGB')
|
| 67 |
+
|
| 68 |
+
inputs = self.processor(image, return_tensors="pt")
|
| 69 |
+
|
| 70 |
+
with torch.no_grad():
|
| 71 |
+
logits = self.model(**inputs).logits # pyright: ignore
|
| 72 |
+
|
| 73 |
+
logprobs = torch.nn.functional.log_softmax(logits[:len(Labels)], dim=-1).tolist()
|
| 74 |
+
mask_bytes = random.randbytes((width*height + 7) // 8)
|
| 75 |
+
mask_bits = base64.b64encode(mask_bytes).decode("utf-8")
|
| 76 |
+
|
| 77 |
+
return PredictionResponse(
|
| 78 |
+
logprobs=logprobs,
|
| 79 |
+
localizationMask=LocalizationMask(
|
| 80 |
+
width=width, height=height, bitsRowMajor=mask_bits
|
| 81 |
+
)
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
@property
|
| 85 |
+
def is_loaded(self) -> bool:
|
| 86 |
+
return self._is_loaded
|
main.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Main FastAPI application entry point.
|
| 3 |
+
"""
|
| 4 |
+
from app.core.app import create_app
|
| 5 |
+
|
| 6 |
+
app = create_app()
|
requirements.in
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Web framework
|
| 2 |
+
fastapi==0.104.1
|
| 3 |
+
uvicorn[standard]==0.24.0
|
| 4 |
+
|
| 5 |
+
# Configuration management
|
| 6 |
+
pydantic==2.5.0
|
| 7 |
+
pydantic-settings==2.0.3
|
| 8 |
+
python-dotenv==0.21.0
|
| 9 |
+
|
| 10 |
+
# File upload handling
|
| 11 |
+
python-multipart==0.0.6
|
| 12 |
+
|
| 13 |
+
# ML/AI dependencies (newly added)
|
| 14 |
+
transformers>=4.35.0
|
| 15 |
+
torch>=2.4.0 # Newer PyTorch with NumPy 2.x support
|
| 16 |
+
pillow>=10.0.0
|
| 17 |
+
|
| 18 |
+
# Dataset generation and testing
|
| 19 |
+
pyarrow>=14.0.0
|
| 20 |
+
numpy>=1.24.0
|
| 21 |
+
pandas>=2.0.0
|
| 22 |
+
requests>=2.25.0
|
requirements.txt
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#
|
| 2 |
+
# This file is autogenerated by pip-compile with Python 3.12
|
| 3 |
+
# by the following command:
|
| 4 |
+
#
|
| 5 |
+
# pip-compile requirements.in
|
| 6 |
+
#
|
| 7 |
+
annotated-types==0.7.0
|
| 8 |
+
# via pydantic
|
| 9 |
+
anyio==3.7.1
|
| 10 |
+
# via
|
| 11 |
+
# fastapi
|
| 12 |
+
# starlette
|
| 13 |
+
# watchfiles
|
| 14 |
+
certifi==2025.8.3
|
| 15 |
+
# via requests
|
| 16 |
+
charset-normalizer==3.4.3
|
| 17 |
+
# via requests
|
| 18 |
+
click==8.2.1
|
| 19 |
+
# via uvicorn
|
| 20 |
+
fastapi==0.104.1
|
| 21 |
+
# via -r requirements.in
|
| 22 |
+
filelock==3.19.1
|
| 23 |
+
# via
|
| 24 |
+
# huggingface-hub
|
| 25 |
+
# torch
|
| 26 |
+
# transformers
|
| 27 |
+
fsspec==2025.7.0
|
| 28 |
+
# via
|
| 29 |
+
# huggingface-hub
|
| 30 |
+
# torch
|
| 31 |
+
h11==0.16.0
|
| 32 |
+
# via uvicorn
|
| 33 |
+
hf-xet==1.1.8
|
| 34 |
+
# via huggingface-hub
|
| 35 |
+
httptools==0.6.4
|
| 36 |
+
# via uvicorn
|
| 37 |
+
huggingface-hub==0.34.4
|
| 38 |
+
# via
|
| 39 |
+
# tokenizers
|
| 40 |
+
# transformers
|
| 41 |
+
idna==3.10
|
| 42 |
+
# via
|
| 43 |
+
# anyio
|
| 44 |
+
# requests
|
| 45 |
+
jinja2==3.1.6
|
| 46 |
+
# via torch
|
| 47 |
+
markupsafe==3.0.2
|
| 48 |
+
# via jinja2
|
| 49 |
+
mpmath==1.3.0
|
| 50 |
+
# via sympy
|
| 51 |
+
networkx==3.5
|
| 52 |
+
# via torch
|
| 53 |
+
numpy==2.3.2
|
| 54 |
+
# via
|
| 55 |
+
# -r requirements.in
|
| 56 |
+
# pandas
|
| 57 |
+
# transformers
|
| 58 |
+
nvidia-cublas-cu12==12.8.4.1
|
| 59 |
+
# via
|
| 60 |
+
# nvidia-cudnn-cu12
|
| 61 |
+
# nvidia-cusolver-cu12
|
| 62 |
+
# torch
|
| 63 |
+
nvidia-cuda-cupti-cu12==12.8.90
|
| 64 |
+
# via torch
|
| 65 |
+
nvidia-cuda-nvrtc-cu12==12.8.93
|
| 66 |
+
# via torch
|
| 67 |
+
nvidia-cuda-runtime-cu12==12.8.90
|
| 68 |
+
# via torch
|
| 69 |
+
nvidia-cudnn-cu12==9.10.2.21
|
| 70 |
+
# via torch
|
| 71 |
+
nvidia-cufft-cu12==11.3.3.83
|
| 72 |
+
# via torch
|
| 73 |
+
nvidia-cufile-cu12==1.13.1.3
|
| 74 |
+
# via torch
|
| 75 |
+
nvidia-curand-cu12==10.3.9.90
|
| 76 |
+
# via torch
|
| 77 |
+
nvidia-cusolver-cu12==11.7.3.90
|
| 78 |
+
# via torch
|
| 79 |
+
nvidia-cusparse-cu12==12.5.8.93
|
| 80 |
+
# via
|
| 81 |
+
# nvidia-cusolver-cu12
|
| 82 |
+
# torch
|
| 83 |
+
nvidia-cusparselt-cu12==0.7.1
|
| 84 |
+
# via torch
|
| 85 |
+
nvidia-nccl-cu12==2.27.3
|
| 86 |
+
# via torch
|
| 87 |
+
nvidia-nvjitlink-cu12==12.8.93
|
| 88 |
+
# via
|
| 89 |
+
# nvidia-cufft-cu12
|
| 90 |
+
# nvidia-cusolver-cu12
|
| 91 |
+
# nvidia-cusparse-cu12
|
| 92 |
+
# torch
|
| 93 |
+
nvidia-nvtx-cu12==12.8.90
|
| 94 |
+
# via torch
|
| 95 |
+
packaging==25.0
|
| 96 |
+
# via
|
| 97 |
+
# huggingface-hub
|
| 98 |
+
# transformers
|
| 99 |
+
pandas==2.3.2
|
| 100 |
+
# via -r requirements.in
|
| 101 |
+
pillow==10.1.0
|
| 102 |
+
# via -r requirements.in
|
| 103 |
+
pyarrow==21.0.0
|
| 104 |
+
# via -r requirements.in
|
| 105 |
+
pydantic==2.5.0
|
| 106 |
+
# via
|
| 107 |
+
# -r requirements.in
|
| 108 |
+
# fastapi
|
| 109 |
+
# pydantic-settings
|
| 110 |
+
pydantic-core==2.14.1
|
| 111 |
+
# via pydantic
|
| 112 |
+
pydantic-settings==2.0.3
|
| 113 |
+
# via -r requirements.in
|
| 114 |
+
python-dateutil==2.9.0.post0
|
| 115 |
+
# via pandas
|
| 116 |
+
python-dotenv==0.21.0
|
| 117 |
+
# via
|
| 118 |
+
# -r requirements.in
|
| 119 |
+
# pydantic-settings
|
| 120 |
+
# uvicorn
|
| 121 |
+
python-multipart==0.0.6
|
| 122 |
+
# via -r requirements.in
|
| 123 |
+
pytz==2025.2
|
| 124 |
+
# via pandas
|
| 125 |
+
pyyaml==6.0.2
|
| 126 |
+
# via
|
| 127 |
+
# huggingface-hub
|
| 128 |
+
# transformers
|
| 129 |
+
# uvicorn
|
| 130 |
+
regex==2025.7.34
|
| 131 |
+
# via transformers
|
| 132 |
+
requests==2.32.5
|
| 133 |
+
# via
|
| 134 |
+
# -r requirements.in
|
| 135 |
+
# huggingface-hub
|
| 136 |
+
# transformers
|
| 137 |
+
safetensors==0.6.2
|
| 138 |
+
# via transformers
|
| 139 |
+
six==1.17.0
|
| 140 |
+
# via python-dateutil
|
| 141 |
+
sniffio==1.3.1
|
| 142 |
+
# via anyio
|
| 143 |
+
starlette==0.27.0
|
| 144 |
+
# via fastapi
|
| 145 |
+
sympy==1.14.0
|
| 146 |
+
# via torch
|
| 147 |
+
tokenizers==0.15.2
|
| 148 |
+
# via transformers
|
| 149 |
+
torch==2.8.0
|
| 150 |
+
# via -r requirements.in
|
| 151 |
+
tqdm==4.67.1
|
| 152 |
+
# via
|
| 153 |
+
# huggingface-hub
|
| 154 |
+
# transformers
|
| 155 |
+
transformers==4.35.2
|
| 156 |
+
# via -r requirements.in
|
| 157 |
+
triton==3.4.0
|
| 158 |
+
# via torch
|
| 159 |
+
typing-extensions==4.15.0
|
| 160 |
+
# via
|
| 161 |
+
# fastapi
|
| 162 |
+
# huggingface-hub
|
| 163 |
+
# pydantic
|
| 164 |
+
# pydantic-core
|
| 165 |
+
# torch
|
| 166 |
+
tzdata==2025.2
|
| 167 |
+
# via pandas
|
| 168 |
+
urllib3==2.5.0
|
| 169 |
+
# via requests
|
| 170 |
+
uvicorn[standard]==0.24.0
|
| 171 |
+
# via -r requirements.in
|
| 172 |
+
uvloop==0.21.0
|
| 173 |
+
# via uvicorn
|
| 174 |
+
watchfiles==1.1.0
|
| 175 |
+
# via uvicorn
|
| 176 |
+
websockets==15.0.1
|
| 177 |
+
# via uvicorn
|
| 178 |
+
|
| 179 |
+
# The following packages are considered to be unsafe in a requirements file:
|
| 180 |
+
# setuptools
|
scripts/generate_test_datasets.py
ADDED
|
@@ -0,0 +1,411 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
PyArrow Dataset Generator for ML Inference Service
|
| 4 |
+
|
| 5 |
+
Generates test datasets for academic challenges and model validation.
|
| 6 |
+
Creates 100 PyArrow datasets with various image types and test scenarios.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import base64
|
| 10 |
+
import json
|
| 11 |
+
import random
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
from typing import Dict, List, Any, Tuple
|
| 14 |
+
import io
|
| 15 |
+
|
| 16 |
+
import numpy as np
|
| 17 |
+
import pyarrow as pa
|
| 18 |
+
import pyarrow.parquet as pq
|
| 19 |
+
from PIL import Image, ImageDraw, ImageFont
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class TestDatasetGenerator:
|
| 23 |
+
def __init__(self, output_dir: str = "test_datasets"):
|
| 24 |
+
self.output_dir = Path(output_dir)
|
| 25 |
+
self.output_dir.mkdir(exist_ok=True)
|
| 26 |
+
|
| 27 |
+
# ImageNet class labels (sample for testing)
|
| 28 |
+
self.imagenet_labels = [
|
| 29 |
+
"tench", "goldfish", "great_white_shark", "tiger_shark", "hammerhead",
|
| 30 |
+
"electric_ray", "stingray", "cock", "hen", "ostrich", "brambling",
|
| 31 |
+
"goldfinch", "house_finch", "junco", "indigo_bunting", "robin",
|
| 32 |
+
"bulbul", "jay", "magpie", "chickadee", "water_ouzel", "kite",
|
| 33 |
+
"bald_eagle", "vulture", "great_grey_owl", "European_fire_salamander",
|
| 34 |
+
"common_newt", "eft", "spotted_salamander", "axolotl", "bullfrog",
|
| 35 |
+
"tree_frog", "tailed_frog", "loggerhead", "leatherback_turtle",
|
| 36 |
+
"mud_turtle", "terrapin", "box_turtle", "banded_gecko", "common_iguana",
|
| 37 |
+
"American_chameleon", "whiptail", "agama", "frilled_lizard", "alligator_lizard",
|
| 38 |
+
"Gila_monster", "green_lizard", "African_chameleon", "Komodo_dragon",
|
| 39 |
+
"African_crocodile", "American_alligator", "triceratops", "thunder_snake"
|
| 40 |
+
]
|
| 41 |
+
|
| 42 |
+
def create_synthetic_image(self, width: int = 224, height: int = 224,
|
| 43 |
+
image_type: str = "random") -> Image.Image:
|
| 44 |
+
"""Create synthetic images for testing."""
|
| 45 |
+
if image_type == "random":
|
| 46 |
+
# Random noise image
|
| 47 |
+
array = np.random.randint(0, 256, (height, width, 3), dtype=np.uint8)
|
| 48 |
+
return Image.fromarray(array)
|
| 49 |
+
|
| 50 |
+
elif image_type == "geometric":
|
| 51 |
+
# Geometric patterns
|
| 52 |
+
img = Image.new('RGB', (width, height), color='white')
|
| 53 |
+
draw = ImageDraw.Draw(img)
|
| 54 |
+
|
| 55 |
+
# Draw random shapes
|
| 56 |
+
for _ in range(random.randint(3, 8)):
|
| 57 |
+
color = tuple(random.randint(0, 255) for _ in range(3))
|
| 58 |
+
shape_type = random.choice(['rectangle', 'ellipse'])
|
| 59 |
+
x1, y1 = random.randint(0, width//2), random.randint(0, height//2)
|
| 60 |
+
x2, y2 = x1 + random.randint(20, width//2), y1 + random.randint(20, height//2)
|
| 61 |
+
|
| 62 |
+
if shape_type == 'rectangle':
|
| 63 |
+
draw.rectangle([x1, y1, x2, y2], fill=color)
|
| 64 |
+
else:
|
| 65 |
+
draw.ellipse([x1, y1, x2, y2], fill=color)
|
| 66 |
+
|
| 67 |
+
return img
|
| 68 |
+
|
| 69 |
+
elif image_type == "gradient":
|
| 70 |
+
array = np.zeros((height, width, 3), dtype=np.uint8)
|
| 71 |
+
for i in range(height):
|
| 72 |
+
for j in range(width):
|
| 73 |
+
array[i, j] = [i * 255 // height, j * 255 // width, (i + j) * 255 // (height + width)]
|
| 74 |
+
return Image.fromarray(array)
|
| 75 |
+
|
| 76 |
+
elif image_type == "text":
|
| 77 |
+
img = Image.new('RGB', (width, height), color='white')
|
| 78 |
+
draw = ImageDraw.Draw(img)
|
| 79 |
+
|
| 80 |
+
try:
|
| 81 |
+
font = ImageFont.load_default()
|
| 82 |
+
except:
|
| 83 |
+
font = None
|
| 84 |
+
|
| 85 |
+
text = f"Test Image {random.randint(1, 1000)}"
|
| 86 |
+
draw.text((width//4, height//2), text, fill='black', font=font)
|
| 87 |
+
return img
|
| 88 |
+
|
| 89 |
+
else:
|
| 90 |
+
color = tuple(random.randint(0, 255) for _ in range(3))
|
| 91 |
+
return Image.new('RGB', (width, height), color=color)
|
| 92 |
+
|
| 93 |
+
def image_to_base64(self, image: Image.Image, format: str = "JPEG") -> str:
|
| 94 |
+
"""Convert PIL image to base64 string."""
|
| 95 |
+
buffer = io.BytesIO()
|
| 96 |
+
image.save(buffer, format=format)
|
| 97 |
+
image_bytes = buffer.getvalue()
|
| 98 |
+
return base64.b64encode(image_bytes).decode('utf-8')
|
| 99 |
+
|
| 100 |
+
def create_api_request(self, image_b64: str, media_type: str = "image/jpeg") -> Dict[str, Any]:
|
| 101 |
+
"""Create API request structure matching your service."""
|
| 102 |
+
return {
|
| 103 |
+
"image": {
|
| 104 |
+
"mediaType": media_type,
|
| 105 |
+
"data": image_b64
|
| 106 |
+
}
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
def create_expected_response(self, model_name: str = "microsoft/resnet-18",
|
| 110 |
+
media_type: str = "image/jpeg") -> Dict[str, Any]:
|
| 111 |
+
"""Create expected response structure."""
|
| 112 |
+
prediction = random.choice(self.imagenet_labels)
|
| 113 |
+
return {
|
| 114 |
+
"prediction": prediction,
|
| 115 |
+
"confidence": round(random.uniform(0.3, 0.99), 4),
|
| 116 |
+
"predicted_label": random.randint(0, len(self.imagenet_labels) - 1),
|
| 117 |
+
"model": model_name,
|
| 118 |
+
"mediaType": media_type
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
def generate_standard_datasets(self, count: int = 25) -> List[Dict[str, Any]]:
|
| 122 |
+
"""Generate standard test cases with normal images."""
|
| 123 |
+
datasets = []
|
| 124 |
+
|
| 125 |
+
for i in range(count):
|
| 126 |
+
image_types = ["random", "geometric", "gradient", "text", "solid"]
|
| 127 |
+
sizes = [(224, 224), (256, 256), (299, 299), (384, 384)]
|
| 128 |
+
formats = [("JPEG", "image/jpeg"), ("PNG", "image/png")]
|
| 129 |
+
|
| 130 |
+
records = []
|
| 131 |
+
for j in range(random.randint(5, 20)): # 5-20 images per dataset
|
| 132 |
+
img_type = random.choice(image_types)
|
| 133 |
+
size = random.choice(sizes)
|
| 134 |
+
format_info = random.choice(formats)
|
| 135 |
+
|
| 136 |
+
image = self.create_synthetic_image(size[0], size[1], img_type)
|
| 137 |
+
image_b64 = self.image_to_base64(image, format_info[0])
|
| 138 |
+
|
| 139 |
+
api_request = self.create_api_request(image_b64, format_info[1])
|
| 140 |
+
expected_response = self.create_expected_response()
|
| 141 |
+
|
| 142 |
+
record = {
|
| 143 |
+
"dataset_id": f"standard_{i:03d}",
|
| 144 |
+
"image_id": f"img_{j:03d}",
|
| 145 |
+
"image_type": img_type,
|
| 146 |
+
"image_size": f"{size[0]}x{size[1]}",
|
| 147 |
+
"format": format_info[0],
|
| 148 |
+
"media_type": format_info[1],
|
| 149 |
+
"api_request": json.dumps(api_request),
|
| 150 |
+
"expected_response": json.dumps(expected_response),
|
| 151 |
+
"test_category": "standard",
|
| 152 |
+
"difficulty": "normal"
|
| 153 |
+
}
|
| 154 |
+
records.append(record)
|
| 155 |
+
|
| 156 |
+
datasets.append({
|
| 157 |
+
"name": f"standard_test_{i:03d}",
|
| 158 |
+
"category": "standard",
|
| 159 |
+
"description": f"Standard test dataset {i+1} with {len(records)} images",
|
| 160 |
+
"records": records
|
| 161 |
+
})
|
| 162 |
+
|
| 163 |
+
return datasets
|
| 164 |
+
|
| 165 |
+
def generate_edge_case_datasets(self, count: int = 25) -> List[Dict[str, Any]]:
|
| 166 |
+
"""Generate datasets for edge case scenarios."""
|
| 167 |
+
datasets = []
|
| 168 |
+
|
| 169 |
+
for i in range(count):
|
| 170 |
+
records = []
|
| 171 |
+
edge_cases = [
|
| 172 |
+
{"type": "tiny", "size": (32, 32), "difficulty": "high"},
|
| 173 |
+
{"type": "huge", "size": (2048, 2048), "difficulty": "high"},
|
| 174 |
+
{"type": "extreme_aspect", "size": (1000, 50), "difficulty": "medium"},
|
| 175 |
+
{"type": "single_pixel", "size": (1, 1), "difficulty": "extreme"},
|
| 176 |
+
{"type": "corrupted_base64", "size": (224, 224), "difficulty": "extreme"}
|
| 177 |
+
]
|
| 178 |
+
|
| 179 |
+
for j, edge_case in enumerate(edge_cases):
|
| 180 |
+
if edge_case["type"] == "corrupted_base64":
|
| 181 |
+
image = self.create_synthetic_image(224, 224, "random")
|
| 182 |
+
image_b64 = self.image_to_base64(image, "JPEG")
|
| 183 |
+
corrupted_b64 = image_b64[:-20] + "CORRUPTED_DATA"
|
| 184 |
+
api_request = self.create_api_request(corrupted_b64)
|
| 185 |
+
expected_response = {
|
| 186 |
+
"error": "Invalid image data",
|
| 187 |
+
"status": "failed"
|
| 188 |
+
}
|
| 189 |
+
else:
|
| 190 |
+
image = self.create_synthetic_image(
|
| 191 |
+
edge_case["size"][0], edge_case["size"][1], "random"
|
| 192 |
+
)
|
| 193 |
+
image_b64 = self.image_to_base64(image, "PNG")
|
| 194 |
+
api_request = self.create_api_request(image_b64, "image/png")
|
| 195 |
+
expected_response = self.create_expected_response()
|
| 196 |
+
|
| 197 |
+
record = {
|
| 198 |
+
"dataset_id": f"edge_{i:03d}",
|
| 199 |
+
"image_id": f"edge_{j:03d}",
|
| 200 |
+
"image_type": edge_case["type"],
|
| 201 |
+
"image_size": f"{edge_case['size'][0]}x{edge_case['size'][1]}",
|
| 202 |
+
"format": "PNG",
|
| 203 |
+
"media_type": "image/png",
|
| 204 |
+
"api_request": json.dumps(api_request),
|
| 205 |
+
"expected_response": json.dumps(expected_response),
|
| 206 |
+
"test_category": "edge_case",
|
| 207 |
+
"difficulty": edge_case["difficulty"]
|
| 208 |
+
}
|
| 209 |
+
records.append(record)
|
| 210 |
+
|
| 211 |
+
datasets.append({
|
| 212 |
+
"name": f"edge_case_{i:03d}",
|
| 213 |
+
"category": "edge_case",
|
| 214 |
+
"description": f"Edge case dataset {i+1} with challenging scenarios",
|
| 215 |
+
"records": records
|
| 216 |
+
})
|
| 217 |
+
|
| 218 |
+
return datasets
|
| 219 |
+
|
| 220 |
+
def generate_performance_datasets(self, count: int = 25) -> List[Dict[str, Any]]:
|
| 221 |
+
"""Generate performance benchmark datasets."""
|
| 222 |
+
datasets = []
|
| 223 |
+
|
| 224 |
+
for i in range(count):
|
| 225 |
+
batch_sizes = [1, 5, 10, 25, 50, 100]
|
| 226 |
+
batch_size = random.choice(batch_sizes)
|
| 227 |
+
|
| 228 |
+
records = []
|
| 229 |
+
for j in range(batch_size):
|
| 230 |
+
image = self.create_synthetic_image(224, 224, "random")
|
| 231 |
+
image_b64 = self.image_to_base64(image, "JPEG")
|
| 232 |
+
api_request = self.create_api_request(image_b64)
|
| 233 |
+
expected_response = self.create_expected_response()
|
| 234 |
+
|
| 235 |
+
record = {
|
| 236 |
+
"dataset_id": f"perf_{i:03d}",
|
| 237 |
+
"image_id": f"batch_{j:03d}",
|
| 238 |
+
"image_type": "performance_test",
|
| 239 |
+
"image_size": "224x224",
|
| 240 |
+
"format": "JPEG",
|
| 241 |
+
"media_type": "image/jpeg",
|
| 242 |
+
"api_request": json.dumps(api_request),
|
| 243 |
+
"expected_response": json.dumps(expected_response),
|
| 244 |
+
"test_category": "performance",
|
| 245 |
+
"difficulty": "normal",
|
| 246 |
+
"batch_size": batch_size,
|
| 247 |
+
"expected_max_latency_ms": batch_size * 100
|
| 248 |
+
}
|
| 249 |
+
records.append(record)
|
| 250 |
+
|
| 251 |
+
datasets.append({
|
| 252 |
+
"name": f"performance_test_{i:03d}",
|
| 253 |
+
"category": "performance",
|
| 254 |
+
"description": f"Performance dataset {i+1} with batch size {batch_size}",
|
| 255 |
+
"records": records
|
| 256 |
+
})
|
| 257 |
+
|
| 258 |
+
return datasets
|
| 259 |
+
|
| 260 |
+
def generate_model_comparison_datasets(self, count: int = 25) -> List[Dict[str, Any]]:
|
| 261 |
+
"""Generate datasets for comparing different models."""
|
| 262 |
+
datasets = []
|
| 263 |
+
|
| 264 |
+
model_types = [
|
| 265 |
+
"microsoft/resnet-18", "microsoft/resnet-50", "google/vit-base-patch16-224",
|
| 266 |
+
"facebook/convnext-tiny-224", "microsoft/swin-tiny-patch4-window7-224"
|
| 267 |
+
]
|
| 268 |
+
|
| 269 |
+
for i in range(count):
|
| 270 |
+
# Same images tested across different model types
|
| 271 |
+
base_images = []
|
| 272 |
+
for _ in range(10): # 10 base images per comparison dataset
|
| 273 |
+
image = self.create_synthetic_image(224, 224, "geometric")
|
| 274 |
+
base_images.append(self.image_to_base64(image, "JPEG"))
|
| 275 |
+
|
| 276 |
+
records = []
|
| 277 |
+
for j, model in enumerate(model_types):
|
| 278 |
+
for k, image_b64 in enumerate(base_images):
|
| 279 |
+
api_request = self.create_api_request(image_b64)
|
| 280 |
+
expected_response = self.create_expected_response(model)
|
| 281 |
+
|
| 282 |
+
record = {
|
| 283 |
+
"dataset_id": f"comparison_{i:03d}",
|
| 284 |
+
"image_id": f"img_{k:03d}_model_{j}",
|
| 285 |
+
"image_type": "comparison_base",
|
| 286 |
+
"image_size": "224x224",
|
| 287 |
+
"format": "JPEG",
|
| 288 |
+
"media_type": "image/jpeg",
|
| 289 |
+
"api_request": json.dumps(api_request),
|
| 290 |
+
"expected_response": json.dumps(expected_response),
|
| 291 |
+
"test_category": "model_comparison",
|
| 292 |
+
"difficulty": "normal",
|
| 293 |
+
"model_type": model,
|
| 294 |
+
"comparison_group": k
|
| 295 |
+
}
|
| 296 |
+
records.append(record)
|
| 297 |
+
|
| 298 |
+
datasets.append({
|
| 299 |
+
"name": f"model_comparison_{i:03d}",
|
| 300 |
+
"category": "model_comparison",
|
| 301 |
+
"description": f"Model comparison dataset {i+1} testing {len(model_types)} models",
|
| 302 |
+
"records": records
|
| 303 |
+
})
|
| 304 |
+
|
| 305 |
+
return datasets
|
| 306 |
+
|
| 307 |
+
def save_dataset_to_parquet(self, dataset: Dict[str, Any]):
|
| 308 |
+
"""Save a dataset to PyArrow Parquet format."""
|
| 309 |
+
records = dataset["records"]
|
| 310 |
+
|
| 311 |
+
# Convert to PyArrow table
|
| 312 |
+
table = pa.table({
|
| 313 |
+
"dataset_id": [r["dataset_id"] for r in records],
|
| 314 |
+
"image_id": [r["image_id"] for r in records],
|
| 315 |
+
"image_type": [r["image_type"] for r in records],
|
| 316 |
+
"image_size": [r["image_size"] for r in records],
|
| 317 |
+
"format": [r["format"] for r in records],
|
| 318 |
+
"media_type": [r["media_type"] for r in records],
|
| 319 |
+
"api_request": [r["api_request"] for r in records],
|
| 320 |
+
"expected_response": [r["expected_response"] for r in records],
|
| 321 |
+
"test_category": [r["test_category"] for r in records],
|
| 322 |
+
"difficulty": [r["difficulty"] for r in records],
|
| 323 |
+
# Optional fields with defaults
|
| 324 |
+
"batch_size": [r.get("batch_size", 1) for r in records],
|
| 325 |
+
"expected_max_latency_ms": [r.get("expected_max_latency_ms", 1000) for r in records],
|
| 326 |
+
"model_type": [r.get("model_type", "microsoft/resnet-18") for r in records],
|
| 327 |
+
"comparison_group": [r.get("comparison_group", 0) for r in records]
|
| 328 |
+
})
|
| 329 |
+
|
| 330 |
+
output_path = self.output_dir / f"{dataset['name']}.parquet"
|
| 331 |
+
pq.write_table(table, output_path)
|
| 332 |
+
|
| 333 |
+
# Save metadata as JSON
|
| 334 |
+
metadata = {
|
| 335 |
+
"name": dataset["name"],
|
| 336 |
+
"category": dataset["category"],
|
| 337 |
+
"description": dataset["description"],
|
| 338 |
+
"record_count": len(records),
|
| 339 |
+
"file_size_mb": round(output_path.stat().st_size / (1024 * 1024), 2),
|
| 340 |
+
"schema": [field.name for field in table.schema]
|
| 341 |
+
}
|
| 342 |
+
|
| 343 |
+
metadata_path = self.output_dir / f"{dataset['name']}_metadata.json"
|
| 344 |
+
with open(metadata_path, 'w') as f:
|
| 345 |
+
json.dump(metadata, f, indent=2)
|
| 346 |
+
|
| 347 |
+
def generate_all_datasets(self):
|
| 348 |
+
"""Generate all 100 datasets."""
|
| 349 |
+
print(" Starting dataset generation...")
|
| 350 |
+
|
| 351 |
+
print("📊 Generating standard test datasets (25)...")
|
| 352 |
+
standard_datasets = self.generate_standard_datasets(25)
|
| 353 |
+
for dataset in standard_datasets:
|
| 354 |
+
self.save_dataset_to_parquet(dataset)
|
| 355 |
+
|
| 356 |
+
print("⚡ Generating edge case datasets (25)...")
|
| 357 |
+
edge_datasets = self.generate_edge_case_datasets(25)
|
| 358 |
+
for dataset in edge_datasets:
|
| 359 |
+
self.save_dataset_to_parquet(dataset)
|
| 360 |
+
|
| 361 |
+
print("🏁 Generating performance datasets (25)...")
|
| 362 |
+
performance_datasets = self.generate_performance_datasets(25)
|
| 363 |
+
for dataset in performance_datasets:
|
| 364 |
+
self.save_dataset_to_parquet(dataset)
|
| 365 |
+
|
| 366 |
+
print("🔄 Generating model comparison datasets (25)...")
|
| 367 |
+
comparison_datasets = self.generate_model_comparison_datasets(25)
|
| 368 |
+
for dataset in comparison_datasets:
|
| 369 |
+
self.save_dataset_to_parquet(dataset)
|
| 370 |
+
|
| 371 |
+
print(f"✅ Generated 100 datasets in {self.output_dir}/")
|
| 372 |
+
|
| 373 |
+
self.generate_summary()
|
| 374 |
+
|
| 375 |
+
def generate_summary(self):
|
| 376 |
+
"""Generate a summary of all datasets."""
|
| 377 |
+
summary = {
|
| 378 |
+
"total_datasets": 100,
|
| 379 |
+
"categories": {
|
| 380 |
+
"standard": 25,
|
| 381 |
+
"edge_case": 25,
|
| 382 |
+
"performance": 25,
|
| 383 |
+
"model_comparison": 25
|
| 384 |
+
},
|
| 385 |
+
"dataset_info": [],
|
| 386 |
+
"usage_instructions": {
|
| 387 |
+
"loading": "Use pyarrow.parquet.read_table('dataset.parquet')",
|
| 388 |
+
"testing": "Run python scripts/test_datasets.py",
|
| 389 |
+
"api_endpoint": "POST /predict/resnet",
|
| 390 |
+
"request_format": "See api_request column in datasets"
|
| 391 |
+
}
|
| 392 |
+
}
|
| 393 |
+
|
| 394 |
+
# Add individual dataset info
|
| 395 |
+
for parquet_file in self.output_dir.glob("*.parquet"):
|
| 396 |
+
metadata_file = self.output_dir / f"{parquet_file.stem}_metadata.json"
|
| 397 |
+
if metadata_file.exists():
|
| 398 |
+
with open(metadata_file, 'r') as f:
|
| 399 |
+
metadata = json.load(f)
|
| 400 |
+
summary["dataset_info"].append(metadata)
|
| 401 |
+
|
| 402 |
+
summary_path = self.output_dir / "datasets_summary.json"
|
| 403 |
+
with open(summary_path, 'w') as f:
|
| 404 |
+
json.dump(summary, f, indent=2)
|
| 405 |
+
|
| 406 |
+
print(f"📋 Summary saved to {summary_path}")
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
if __name__ == "__main__":
|
| 410 |
+
generator = TestDatasetGenerator()
|
| 411 |
+
generator.generate_all_datasets()
|
scripts/model_download.bash
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
python - <<'PY'
|
| 2 |
+
from huggingface_hub import snapshot_download
|
| 3 |
+
snapshot_download(
|
| 4 |
+
repo_id="microsoft/resnet-18",
|
| 5 |
+
local_dir="models/microsoft/resnet-18",
|
| 6 |
+
local_dir_use_symlinks=False # copies files; safer for containers
|
| 7 |
+
)
|
| 8 |
+
PY
|
scripts/test_datasets.py
ADDED
|
@@ -0,0 +1,382 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Dataset Tester for ML Inference Service
|
| 4 |
+
|
| 5 |
+
Tests the generated PyArrow datasets against the running ML inference service.
|
| 6 |
+
Validates API requests/responses and measures performance metrics.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import json
|
| 10 |
+
import time
|
| 11 |
+
import asyncio
|
| 12 |
+
import statistics
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
from typing import Dict, List, Any, Optional
|
| 15 |
+
import argparse
|
| 16 |
+
|
| 17 |
+
import pyarrow.parquet as pq
|
| 18 |
+
import requests
|
| 19 |
+
import pandas as pd
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class DatasetTester:
|
| 23 |
+
def __init__(self, base_url: str = "http://127.0.0.1:8000", datasets_dir: str = "test_datasets"):
|
| 24 |
+
self.base_url = base_url.rstrip('/')
|
| 25 |
+
self.datasets_dir = Path(datasets_dir)
|
| 26 |
+
self.endpoint = f"{self.base_url}/predict/resnet"
|
| 27 |
+
self.results = []
|
| 28 |
+
|
| 29 |
+
def load_dataset(self, dataset_path: Path) -> pd.DataFrame:
|
| 30 |
+
"""Load a PyArrow dataset."""
|
| 31 |
+
table = pq.read_table(dataset_path)
|
| 32 |
+
return table.to_pandas()
|
| 33 |
+
|
| 34 |
+
def test_api_connection(self) -> bool:
|
| 35 |
+
"""Test if the API is running and accessible."""
|
| 36 |
+
try:
|
| 37 |
+
response = requests.get(f"{self.base_url}/docs", timeout=5)
|
| 38 |
+
return response.status_code == 200
|
| 39 |
+
except requests.RequestException:
|
| 40 |
+
return False
|
| 41 |
+
|
| 42 |
+
def send_prediction_request(self, api_request_json: str) -> Dict[str, Any]:
|
| 43 |
+
"""Send a single prediction request to the API."""
|
| 44 |
+
try:
|
| 45 |
+
request_data = json.loads(api_request_json)
|
| 46 |
+
start_time = time.time()
|
| 47 |
+
|
| 48 |
+
response = requests.post(
|
| 49 |
+
self.endpoint,
|
| 50 |
+
json=request_data,
|
| 51 |
+
headers={"Content-Type": "application/json"},
|
| 52 |
+
timeout=30
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
end_time = time.time()
|
| 56 |
+
latency_ms = (end_time - start_time) * 1000
|
| 57 |
+
|
| 58 |
+
return {
|
| 59 |
+
"success": response.status_code == 200,
|
| 60 |
+
"status_code": response.status_code,
|
| 61 |
+
"response": response.json() if response.status_code == 200 else response.text,
|
| 62 |
+
"latency_ms": round(latency_ms, 2),
|
| 63 |
+
"error": None
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
except requests.RequestException as e:
|
| 67 |
+
return {
|
| 68 |
+
"success": False,
|
| 69 |
+
"status_code": None,
|
| 70 |
+
"response": None,
|
| 71 |
+
"latency_ms": None,
|
| 72 |
+
"error": str(e)
|
| 73 |
+
}
|
| 74 |
+
except json.JSONDecodeError as e:
|
| 75 |
+
return {
|
| 76 |
+
"success": False,
|
| 77 |
+
"status_code": None,
|
| 78 |
+
"response": None,
|
| 79 |
+
"latency_ms": None,
|
| 80 |
+
"error": f"JSON decode error: {str(e)}"
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
def validate_response(self, actual_response: Dict[str, Any],
|
| 84 |
+
expected_response_json: str) -> Dict[str, Any]:
|
| 85 |
+
"""Validate API response against expected response."""
|
| 86 |
+
try:
|
| 87 |
+
expected = json.loads(expected_response_json)
|
| 88 |
+
|
| 89 |
+
validation = {
|
| 90 |
+
"structure_valid": True,
|
| 91 |
+
"field_errors": []
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
# Check required fields exist
|
| 95 |
+
required_fields = ["prediction", "confidence", "predicted_label", "model", "mediaType"]
|
| 96 |
+
for field in required_fields:
|
| 97 |
+
if field not in actual_response:
|
| 98 |
+
validation["structure_valid"] = False
|
| 99 |
+
validation["field_errors"].append(f"Missing field: {field}")
|
| 100 |
+
|
| 101 |
+
# Validate field types
|
| 102 |
+
if "confidence" in actual_response:
|
| 103 |
+
if not isinstance(actual_response["confidence"], (int, float)):
|
| 104 |
+
validation["field_errors"].append("confidence must be numeric")
|
| 105 |
+
elif not (0 <= actual_response["confidence"] <= 1):
|
| 106 |
+
validation["field_errors"].append("confidence must be between 0 and 1")
|
| 107 |
+
|
| 108 |
+
if "predicted_label" in actual_response:
|
| 109 |
+
if not isinstance(actual_response["predicted_label"], int):
|
| 110 |
+
validation["field_errors"].append("predicted_label must be integer")
|
| 111 |
+
|
| 112 |
+
return validation
|
| 113 |
+
|
| 114 |
+
except json.JSONDecodeError:
|
| 115 |
+
return {
|
| 116 |
+
"structure_valid": False,
|
| 117 |
+
"field_errors": ["Invalid expected response JSON"]
|
| 118 |
+
}
|
| 119 |
+
|
| 120 |
+
def test_dataset(self, dataset_path: Path, max_samples: Optional[int] = None) -> Dict[str, Any]:
|
| 121 |
+
"""Test a single dataset."""
|
| 122 |
+
print(f"📊 Testing dataset: {dataset_path.name}")
|
| 123 |
+
|
| 124 |
+
try:
|
| 125 |
+
df = self.load_dataset(dataset_path)
|
| 126 |
+
if max_samples:
|
| 127 |
+
df = df.head(max_samples)
|
| 128 |
+
|
| 129 |
+
results = {
|
| 130 |
+
"dataset_name": dataset_path.stem,
|
| 131 |
+
"total_samples": len(df),
|
| 132 |
+
"tested_samples": 0,
|
| 133 |
+
"successful_requests": 0,
|
| 134 |
+
"failed_requests": 0,
|
| 135 |
+
"validation_errors": 0,
|
| 136 |
+
"latencies_ms": [],
|
| 137 |
+
"errors": [],
|
| 138 |
+
"category": df['test_category'].iloc[0] if not df.empty else "unknown"
|
| 139 |
+
}
|
| 140 |
+
|
| 141 |
+
for idx, row in df.iterrows():
|
| 142 |
+
print(f" Testing sample {idx + 1}/{len(df)}", end="\r")
|
| 143 |
+
|
| 144 |
+
# Send API request
|
| 145 |
+
api_result = self.send_prediction_request(row['api_request'])
|
| 146 |
+
results["tested_samples"] += 1
|
| 147 |
+
|
| 148 |
+
if api_result["success"]:
|
| 149 |
+
results["successful_requests"] += 1
|
| 150 |
+
results["latencies_ms"].append(api_result["latency_ms"])
|
| 151 |
+
|
| 152 |
+
# Validate response structure
|
| 153 |
+
validation = self.validate_response(
|
| 154 |
+
api_result["response"],
|
| 155 |
+
row['expected_response']
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
if not validation["structure_valid"]:
|
| 159 |
+
results["validation_errors"] += 1
|
| 160 |
+
results["errors"].append({
|
| 161 |
+
"sample_id": row['image_id'],
|
| 162 |
+
"type": "validation_error",
|
| 163 |
+
"details": validation["field_errors"]
|
| 164 |
+
})
|
| 165 |
+
|
| 166 |
+
else:
|
| 167 |
+
results["failed_requests"] += 1
|
| 168 |
+
results["errors"].append({
|
| 169 |
+
"sample_id": row['image_id'],
|
| 170 |
+
"type": "request_failed",
|
| 171 |
+
"status_code": api_result["status_code"],
|
| 172 |
+
"error": api_result["error"]
|
| 173 |
+
})
|
| 174 |
+
|
| 175 |
+
# Calculate statistics
|
| 176 |
+
if results["latencies_ms"]:
|
| 177 |
+
results["avg_latency_ms"] = round(statistics.mean(results["latencies_ms"]), 2)
|
| 178 |
+
results["min_latency_ms"] = round(min(results["latencies_ms"]), 2)
|
| 179 |
+
results["max_latency_ms"] = round(max(results["latencies_ms"]), 2)
|
| 180 |
+
results["median_latency_ms"] = round(statistics.median(results["latencies_ms"]), 2)
|
| 181 |
+
else:
|
| 182 |
+
results.update({
|
| 183 |
+
"avg_latency_ms": None,
|
| 184 |
+
"min_latency_ms": None,
|
| 185 |
+
"max_latency_ms": None,
|
| 186 |
+
"median_latency_ms": None
|
| 187 |
+
})
|
| 188 |
+
|
| 189 |
+
results["success_rate"] = round(
|
| 190 |
+
results["successful_requests"] / results["tested_samples"] * 100, 2
|
| 191 |
+
) if results["tested_samples"] > 0 else 0
|
| 192 |
+
|
| 193 |
+
print(f"\n ✅ Completed: {results['success_rate']}% success rate")
|
| 194 |
+
return results
|
| 195 |
+
|
| 196 |
+
except Exception as e:
|
| 197 |
+
print(f"\n ❌ Failed to test dataset: {str(e)}")
|
| 198 |
+
return {
|
| 199 |
+
"dataset_name": dataset_path.stem,
|
| 200 |
+
"error": str(e),
|
| 201 |
+
"success_rate": 0
|
| 202 |
+
}
|
| 203 |
+
|
| 204 |
+
def test_all_datasets(self, max_samples_per_dataset: Optional[int] = None,
|
| 205 |
+
category_filter: Optional[str] = None) -> Dict[str, Any]:
|
| 206 |
+
"""Test all datasets or filtered by category."""
|
| 207 |
+
if not self.test_api_connection():
|
| 208 |
+
print("❌ API is not accessible. Please start the service first:")
|
| 209 |
+
print(" uvicorn main:app --reload")
|
| 210 |
+
return {"error": "API not accessible"}
|
| 211 |
+
|
| 212 |
+
print(f" Starting dataset testing against {self.endpoint}")
|
| 213 |
+
|
| 214 |
+
parquet_files = list(self.datasets_dir.glob("*.parquet"))
|
| 215 |
+
if not parquet_files:
|
| 216 |
+
print(f"❌ No datasets found in {self.datasets_dir}")
|
| 217 |
+
return {"error": "No datasets found"}
|
| 218 |
+
|
| 219 |
+
if category_filter:
|
| 220 |
+
parquet_files = [f for f in parquet_files if category_filter in f.name]
|
| 221 |
+
|
| 222 |
+
print(f" Found {len(parquet_files)} datasets to test")
|
| 223 |
+
|
| 224 |
+
all_results = []
|
| 225 |
+
start_time = time.time()
|
| 226 |
+
|
| 227 |
+
for dataset_file in parquet_files:
|
| 228 |
+
result = self.test_dataset(dataset_file, max_samples_per_dataset)
|
| 229 |
+
all_results.append(result)
|
| 230 |
+
|
| 231 |
+
end_time = time.time()
|
| 232 |
+
total_time = end_time - start_time
|
| 233 |
+
|
| 234 |
+
summary = self.generate_summary(all_results, total_time)
|
| 235 |
+
|
| 236 |
+
self.save_results(summary, all_results)
|
| 237 |
+
|
| 238 |
+
return summary
|
| 239 |
+
|
| 240 |
+
def generate_summary(self, results: List[Dict[str, Any]], total_time: float) -> Dict[str, Any]:
|
| 241 |
+
"""Generate summary of all test results."""
|
| 242 |
+
successful_datasets = [r for r in results if r.get("success_rate", 0) > 0]
|
| 243 |
+
failed_datasets = [r for r in results if r.get("error") or r.get("success_rate", 0) == 0]
|
| 244 |
+
|
| 245 |
+
total_samples = sum(r.get("tested_samples", 0) for r in results)
|
| 246 |
+
total_successful = sum(r.get("successful_requests", 0) for r in results)
|
| 247 |
+
total_failed = sum(r.get("failed_requests", 0) for r in results)
|
| 248 |
+
|
| 249 |
+
all_latencies = []
|
| 250 |
+
for r in results:
|
| 251 |
+
all_latencies.extend(r.get("latencies_ms", []))
|
| 252 |
+
|
| 253 |
+
summary = {
|
| 254 |
+
"test_summary": {
|
| 255 |
+
"total_datasets": len(results),
|
| 256 |
+
"successful_datasets": len(successful_datasets),
|
| 257 |
+
"failed_datasets": len(failed_datasets),
|
| 258 |
+
"total_samples_tested": total_samples,
|
| 259 |
+
"total_successful_requests": total_successful,
|
| 260 |
+
"total_failed_requests": total_failed,
|
| 261 |
+
"overall_success_rate": round(
|
| 262 |
+
total_successful / total_samples * 100, 2
|
| 263 |
+
) if total_samples > 0 else 0,
|
| 264 |
+
"total_test_time_seconds": round(total_time, 2)
|
| 265 |
+
},
|
| 266 |
+
"performance_metrics": {
|
| 267 |
+
"avg_latency_ms": round(statistics.mean(all_latencies), 2) if all_latencies else None,
|
| 268 |
+
"median_latency_ms": round(statistics.median(all_latencies), 2) if all_latencies else None,
|
| 269 |
+
"min_latency_ms": round(min(all_latencies), 2) if all_latencies else None,
|
| 270 |
+
"max_latency_ms": round(max(all_latencies), 2) if all_latencies else None,
|
| 271 |
+
"requests_per_second": round(
|
| 272 |
+
total_successful / total_time, 2
|
| 273 |
+
) if total_time > 0 else 0
|
| 274 |
+
},
|
| 275 |
+
"category_breakdown": {},
|
| 276 |
+
"failed_datasets": [r["dataset_name"] for r in failed_datasets]
|
| 277 |
+
}
|
| 278 |
+
|
| 279 |
+
categories = {}
|
| 280 |
+
for result in results:
|
| 281 |
+
category = result.get("category", "unknown")
|
| 282 |
+
if category not in categories:
|
| 283 |
+
categories[category] = {
|
| 284 |
+
"count": 0,
|
| 285 |
+
"success_rates": [],
|
| 286 |
+
"avg_success_rate": 0
|
| 287 |
+
}
|
| 288 |
+
categories[category]["count"] += 1
|
| 289 |
+
categories[category]["success_rates"].append(result.get("success_rate", 0))
|
| 290 |
+
|
| 291 |
+
for category, data in categories.items():
|
| 292 |
+
data["avg_success_rate"] = round(
|
| 293 |
+
statistics.mean(data["success_rates"]), 2
|
| 294 |
+
) if data["success_rates"] else 0
|
| 295 |
+
|
| 296 |
+
summary["category_breakdown"] = categories
|
| 297 |
+
|
| 298 |
+
return summary
|
| 299 |
+
|
| 300 |
+
def save_results(self, summary: Dict[str, Any], detailed_results: List[Dict[str, Any]]):
|
| 301 |
+
"""Save test results to files."""
|
| 302 |
+
results_dir = Path("test_results")
|
| 303 |
+
results_dir.mkdir(exist_ok=True)
|
| 304 |
+
|
| 305 |
+
timestamp = int(time.time())
|
| 306 |
+
|
| 307 |
+
# Save summary
|
| 308 |
+
summary_path = results_dir / f"test_summary_{timestamp}.json"
|
| 309 |
+
with open(summary_path, 'w') as f:
|
| 310 |
+
json.dump(summary, f, indent=2)
|
| 311 |
+
|
| 312 |
+
# Save detailed results
|
| 313 |
+
detailed_path = results_dir / f"test_detailed_{timestamp}.json"
|
| 314 |
+
with open(detailed_path, 'w') as f:
|
| 315 |
+
json.dump(detailed_results, f, indent=2)
|
| 316 |
+
|
| 317 |
+
print(f" Results saved:")
|
| 318 |
+
print(f" Summary: {summary_path}")
|
| 319 |
+
print(f" Details: {detailed_path}")
|
| 320 |
+
|
| 321 |
+
def print_summary(self, summary: Dict[str, Any]):
|
| 322 |
+
"""Print test summary to console."""
|
| 323 |
+
print("\n" + "="*60)
|
| 324 |
+
print("🏁 DATASET TESTING SUMMARY")
|
| 325 |
+
print("="*60)
|
| 326 |
+
|
| 327 |
+
ts = summary["test_summary"]
|
| 328 |
+
print(f"Datasets tested: {ts['total_datasets']}")
|
| 329 |
+
print(f"Successful datasets: {ts['successful_datasets']}")
|
| 330 |
+
print(f"Failed datasets: {ts['failed_datasets']}")
|
| 331 |
+
print(f"Total samples: {ts['total_samples_tested']}")
|
| 332 |
+
print(f"Overall success rate: {ts['overall_success_rate']}%")
|
| 333 |
+
print(f"Test duration: {ts['total_test_time_seconds']}s")
|
| 334 |
+
|
| 335 |
+
pm = summary["performance_metrics"]
|
| 336 |
+
if pm["avg_latency_ms"]:
|
| 337 |
+
print(f"\nPerformance:")
|
| 338 |
+
print(f" Avg latency: {pm['avg_latency_ms']}ms")
|
| 339 |
+
print(f" Median latency: {pm['median_latency_ms']}ms")
|
| 340 |
+
print(f" Min latency: {pm['min_latency_ms']}ms")
|
| 341 |
+
print(f" Max latency: {pm['max_latency_ms']}ms")
|
| 342 |
+
print(f" Requests/sec: {pm['requests_per_second']}")
|
| 343 |
+
|
| 344 |
+
print(f"\nCategory breakdown:")
|
| 345 |
+
for category, data in summary["category_breakdown"].items():
|
| 346 |
+
print(f" {category}: {data['count']} datasets, {data['avg_success_rate']}% avg success")
|
| 347 |
+
|
| 348 |
+
if summary["failed_datasets"]:
|
| 349 |
+
print(f"\nFailed datasets: {', '.join(summary['failed_datasets'])}")
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
def main():
|
| 353 |
+
parser = argparse.ArgumentParser(description="Test PyArrow datasets against ML inference service")
|
| 354 |
+
parser.add_argument("--base-url", default="http://127.0.0.1:8000", help="Base URL of the API")
|
| 355 |
+
parser.add_argument("--datasets-dir", default="scripts/test_datasets", help="Directory containing datasets")
|
| 356 |
+
parser.add_argument("--max-samples", type=int, help="Max samples per dataset to test")
|
| 357 |
+
parser.add_argument("--category", help="Filter datasets by category (standard, edge_case, performance, model_comparison)")
|
| 358 |
+
parser.add_argument("--quick", action="store_true", help="Quick test with max 5 samples per dataset")
|
| 359 |
+
|
| 360 |
+
args = parser.parse_args()
|
| 361 |
+
|
| 362 |
+
tester = DatasetTester(args.base_url, args.datasets_dir)
|
| 363 |
+
|
| 364 |
+
max_samples = args.max_samples
|
| 365 |
+
if args.quick:
|
| 366 |
+
max_samples = 5
|
| 367 |
+
|
| 368 |
+
results = tester.test_all_datasets(max_samples, args.category)
|
| 369 |
+
|
| 370 |
+
if "error" not in results:
|
| 371 |
+
tester.print_summary(results)
|
| 372 |
+
|
| 373 |
+
if results["test_summary"]["overall_success_rate"] > 90:
|
| 374 |
+
print("\n🎉 Excellent! API is working great with the datasets!")
|
| 375 |
+
elif results["test_summary"]["overall_success_rate"] > 70:
|
| 376 |
+
print("\n👍 Good! API works well, minor issues detected.")
|
| 377 |
+
else:
|
| 378 |
+
print("\n⚠️ Warning: Several issues detected. Check the detailed results.")
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
if __name__ == "__main__":
|
| 382 |
+
main()
|
test_main.http
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Test Prediction Endpoint
|
| 2 |
+
# Works with any model configured at startup (default: ResNet-18)
|
| 3 |
+
|
| 4 |
+
POST http://127.0.0.1:8000/predict
|
| 5 |
+
Content-Type: application/json
|
| 6 |
+
|
| 7 |
+
{
|
| 8 |
+
"image": {
|
| 9 |
+
"mediaType": "image/jpeg",
|
| 10 |
+
"data": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBxMTEhUTExIWFhUVFhoYGBgYGBgYGBgXGBUYFxUYGBcYHSggGBolGxUXITEhJSorLi4uFx8zODMtNygtLisBCgoKDg0OGxAQGy0mHyUtLS0tMjAtLS8tLS0tLy0tLS01LS8vLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLf/AABEIAKoBKQMBIgACEQEDEQH/xAAbAAACAwEBAQAAAAAAAAAAAAAEBQIDBgEAB//EAEMQAAECAwYDBQUGBAUDBQAAAAECEQADIQQFEjFBUWFxkRMigaGxMlLB0fAGFBVCkuFigqLxI1Nyk9IHFkMzsrPC4v/EABkBAAMBAQEAAAAAAAAAAAAAAAECAwQABf/EAC4RAAIBAwMEAAUEAwEBAAAAAAABAgMRIRIxQQQTIlEUYXGRsTJCofAjgdHhBf/aAAwDAQACEQMRAD8Aw5s1A0U9kNocCynNNQ0DLs6tmEC63NOiSlpYJ2HCPJlNpDGXZ2rBgsQUlwKjOJdzNjQ6D03QtkyH08oum2cJGQi8yCPDrE2ejRotg89zd7oEkqAIoOkNhaAQE4RXWkLTZ1Pk4i2y5784TQi3flawzTYhsOkXy7MNh0gizFwKNBQlxS5ltkETZhsOgi5NmGw6CCBLbWCESHaBc7SwEWQbDpHZlhSQRQcWhl2MCrtKUKGJgPFx5QssrcvStGWUmI1XEp3IcHhExcWQYeNI1UmcFB0mhyjxDcYwOjWk9z2Y9bQgtjMKu6UipFdqExXaJiQHCR4t8Ic2iypxYi/wim1XbiY67fOO+GUWtTuxo9c6ibirL2ZS3SysvhFdhACrEraPoCLAQ1AeAELrXdgUXAz+sodudNaUkCn2K0nJt/7MYLPwiwWfhGgXdvARxFgHDpGqnGTyzLWnTV1ERfd+HpEhZeHpD37kNh0ixNjGw6RrUTyZVBALJw9IvRd/DzEPkWQbDpF6LKNh0htiblcRy7u/h80xb+Hjb0h6iQNh0iX3bl0g6iTiIfuHD0jyrAPd9Ieqs/LpEV2fl0jtQNJnV2Lh6RUbHXL0jQLkDh0ildmrp0g3BpEa7INvSK1WYbekPV2Xl0ildl5dI64bCc2UbekV/dht6Q6VZqftEBZuHlHINhKqzDaBewG0aM2T6aAfu300cwJASJymppB9ktYyUIVS1NR4mCTlGNHozWTUWaUkhwxHCDJNnABDZ8PLlGPkWlSS4Lcobyr6mEhyB4CFlSTGhXqQ5uhjOlJByPygKcEDIkHQ/N4eyLRKXXEAfntHLTIkqVgWCDnl8YKbW5F2bwI0CoCWL1NKdXhpJugZ5P6xcLtTKLoDvXEch4HOI2K/CtfZplYqs4LPxY5R177AsRstjmhR71DkaMPAwwtM5EpJMw5DqWp1hhMUhIYqCSqgBLExmL+uObNI7N1M7qJoBpXrAGVjky/5KsgrPLgc+BiVqtgWlAQCSMhw/M+1YrsH2KmYhjUGIdx5jnGl+zlx9kCpQ7xpvT94LaCUXTJUZbkkYXehd826bQDOXMWzoolRAUB7QNXz+Gkay19nKQ61JQjV9eAGpppFCLxshT/6yGpQltaUPEQEwGYu+7FKmv2ndDsRlUsQDGuFkYeEQVZbNZpa7QAMOalJ7z4jn56QnsP2yQcRmIIr3QPdrmSz6c+Edl7HYG8y7Bm3ebWK50pMtJVMUBuTlySIJkX5JKEqMxCSfyk1o/TLWMFfX2hmTz3gAgZJGQrmXzPGDGOQuTasaFd+2erEnCMsiasyd4BvO14UCYO6lTcSX32yeMcu0h8oqm2t+W0NpQY6th9OvNSi9AOIL+sdl20anyMZ2Vajx6xaZyuPUw6kkh+1J8D025O/kYnJtD7dDCIKL6+cMLGo7mGuuAdvhodygTF7GF8qcRv1ghNp4HTWDcjKkHS0RahML/vJaj668YNlzOefwjrk3AsVLMeMqJ9rTKOLUdjHC2BJkqIGRWClJOxiKJKn16GGTFaKOxilciGIknj0Me+7E6HpHXOsKjIeJCzfTw6s93l2LjiU5UgwWRCM1IOlQB6mElWhDdl6fS1KmyEKbrUfy67iFv4cv3fMfOH96XskA4UgUB7qwPzcBGc/Fuf6/wBozrqZS2R6a/8AmU0vJu/9+Rip1oxF2YxZKKgHYtvEJNNQecNJgRMQEpUArV6B9IDnosrYJxoKtd6s+vYImYOESC6wFNQpJYxwTYsmmYpQknZmgu219mXfw02qIMt17qUkIQmvvByrOgB0jNSpu5g2TawmteBByjrISzDECaokl6ZkuW59YayLaZSQqWAfeWdTqG1MJrbfJmJSmjJq7VLgO/SAjaTA3Gs0M7dbitZWrM+HlBtzXsqScSTXIg1BHEPWM+pZOcSwGGxawLPc+m3X9p5cxSUKwSw3tElnbdxhru8aeStKgClSSDkQQX3bePiMte8N5N9qlpTgLKANSKir0MSlT9Dxb5Dft7eOO04KYZQwgjUlioHkaeBjM9qSWakSt9sVNWVqqo5nc5aQPKVWHUtKsOqTkzUSr3TLsi7KHV2jKUrEQyu6WAaoZLHfwqgM1uMUrcxxElR0JibqpGiHSSfBbOtyiGyEU9sTSLU2RSvymCLFZ5aS6yS2QTqeY0iU+oSWDVT6J3ysFRsC2ds46LuUdINVe89+7LASKDuOQNKkRfOtE7D3+7i91OXAloxSrVebHo06FFbJipFiINRBiLKT/eJ2CzlRNSaah/jDuXYhuOkOq9lkWVBXwKE2SuXnBdnsphqixp3T0EXJsoH5k+Xzh41pSZJ04RQDKs5iZlQTMkbK6f3ir7qr3jmNDvzjTGczLKNNlQkq4Qxs0vff4RXKs5bM5H1hnIsZfPXbhFYzMdeMUsFfYCLAkQbLsf00W/cPrDDazHouL0JEXolwaiyAbdGiJnoScknxGsTl1MYlqfSTnsQRZNx5x6b2aNxVjQnR4WW+9yMnyOS29BCW1XiVHNQqD/6hOjRDu1Kn6cI9On0NOC8x/bb1GE10907xnbdeZJoR7Xu8IEnTFHVXUwJMB458Y6nSSeTa3pXieWomhbKF/ZQahJOp84F7NW584u4+iDqW3YjTKpoYjMs6hXfjB6btWQCx5/3ERN3L3HWB3EjzFQk9mAAGIrlQWuxrBZuhEVrlqGhju4hn08wTLSJpMWqA1cHqIj2UdrO7ByJBcR7OJYYOsXsssTOIiQmmKkiJEQXMMaBaFcYJRZphySS8UyJpTklJ5gxbJmzS+ENi2f5tEJ1JcG6l09Pm7+gQq5ptGzOlAY6izS5ZZQmLV7oSBVst4gi7p6iF1cUCipj4EmGdiuxQqucl3qy3roMqHkYyTrNLyl9jZToxviFimRZEGvZTArRD18SoU84cSrKEh3QkalQdjqyiO8fDSIi0KlDNCf8AUsk8SzH1iKbxQ7hBJOaiXHIA6dOUY5dyptsa04wLxbEthBKyNQwB8AwIiuyS5jv93wbLIDB9e9r4GDEXkGGGnBLJPj3fjEzeS1ZqNN8Pq0TjTnwvuLKoTk2KYoP95UAMwC/9TR1UgJDqWFPus1/lLwDNnKUXxEeMUT0Pq5EOumb3YnczgITbUk90N/K3wEW947QHd8glRzy2fWH8mzcPKNcIQhsQqyk3lgaLOdvOCJdm4ecGosx28oJlWQ/Qi6qsyShFAMuxjbzgmXZRtqNYPlWM/SYIFk+sMPrZnlJcC+XZBtvrxg+VZztrvF8uQw66cYqtiyAQHdtOcF1VFXIqDquxYCE5wPaLySMz/SreFk4zN1/1QHaJUz+PL+LeM0qjmz0aXSQgF2m9E+9v+Uwtn20nUdIiqyrOi/0mJpsCvdV+kw0IpF5aUtwNcpKs/jFYsiB9GGX4Ws+8P5TF8q4Vn85/QfnGmM8Gacobt3FZkp+nipVkB0840ku5CKu/8n7wSi6hsP0QVNIk6vzMim7VaDzEL/uC9vMR9LTZEj8g/SPlCb7un3R0EHvEXM+c2FUxGEpOTEO8Ml30A5nSpZIbJIdlKbP6yhR9mb8lJkNObuszt0AMJ/tPeAVaFdmRgDANkW1Fd3rGWOqU3Frbk1VZ0lTUl9jdzZkgueyALcT5GFc/CGZmJAAIZySwoDErivaTNnKRMKQOzSoKUcIBYlVSc6geEZ/7Q3oDaFJlkYEqABFRTDUF9wYWDk5abBmqUY6kzS9lKI70kPuFK9IpNyy1eyseX7kQTJnSVzxKdNZSVgghsRfECXzZojbkIRapVncPMBL7e5XckGnKFVbNvlcq6cd7823FFoukgsK/XKKVXadx6RprJYkrmzJSZgJlpSS71xO7DgwrxiJsQ7fsKYuzx5aO0N8SlgXsX5RlzYiDxi1CFJqAAd2BPnGpNzHSvIExSu7iPytzT84D6mEi8Ona5M0EKNMzwSPhBNnBBZSlJ4YRDU2Ze5+uERFmO3n+0c5pjxjpe7KJKUEe1lulPkKxcpKSw7x8UgfpaLUWfkD4GL5Vn5xJ6dyuqTB+yQP/ABjm7egiXZqOSQkDb94Yy7EOJ8IPs9l2ST1+BiMq0Yhs+RUmQTqfB/nBybtQ3t12NPM5Q0lyfeSG4GvDNUGy5LCiW8B/yNYzS6l8E5zSE9kupGalYtggYusFGyyxVsPMEK/pL9YbGS4qtXgUt5GKDYEh3HUv6RKVST3ZFVE3uLZIQSQkF9z8yTDGVLLRGXISMkjwEFo5RtoW04Fqy9FkmUX8YNlSoGlGLu1LUeNaZhmmwoIimdaEj83kflAmFfvq6n5xEWRRzV1eG1ehVTXLJqtg38jFPbklw0Eou/iOkFpsg2HSBbljOcVhC/t1cI92i9h9eMMvu42HQQDfN4S7NLC1MXmIQ1PzEP0SSekMvkS1E5Sl8IvQkwPeF5ypIlqUQ01YQDTUEvypCu134E3nJs7gJMhb/wCpRlqS+mST9GjJNitmiCYHm25KVhBLEh/MJHFyTAH2kv1EiWFBSSoqSAMQ94O7HJqPlWpEYL/qNfahaWlKDJCA4OIEM5Aagqa8oaMXIXk+qMYXXjeqZK5SVFhMKn4BKCX6gDxA1hJcX2pSbJLUopxBABGMAummR5fWUfO/t3fhnT3SaIDJdlMXeh2y8RBhTbYXg+x2S29oVhLdxQDuC7oSsHh7XlCx1RlfsF9qUgTjOVVRCwVKAySElLqNS4cAZAx7/veV7p/XHODTsdufHQmOhMNU3cWyiYu07RbWiSpsUlEdww3F2K2iYupW0LrQ/bYpSohi8XzLZMMxMzEcSQADSjUEMfwlW0d/CFbQrlG9x1GaRVdd8zZE1U1Ku8oMpw4IO7xcm95vaqnEurBhJ4FOH0iQudW0XJuhexictDyVjrWBlL+2K2kBSB/hPiId19wpS9eJeHU37YyGThlKqAVB6ineAJO7eEZhNzLP5TBAuJZ0jPKjSfBeFWquTbWW2WWYpKUzgSpRSBlUAHUcQHg+TdiFpC0EKSqoLpbyMYBNxL2MHWe6JoyKhpQkZVHmYzS6dftkaFXmbH8HGxjv4YBv0hbd8+1oThCiRhYYg7cQTWHVkvif/wCSUk8g2/PhEJUqvsPfnwVy7E2QEEy7Fw9YZ2S3IXmhSNnDv0hkiWDlEZQnyRn1UluhRIsja+vxg6VZ+MHJk8IsTL4QY0W9zJOvcDMvjFE2zgw4RLjk2VSkafg5OGtEo17MQfdhFiZEHrkxDA0dTi0rGjvXBhKixMqIzrUlOhPKF1pvZY9mWdc66UyjTGEmFKUhuhERtM9MtClrUAlIck6CMpeFstMwNiKB/CCH3rnCKdd0wu5WXDH2qh3Y71DxeNNndp8s2ls+1NklKwqnAl2LAlqO5YZct4Ro/wCoEvtS6T2dBSp7ruQMquOkZw3GrY/pMQmXGpsj+gxZU0DtpBdv/wCoK+3lTUSwyJakqSciVEEts2FPQxl74+0E+0Yu0UGVM7RgG72EIDcGEMplyK2P6DA825lNkf8AbMVUUhHH5C+9L8nTky0TJjpll0hgGO7gOTziidfE1U9M8realmUQNEhIozZAdILXc6joofyGKlXSt/ZV+gxRRQjT9A943zNm4cZBYN7IELpk/l0hv+EK2V+gxFdyK91X+2YooxJtTFcq2KbDSvCK5q4ci5le6r/bMVrudb5K/QYZJAakJRNPCKsf00O1XKvZX+2YD/DFbK/QYNkTambtCpDJ7qMhmeHOLEqle4nqP+UZWUSwPZpelRXLh9ZRei1N/wCM/pEeG1LhnsJx5RqCuSPyJ+vGJCdJ9xPUfOMwbfoEp/mGrR4Ty1EpfavzELpl7DePCNaJ0r/LH14x42mV/lj9JMZNE5fuS+RB+cUrt6sTdjLU2oA+Md25eznKK4NmLxkDNKRzYepif4vZx+VHKh9IxybSvMJlo/lST0Ecl3wx709XhLQPWC6UhdcTdS71le6j64GLk3vK2SeQf0EYtN4YgCkzCd+78IIVaJgFZiv6Q3nEnB+/5HsnsjYpvhGYQP0EeqYtTfKRmEeX/GMEu+QAxExR5GsTk3yMlBXiTlAdOYPA3qb+Gw8A/oIuRfnL+gesfPZN8SnoS2tEv5xem2k1lomKGrhKa0Pz6wHCa5OtTexv/wDuAjUf0x0faBW48SkRh02yaR7BT+k+hETlXo2eNxsk/ExKWtc/yN2qfo3Qv4+95hvKJi+1ak/XKPn0y8VBWJMuYRqWB8qfQgk24kPgmDoPKJt1fZ3Yp+jefjR1V6wIq/Ve8esY+VbD+VJB4gPEROXWp6o+CTGno9cpNSY0enprg1y75V73nFS73Vv6RkV2guAZiknbAFcq0aIptuYxTCRniCUjoTHo9tDaaadrfg1/4wrL5RA3qdvL9ox8y8xlWmZdB8gaxGVeKQCSZhzoEHTZtIdUzv8AEvRr/wAV3w+Xyjn4oNh5RlpFtJGLs5gG6gB5Kyga0W8g1mtwYK8wqCoXwc+2lf8Av8mu/FkuzDoflEF3mk0+BjLpthNHWTuHSPi0cNqdTEKp9VJEFROtD0jSKtaYqXORxhCbakaN/NHUWknceMGzKRVO+y+w77RHHzjgXLfXzhP95zd6cYslz3rUeMdlD6aT4GRmSuPnEwqUd/OFSpvPrHEzq5keMdZh0U/SGyjK2PnER2Wx84RqtR3V+qJCeWd1fqMdZ+zlGk+Byex4+cLOzlbHziEy0NqfBULfvX+r9UMk/Ys6dL0ZIWxQyrTeLBal69IhKkJcd6jfD5xebRhYhIfjyMT5wjw0na7ZMXiRQp+ucdC5inKEkDkdn+ucULt4OaU8+r7xZZrSTQZH6Hp5wko2zYeMtWNR4KmULqqPrXlBGJRGY55Goi6RIQc1e0aas523j0+6kl8KwSOeVM+HzhZVE3kpGjJLx/ILKQoHEC/jBcu0rz7hBOvjxgKfd6gHxUA9Sz+kUJJQWJB8Y5pSyBNxxZoZfeJhOaQ21ItwrA9tLcVE604QnmzgajOIDGugVTjlw8I7tpgda2Ms0EuclJZS3JDgvwGjHcxNFsSNH4kPV4SSrsmPmH5wZOu9ftEuNSCOGY5vCypxezKQqSX7bDgTyWKVpHBgPNmip11xTta5noBxhQuZLQQS6nBOeVWgj8aQAwQw2/fwiPbfCLdyPLt/sZWeyqaigdWdiYsCVADvl6ZVH75QpTfcshsJH8Tl/XhHpiisYkKpsebFzzIhHCXI6lHdMZTLTPTkkq8QGiUtU13Vm+poOJ4QhwzCogE0+vjFk2yziD3nEF0rehVVzyP13tLTmoE/wgepzio3lJPsqUkqo4YtzDRnvw3+J/D1ia7qAFZgJ2HzMaOkpU4yeRZ1qlsRG6J05NUWhKgdHIpuQdeFYOlqnBGIS0zNyApRrpUcNIVWW85VnHcl9/UqL6vTYwDO+1NoJJxnhwrG3S5vCF70Ka8pP82+5ok2tnM1SZb/AMAfhrtwgebbpJHcnYXZ2SEqPiPCEh+0S1pwzQFjiPj9ZCOJNmWaOgmjZDqfrKD2tOWL8UpYg0/rdP8A4OVSEKS6lTa/xAvxr9VidkkywD2ZNNVAO+wJ56CFGBKa41KDigOgj1qvJRDSkYQKd0acTvxhcvCYdcF5SWfux5PmMaHF/N8B9VgY29CQ6mfQMT8TCGVZZkwuVgEnJ6wzlXM5AKiQMz40A2o/SGUYrdgdarP9MfuEm1ylVDOdMuUeUvQM+0DruZOeJhpn8vSLkXaEd5CqjfOtI7VBcjqNV7xS+gR2E1nUgAAO7jIByWfaB1WutMJ8DHBeExL4ypSdRoUjMeIpFtmvCQst2AHgnSC9Vr2v9Du5G9k7P5/+BUjCpJ38YrmSpnsoQC31qeMBT7UUTJaUkgLOlAK69YYJtZSyiol9jX6pAeqI0ZwnvhrewFNVRlsFn2QHqPPjFaitKQWDGg+gYYXdYwpJmKUlWEsxqchk/OOTpImKKQQkJL8NqCG1q9kBwla/2BkzWFWbkYG+8J3HQxJSq5FtdoGxJ2HlDYJOvISWezlTMdhrR96cDF05JoFEUptlE7HbUppgHluD84vtlmxgYSBmaDyprnGVzle1sGVUo6bxd2BBKRo+tdo8be1EjyiKLGpiS4wjIg1+UVT7IUkBi5Zs6uzN1jrJiuU4rCsdm21SjyyrxixFrmJNaOIjJksRiHWj8oItWApISAKv5Gg+vhDtRatYVOd9Vy+TOWujUyD70o+WsGSruCmVMoQ3doHFCC7sz7NCpFrCEsKceORMQn3ip3JzGnn5wqptbFnVj+7JoBckpYSy2wkBVQp2YEUGdM4ptsrsckhSFZHNw+41YfTwkk3gt6Nrm2uecGKvHCkJUHoR7RwnUHiQ/nCSgykasLXSsyybbCr2UgNs+2bxFCphDVKX5OeeekUotYNEpDk15Elhs0XLmzBUghyQ+jUSacH84S1sI7VfLZZ+HKUKjL5hxuzGIS7rJFVMHbwicq1E0xF2y3ap8aeW9IhNvA5ca+EC8w6aW4bZ7slpHeLnUCtHz+EFpWEuE5El/HhzIhYm1qUXBAo/pX+3GCpE6gJfXhlmD4t55xKSk9y8HFYiW/iEsA/4ZDsDtw8jFsudiUCkNTfcf2iaES3cgU5bvkenXjFU+1Swe6APD639PCeHhJlVfdtWCLQZYAxHpxETVd0hdQshxqxY6NTjGVtVqCifKrx6VeC05aZMNeY5Rs6Sg75Znq9VBYawN7bcqkg4ZgUSQANcnJPllvCedY1DyLa1D5Qys98Es4Ph8R4mLkIQsE4iC54B3GVeJjQ5zg8meVKnVzTM+hwQAHPJz0hnc9nmziVAJCQQTiAD1zDhj46xcmRKSpy9D4M9c86U5R6dfobClDCgYACgNBxoWhu7qdooFKgoZqysvS5GiblkIUP8U0oRpic1yJoNOAiM+xWcuoKIO2LfLTb165mfbFhzoTvWh8opN4E6xVU78jvqqEXp0DW22lcs0A4ED64QEu2KDkEseNOXnHFXiFe3VhFlkCVqSkJcbZk5A8svCOUVHdGepUc5eEvodkWxY1LaudOUMLJayaACKvuiW9jMBjrmwPB9orl2RUouVu22r0zeFThIvDvQtyvwjTS7vnLQQlDhSWPeSKKGjmOTfs5JlygqapaV4mUAQQASWySdG11hR/3UpAwjHtRW1IHmfaRUwkKMwjOqqUECNOpf0is+o6d/N/MutFmmoUBNThSrKoJKd6E8IslTbOgsFnGMwQqgzzwttHDOXNBJKlEBkuSojgIW4CJhBQokCtC+mcPvuRk3Td4rD5ZdZDLShS0+0D3XBbxpxiSr3Qw7wxfmGFVICNkmZ99AGaWUMXFvrKHNguATEgqKUOHdSc/Es8GUoJXYlONaUrQR6xze1BlZlRyFMmVmaD2Yj+Bq9w/qT84Z3YuTJFUoUtyQruggEAMCa79Y7+Mo2H60xmdSSfisHofDwkk5tXMMZRSyq5Aqpk5I+n1eG9jVTF7oxB+B2gWRNbulRKSzk1O9a11iAnglTUchqlhvU6ROaMFJ6M+wiTOUrEFOS702Bb5w/tEhBUkqBKqTKHYJc58ATzfWF1lnABSZgTLIYggOXBdidDTxjiLwQoDchjt7Ayfd68hEZXbujVCyXk7iu87PMM3DTvKITsyS2goKQBOCgSkioLEEEeuUav76hRlBVGck8SAH5sBrp4Qjt6DjChValFRSBkfaSATU00rpWtLUqnBkr0UvJPkWoRiqSenX4dYNRZsSHb2AScy44cHIHMvwjsyUQvHgo7qGgr3gT4n9od2BEsDCafkUl83Se9UuAVFJ4FNKUg1KllcWlSu7CuVYFLQlgAaCgFQaBXGjvqOzVR6RdZrmUtQQ7AYiXYNhzetTkW4HaGlltQT7RGVDscTKqfHziSbWgFIo63LijBTBuqTlnEe5LNkalQji7BLHYhhwLSwYEkb0BPIpy3Lcoum5AEvVzpnQmn+kRG8pxKxhd1JGvCtcsh4Qmnz1FVSK6U+HOFScsjNxhgYlA2qfKrv5esD2yUCXAOp6s7vxc8o7ZrUAxI8csjWvjHJtrBDDQN4vsKcPCOV0xZOLRTMThLYvDXx216cY9KtZq78urvxf4xZZ5Yd3ajeH03SLLRJDLpn5OXo2XtK6jaHvF4FSksoGFrKqO1R1Km+MCTJpdnrBapTN3agnU+Bo20CTZRBA+q/Q6Q6UeCMnPkqJjqpxjix4/WvSPJlkkAaxoo7mare2Cci2KSXBZtdoMTeaiGJfy0aIi7y4QWL1JTWgr3S3u8IlarsSlScKlFL1cVAfccI0OzWToqrDY7KtAOYcnKpJenX94MRdoUArIsOAHmXf45RXZ7JhwlOpJdy9DRxpl/UXjgvc6ihJ3bfXPPzjJJO/gaYuKX+UBn2JYJdyBmQ5GcVCzFRPdbNtuXR4dIvVLEKYuK8/r1ipFqST3aZM5bgz+MOqk/QjoU2/1AlnulZFEkmn9SgB5ehhhd13TETC7AHCHerLq4fNtYKss8JV3gcScLDINuSztUnMZcostVuSSWZ3/YGgzbLgIDrTl42Lw6SnHyvsUXhZhgxOcRS+FxzAIGbJCiSNtc4RzZ60ird4AimjlvSDbVaaqc1Hu7voYoBSUlzWgFBo3TKKU04rJOtPVK0GL0uS+p+jDSw3aVEFSS3Q8Dyz6RRZZIWoUYFQDvUOWJ40rpGtu6XLSgSu0ol++SHLknydvCDVqaVgPSdL3H5bC24LFaSsGVgYLq5rwjXSrLJCiZ4Pbkd/CVYdGZqeyEwDfloTKCClQ9kksWybMiMlb78UpRIL5fnJ0EQSnVyjdKVLpvFu5qL6mCWwmUUoOnWj6tFUgTbQgIl4TgD1pTKI3TasSSpYxYTmouwYanIRGdakhalJUzn8pb00gW4NFv33w/uZ+8rCsTRQNgcsa0xQD2Y2PnGvvK9pakKUMGP2QAatu4HE04Rlfvi/8xX14xpgnJGCtCNOWHe/99i3EpgSaEZcK6RYhWEeNcsiND4QPNNTz+EMrMHUp69xH/sTEbXdjCgNdoJNSTRn2bLPlEEWgt0PL6YdIEUYOQkfdSpqicA+rYCWfZ4GlIXW2FWN1lKUkBRoHoK5V0OUXSAQtLkgjvAlmYZau+Ib5VgWznuA6gljqO8pq+A6RdOLyUE54lB9WxKLPs5MTaNMHdZDZs0hSiwepoRV1Emnj8I9aZmqQA4pq5FD4/tCtJ/xE/6k+ZDxK8Syqbn1ELoyP3LJs4u0u7u70+PwiCrSWSH9kU2qf3MAg/XhE5Jr4fAxTSjN3Gx7Z5y1oJBHcSqhNcIGJTJ1oGhbMUSSczn8fSIqWQlwSCCGOooTBt9qKbQSCQTLlmlKmUhz4uYRKzKSldZKl1YJGRr575e15cYrlDekUoUWVXb0Mdeo8PhDWFvyGpUd6joRrF8mYQCWOQ6d4eqRlxgeyLLgOWKFU0owFIqCi45/J4RxuUU7ZCp88sxDEE8GL1p4+cDTpgOv1lEbQs4U1NcPkFgeQA8BAhMGMRJ1Gy4qg6zz8gQzBvJn51MKgcuUEyVFs9B6xWCyLGWR3MWXJI2y8NB9ZwvmW04lMaORtQu0CS5hfM5bxeMpnAn/AOQRWK9jVZ6l44PC3KZnozecVTl4hRgBwH99YGRpyiSDn4xRoya28MkpBD50oXzHAvrQ+ceQ43Zq19Yutij2iq5rrxq9YLuI95R1CFkHYhmI2NTE5Ssrjwp6pKIQZ60DEpsS0guCxwmo7p56CrGphbNtRLkqJJ0isrJCiST3t/4jA69INNWY1WtKSVngNVKfvPSjDicmD0q9Yts1kVRbOAVUJbIZirn+0L5Ki+e0aa7pYJlAgEFC3BDg11GsNUk4op01ONV5+X5RxExGMKwBqMAGy4DlrA9pvIAkhwOQp4QAVFl10/8AsRCu0KL5xOMFctU6uUY+KsO7bfRmYQtSlJAZmTltSBOyC1dxgCHY5gBhVuMKwo7w5sZaUDr/APqKtaY4M8JutPzGFinTUy1nH3H7wDVoOEBTr2Dkd7oIIvlRcV0+MKJWU46hNDqKjKFgk1qZpqzkpKnF7f8AGyxdpcUNYH+8GBUGIxbY82VSUnk//9k="
|
| 11 |
+
}
|
| 12 |
+
}
|