File size: 2,110 Bytes
bc37111 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
"""
API Client Module
Handles communication with the evaluation backend.
"""
import logging
from typing import Optional
import requests
from ..core.config import settings
logger = logging.getLogger(__name__)
class EvaluationApiClient:
"""
Client for evaluation API operations.
Handles submission of evaluation requests to the backend.
"""
def __init__(self):
self.api_url = settings.api.url
self.auth = (settings.api.username, settings.api.password)
self.timeout = settings.api.timeout
def submit_evaluation(
self,
model_name: str,
email: str,
batch_size: int = 32
) -> bool:
"""
Submit an evaluation request to the API.
Args:
model_name: HuggingFace model identifier.
email: Email for notifications.
batch_size: Batch size for evaluation.
Returns:
True if submission was successful.
"""
if not settings.api.is_configured:
logger.error("API not configured - cannot submit evaluation")
return False
try:
payload = {
"model_name": model_name,
"model_repo": model_name.split("/")[0] if "/" in model_name else "unknown",
"batch_size": batch_size,
"email": email,
"model_type": "sentence-transformer"
}
response = requests.post(
f"{self.api_url}/api/mteb/request",
json=payload,
timeout=self.timeout,
auth=self.auth
)
if response.status_code == 200:
logger.info(f"Evaluation submitted successfully for {model_name}")
return True
else:
logger.error(f"API returned status {response.status_code}")
return False
except Exception as e:
logger.error(f"Error submitting evaluation: {e}")
return False
|