Spaces:
Sleeping
Sleeping
File size: 2,425 Bytes
dd1cc18 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 |
# app.py
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from datasets import load_dataset
from sklearn.metrics import accuracy_score
import uvicorn
# Initialize FastAPI
app = FastAPI()
# Define a request model
class ModelRequest(BaseModel):
model_id: str
tokenizer_id: str
# In-memory leaderboard to store results
leaderboard = []
def load_model_and_tokenizer(model_id: str, tokenizer_id: str):
"""Load the model and tokenizer from Hugging Face Hub."""
try:
model = AutoModelForSequenceClassification.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
return model, tokenizer
except Exception as e:
raise HTTPException(status_code=400, detail=str(e))
def evaluate_model(model, tokenizer):
"""Evaluate the model using a benchmark dataset."""
# Load a benchmark dataset (replace with your actual dataset)
dataset = load_dataset("glue", "mrpc") # Example: GLUE MRPC dataset
# Tokenize the inputs
inputs = tokenizer(dataset['test']['sentence1'], dataset['test']['sentence2'], padding=True, truncation=True, return_tensors="pt")
# Get predictions from the model
with torch.no_grad():
outputs = model(**inputs)
predictions = outputs.logits.argmax(dim=-1).numpy()
# Calculate accuracy
accuracy = accuracy_score(dataset['test']['label'], predictions)
return accuracy
def update_leaderboard(model_id: str, score: float):
"""Update the leaderboard with new results."""
leaderboard.append({"model_id": model_id, "score": score})
leaderboard.sort(key=lambda x: x['score'], reverse=True) # Sort by score descending
@app.post("/submit")
async def submit_model(request: ModelRequest):
"""Endpoint to submit a model for evaluation."""
model_id = request.model_id
tokenizer_id = request.tokenizer_id
model, tokenizer = load_model_and_tokenizer(model_id, tokenizer_id)
score = evaluate_model(model, tokenizer)
update_leaderboard(model_id, score)
return {"message": "Model evaluated successfully", "score": score}
@app.get("/leaderboard")
async def get_leaderboard():
"""Endpoint to retrieve the current leaderboard."""
return leaderboard
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=8000) |