app.py

from fastapi import FastAPI, Request, Query
from fastapi.responses import StreamingResponse 
from transformers import AutoModelForSequenceClassification, AutoTokenizer 

import torch

import time
app = FastAPI()
model_name = "prajjwal1/bert-tiny"  # Pretrained BERT-Tiny on Hugging Face
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name)

 

# SSE generator
def event_stream(text: str):
    
    time.sleep(1)

    inputs = tokenizer(text, return_tensors="pt", truncation=True, padding=True)
    with torch.no_grad():
        outputs = model(**inputs)
        probs = torch.nn.functional.softmax(outputs.logits, dim=1)
        prediction = torch.argmax(probs, dim=1).item()
 
    yield f"data: {prediction}\n\n"



@app.get("/chatstrm")
async def chat(query: str = Query(..., description="User's message")):
      return StreamingResponse(event_stream(query) , media_type="text/event-stream")

# Entry point
if __name__ == "__main__":
    import uvicorn
    uvicorn.run("app:app", host="0.0.0.0", port=7899)