File size: 651 Bytes
a2791a7 4776abb a2791a7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 |
import os
from fastapi import FastAPI, Request, Response
from langserve import APIHandler
from pydantic import BaseModel
from langchain_huggingface import HuggingFaceEndpoint
app = FastAPI()
@app.get("/")
def greet_json():
return {"Hello": "World!"}
llm = HuggingFaceEndpoint(
repo_id="meta-llama/Llama-3.2-3B-Instruct",
)
api_handler = APIHandler(llm, path="/v1")
# class InputRequest(BaseModel):
# input: str
@app.post("/v1/stream")
async def simple_stream(request: Request):
print(request)
return await api_handler.stream(request)
if __name__ == "__main__":
uvicorn.run("app:app", host="localhost", reload=True) |