add streaming

main.py

@@ -15,8 +15,10 @@ from slowapi.util import get_remote_address
 # Initialize the limiter
 limiter = Limiter(key_func=get_remote_address)
 
+
 # MongoDB client
 client = AsyncIOMotorClient('mongodb+srv://vanaraai:0YzPmeArBjsIVKqd@jobcluster.thp8ohx.mongodb.net/?retryWrites=true&w=majority')
+
 db = client['starlight_tales']
 
 # Load the model
@@ -25,6 +27,7 @@ llm = AutoModelForCausalLM.from_pretrained(
     model_type='llama',
     max_new_tokens=2000,
     threads=3,
+    context_length = 4096,
 )
 
 # FastAPI app
@@ -95,6 +98,10 @@ async def stream_stories(item: Validation):
     """
     prompt = "<|user|>\nA bedtime tiny story for the children<\/s>\n<|assistant|>\n"
     return StreamingResponse(io.StringIO(llm(prompt)), media_type="text/plain")
+    #working code for streaming
+    # for text in llm(f"{prompt}", stream=True):
+    #     print(text, end="", flush=True)   
+    # #return StreamingResponse(llm(prompt,stream=True), media_type="text/event-stream")
 
 
 @app.post("/feedback")