Update stream_api.py

stream_api.py

@@ -12,6 +12,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
 import uvicorn
 import asyncio
 import time
+import zipfile
 
 
 # Config
@@ -29,16 +30,41 @@ app = FastAPI(title="Custom FastVLM onnx gguf API",
 # Load ONNX session once at startup
 ort_session = None
 
+# @app.on_event("startup")
+# async def load_models():
+#     global ort_session
+#     print("Loading ONNX vision encoder...")
+
+#     session_options = ort.SessionOptions()
+#     session_options.enable_mem_pattern = False
+#     session_options.add_session_config_entry("session.use_ort_model_bytes_for_initializers", "0")
+
+#     ort_session = ort.InferenceSession(ONNX_PATH, sess_options=session_options, providers=["CPUExecutionProvider"])
+
+#     print("Providers:", ort_session.get_providers())
+
+#     print("ONNX session ready ✅")
+
+#     await start_llm_server()
+
 @app.on_event("startup")
 async def load_models():
     global ort_session
+
+    ZIPPED_PATH = "vision_projector.zip"
+    ONNX_PATH = "vision_projector_v1_standalone.onnx"
+    
+    if not os.path.exists(ONNX_PATH):
+        with zipfile.ZipFile(ZIPPED_PATH, "r") as zip_ref:
+            zip_ref.extractall()
+        print("Extraction complete!")
+        
     print("Loading ONNX vision encoder...")
 
-    session_options = ort.SessionOptions()
-    session_options.enable_mem_pattern = False
-    session_options.add_session_config_entry("session.use_ort_model_bytes_for_initializers", "0")
+    ort_session = ort.InferenceSession(ONNX_PATH, providers=["CPUExecutionProvider"])
 
-    ort_session = ort.InferenceSession(ONNX_PATH, sess_options=session_options, providers=["CPUExecutionProvider"])
+    print(f"intra_op_num_threads: {ort_session.get_session_options().intra_op_num_threads}")
+    print(f"inter_op_num_threads: {ort_session.get_session_options().inter_op_num_threads}")
 
     print("Providers:", ort_session.get_providers())
 
@@ -46,6 +72,7 @@ async def load_models():
 
     await start_llm_server()
 
+
 @app.on_event("shutdown")
 async def shutdown():
     if llm_process:
@@ -308,4 +335,4 @@ async def predict(image:  UploadFile = File(...), prompt: str = Form(default="De
         return JSONResponse(status_code=500, content={"status": "error", "message": str(e)})
 
 if __name__ == "__main__":
-    uvicorn.run("api:app", host="0.0.0.0", port=8000, reload=False)
+    uvicorn.run("stream_api:app", host="0.0.0.0", port=8000, reload=False)