feature: add quantized model for cpu inference

main.py

@@ -9,11 +9,13 @@ import logging
 import time 
 from contextlib import asynccontextmanager
 from inference_utils import construct_alphabet_list, convert_offsets_to_absolute_coords, encode_text, get_alphabet_map
+from ml.src.inference_benchmark import QUANTIZED_MODEL_NAME
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__) 
 
 MODEL_DIR = Path("./packaged_models") 
+QUANTIZED_MODEL_NAME = "model.scripted.quantized.pt"
 SCRIPTED_MODEL_NAME = "model.scripted.pt" 
 METADATA_MODEL_NAME = "model.pt" 
 
@@ -62,6 +64,8 @@ async def lifespan(app: FastAPI):
         
         scripted_model_path = MODEL_DIR / SCRIPTED_MODEL_NAME
         metadata_model_path = MODEL_DIR / METADATA_MODEL_NAME
+        if device.type == "cpu":
+            scripted_model_path = MODEL_DIR / QUANTIZED_MODEL_NAME
         
         if  not scripted_model_path.exists():
             logger.error(f"Traced model not found at {scripted_model_path}")

packaged_models/model.scripted.quantized.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:793a525a5a8d4f62cc80ddbf0f0ca0fddc13ec202ef2fc6efd9bfaa32c78e306
-size 17674936
+oid sha256:c88342d00b8eb7d0380da89d9cf9f9744a9e6a7caac2f783f6b912d58c3129ac
+size 4509816