Update app.py

app.py

@@ -24,11 +24,16 @@ logging.set_verbosity_error()
 device = "cpu"
 model_id = "kingabzpro/whisper-base-urdu-full"
 
+# Load in fp32 and quantize to int8
 model = AutoModelForSpeechSeq2Seq.from_pretrained(
     model_id,
-    torch_dtype=torch.float16,
+    torch_dtype=torch.float32,
     use_safetensors=True,
 )
+model.eval()
+model = torch.quantization.quantize_dynamic(model, {torch.nn.Linear}, dtype=torch.qint8)
+
+processor = AutoProcessor.from_pretrained(model_id)
 
 processor = AutoProcessor.from_pretrained(model_id)