Update app.py

app.py

@@ -1,9 +1,9 @@
 import streamlit as st
-from transformers import pipeline
 import torch
 import csv
 import re
 import warnings
+from transformers import pipeline
 
 warnings.filterwarnings("ignore")
 
@@ -15,11 +15,13 @@ MAGICODER_PROMPT = """You are an exceptionally intelligent coding assistant that
 """
 
 # Create a text generation pipeline using the Magicoder model, text-generation task, bfloat16 torch data type and auto device mapping.
-generator = pipeline(
-    model="ise-uiuc/Magicoder-S-DS-6.7B",
-    task="text-generation",
-    torch_dtype=torch.bfloat16,
-    device_map="auto",
+generator = torch.quantization.quantize_dynamic(
+    pipeline(
+        model="ise-uiuc/Magicoder-S-DS-6.7B",
+        task="text-generation",
+    ),
+    {torch.nn.Linear},
+    dtype=torch.qint8
 )
 
 # Function to generate response
@@ -61,4 +63,4 @@ def main():
         process_output(correct_output)
 
 if __name__ == "__main__":
-    main()
+    main()