Update app.py

app.py

@@ -20,7 +20,11 @@ print(f"Starting to load the model {model_name}")
 
 m = AutoModelForCausalLM.from_pretrained(
     model_name,
-    device_map=0,
+    llm_int8_enable_fp32_cpu_offload=True,
+    device_map = {
+        "transformer.h": 0,
+        "": "cpu",
+    },
     #load_in_8bit=True,
 )