Update app.py

app.py

@@ -17,14 +17,20 @@ adapter_model = "olacode55/zimble-llama2"
 
 tokenizer = AutoTokenizer.from_pretrained(base_model)
 
+offload_folder = "./offload"  # must exist or be creatable
+os.makedirs(offload_folder, exist_ok=True)
+
+# --- Load model with 8-bit quantization and CPU offload ---
 base = AutoModelForCausalLM.from_pretrained(
     base_model,
     load_in_8bit=True,
     device_map="auto",
-    llm_int8_enable_fp32_cpu_offload=True,  # offload layers to CPU if GPU full
-    use_auth_token="hf_" +hf_token 
+    offload_folder=offload_folder,
+    llm_int8_enable_fp32_cpu_offload=True,
+    use_auth_token="hf_" +hf_token
 )
 
+
 model = PeftModel.from_pretrained(base, adapter_model)
 
 # === STEP 3: Define generation function ===