MOSHNA

app.py

@@ -9,11 +9,21 @@ bnb_config = BitsAndBytesConfig(
     bnb_4bit_quant_type="nf4",
     bnb_4bit_compute_dtype=torch.bfloat16,
     bnb_4bit_use_double_quant=True,
-    llm_int8_enable_fp32_cpu_offload=True
+    llm_int8_enable_fp32_cpu_offload=True,
+
 )
 
 tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, quantization_config=bnb_config, device_map="auto")
+
+device_map = {
+    "transformer.wte": "cpu",
+    "transformer.wpe": "cpu",
+    "transformer.h": "cpu",
+    "transformer.ln_f": "cpu",
+    "lm_head": "cpu",
+}
+
+model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, quantization_config=bnb_config, device_map=device_map)
 model.generation_config = GenerationConfig.from_pretrained(model_name)
 
 messages = [