Update README.md

README.md

@@ -83,6 +83,56 @@ peft_model_id = "fc91/phi3-mini-instruct-full_ethics-lora"
 model = PeftModel.from_pretrained(base_model, peft_model_id)
 ```
 
+Run the model with a quantization configuration
+
+```markdown
+import torch, accelerate, peft
+from transformers import AutoModelForCausalLM, BitsAndBytesConfig, pipeline
+from peft import PeftModel
+
+# Set up quantization configuration
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=getattr(torch, "float16")
+)
+
+# Load the base model with quantization
+base_model = AutoModelForCausalLM.from_pretrained(
+    "microsoft/Phi-3-mini-4k-instruct",
+    quantization_config=quantization_config,
+    device_map="auto",
+    attn_implementation='eager',
+    torch_dtype="auto",
+    trust_remote_code=True,
+)
+
+peft_model_id = "fc91/phi3-mini-instruct-full_ethics-lora"
+model = PeftModel.from_pretrained(base_model, peft_model_id)
+
+messages = [ 
+    {"role": "system", "content": "You are a helpful AI assistant that grounds all of its replies in ethical theories."}, 
+    {"role": "user", "content": """I am driving a car and I have to make a choice. A kid suddenly appear in the middle of the road chasing a ball. To save the kid, I
+    can only swerve to the right, but this would entails crashing the car against two pedstrian on the sidewalk. What should I do?"""}, 
+] 
+
+pipe = pipeline( 
+    "text-generation", 
+    model=model, 
+    tokenizer=tokenizer, 
+) 
+
+generation_args = { 
+    "max_new_tokens": 1000, 
+    "return_full_text": False, 
+    "temperature": 0.5, 
+    "do_sample": False, 
+} 
+
+output = pipe(messages, **generation_args) 
+print(output[0]['generated_text']) 
+```
+
 ## Training Details
 
 ### Training Data