Update README.md

4bit QTZ inference

README.md

@@ -121,15 +121,32 @@ The model was fine-tuned on a custom dataset (`data.jsonl`) consisting of:
 ## Example Usage
 
 ```python
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+import torch
 
-tokenizer = AutoTokenizer.from_pretrained("techpro-saida/msci_software_engineering_slm_v1")
-model = AutoModelForCausalLM.from_pretrained("techpro-saida/msci_software_engineering_slm_v1")
+model_id = "techpro-saida/msci_software_engineering_slm_v1"
+
+# 4-bit config for efficient inference
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.bfloat16,
+    bnb_4bit_quant_type="nf4",
+)
+
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    quantization_config=bnb_config,
+    device_map="auto",  # automatically balances between GPU/CPU
+)
 
 prompt = "Explain SOLID principles in OOP?"
-inputs = tokenizer(prompt, return_tensors="pt")
-outputs = model.generate(**inputs, max_new_tokens=100, temperature=0.7)
+inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+
+outputs = model.generate(**inputs, max_new_tokens=100, temperature=0.7, top_p=0.9)
 print(tokenizer.decode(outputs[0], skip_special_tokens=True))
+
 ```
 
 ## Developer