Upload Llama-3.1-8B quantized with ModelOpt FP8

README.md

@@ -26,23 +26,17 @@ This is a quantized version of [meta-llama/Llama-3.1-8B-Instruct](https://huggin
 ```python
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer
-from quanto import quantize, freeze, qint8, safe_load
 
 # Load base model structure
 model = AutoModelForCausalLM.from_pretrained(
-    "meta-llama/Llama-3.1-8B-Instruct",
+    "tokenlabsdotrun/Llama-3.1-8B-ModelOpt-FP8",
     torch_dtype=torch.bfloat16,
     low_cpu_mem_usage=True
 )
 
-# Quantize structure and load weights
-quantize(model, weights=qint8)
-state_dict = safe_load("model.safetensors")  # Use quanto's safe_load
-model.load_state_dict(state_dict)
-freeze(model)
-
 # Load tokenizer and generate
-tokenizer = AutoTokenizer.from_pretrained("tokenlabsdotrun/Llama-3.1-8B-Quanto-Int8")
+tokenizer = AutoTokenizer.from_pretrained("tokenlabsdotrun/Llama-3.1-8B-ModelOpt-FP8")
+
 inputs = tokenizer("Hello, my name is", return_tensors="pt")
 outputs = model.generate(**inputs, max_new_tokens=10)
 print(tokenizer.decode(outputs[0], skip_special_tokens=True))