RedHatAI
/

Meta-Llama-3-8B-Instruct-FP8

Text Generation

text-generation-inference

Model card Files Files and versions

Lin-K76 commited on Jul 15, 2024

Commit

3a04ae8

·

verified ·

1 Parent(s): 0035cbf

Update README.md

Files changed (1) hide show

README.md +6 -1

README.md CHANGED Viewed

@@ -84,7 +84,12 @@ ds = load_dataset("mgoin/ultrachat_2k", split="train_sft").select(range(512))
 examples = [tokenizer.apply_chat_template(batch["messages"], tokenize=False) for batch in ds]
 examples = tokenizer(examples, padding=True, truncation=True, return_tensors="pt").to("cuda")
-quantize_config = BaseQuantizeConfig(quant_method="fp8", activation_scheme="static")
 model = AutoFP8ForCausalLM.from_pretrained(
     pretrained_model_dir, quantize_config=quantize_config

 examples = [tokenizer.apply_chat_template(batch["messages"], tokenize=False) for batch in ds]
 examples = tokenizer(examples, padding=True, truncation=True, return_tensors="pt").to("cuda")
+quantize_config = BaseQuantizeConfig(
+    quant_method="fp8",
+    activation_scheme="static"
+    ignore_patterns=["re:.*lm_head"],
+)
 model = AutoFP8ForCausalLM.from_pretrained(
     pretrained_model_dir, quantize_config=quantize_config