Update README.md

README.md

@@ -53,22 +53,39 @@ Training details:
 
 # 2. Usage
 
+## 2.1 Usage of Our BF16 Model
+
+1. Please upgrade the `transformers` package to ensure it supports Llama3.1 models. The current version we are using is `4.43.0`.
+
+2. Use the following Python script to download our BF16 model
+
 ```python
-from transformers import AutoTokenizer, AutoModelForCausalLM
+from huggingface_hub import snapshot_download
+snapshot_download(repo_id="shenzhi-wang/Llama3.1-8B-Chinese-Chat", ignore_patterns=["*.gguf"])  # Download our BF16 model without downloading GGUF models.
+```
 
-model_id = "shenzhi-wang/Llama3.1-8B-Chinese-Chat"
+3. Inference with the BF16 model
+
+```python
+import torch
+import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_id = "/Your/Local/Path/to/Llama3.1-8B-Chinese-Chat"
+dtype = torch.bfloat16
 
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 model = AutoModelForCausalLM.from_pretrained(
-    model_id, torch_dtype="auto", device_map="auto"
+    model_id,
+    device_map="cuda",
+    torch_dtype=dtype,
 )
 
-messages = [
-    {"role": "user", "content": "写一首诗吧"},
+chat = [
+    {"role": "user", "content": "写一首关于机器学习的诗。"},
 ]
-
 input_ids = tokenizer.apply_chat_template(
-    messages, add_generation_prompt=True, return_tensors="pt"
+    chat, tokenize=True, add_generation_prompt=True, return_tensors="pt"
 ).to(model.device)
 
 outputs = model.generate(
@@ -78,6 +95,12 @@ outputs = model.generate(
     temperature=0.6,
     top_p=0.9,
 )
-response = outputs[0][input_ids.shape[-1]:]
+response = outputs[0][input_ids.shape[-1] :]
 print(tokenizer.decode(response, skip_special_tokens=True))
-```
+```
+
+## 2.2 Usage of Our GGUF Models
+
+1. Download our GGUF models from the [gguf_models folder](https://huggingface.co/shenzhi-wang/Llama3.1-8B-Chinese-Chat/tree/main/gguf);
+2. Use the GGUF models with [LM Studio](https://lmstudio.ai/);
+3. Or, you can follow the instructions from https://github.com/ggerganov/llama.cpp/tree/master#usage to use gguf models.