Update README.md

README.md

@@ -132,6 +132,7 @@ Note:
 |                           | AlignBench v1.1             | 6.8             | 5.99        | 6.95               | 6.3           | 6.33     |
 |                           | LiveBench 1125              | 48.7            | 25.5        | 52.1               | 43.1          | 40       |
 
+
 ## 3. Quick start
 
 ### Inference with huggingface
@@ -144,10 +145,10 @@ You can now inference in Transformers starting from version `4.56.0`.
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM
 
-model_name = "/path/to/Klear-Base"
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+model_path = "/path/to/Klear-Base"
+tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
     
-model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", dtype=torch.bfloat16, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(model_path, device_map="auto", dtype=torch.bfloat16, trust_remote_code=True)
 
 text = "世界上最大的湖是"
 inputs = tokenizer(text, return_tensors="pt")
@@ -162,10 +163,10 @@ print(result)
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
 
-model_name = "/path/to/Klear-Inst."
-tokenizer = AutoTokenizer.from_pretrained(model_name)
+model_path = "/path/to/Klear-Inst."
+tokenizer = AutoTokenizer.from_pretrained(model_path)
 
-model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", torch_dtype=torch.bfloat16)
+model = AutoModelForCausalLM.from_pretrained(model_path, device_map="auto", torch_dtype=torch.bfloat16)
 
 messages = [
     {"role": "user", "content": "帮我用 python 写一个计算器的代码吧。"}
@@ -193,36 +194,34 @@ An OpenAI-compatible API will be available at `http://localhost:8000/v1`.
 Or you can refer to the following Python script for offline inference
 ```python
 from vllm import LLM, SamplingParams
+from transformers import AutoTokenizer
+
+model_path = "/path/to/Klear-Inst."
+tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
 
-model_path = "/path/to/Klear"
 llm = LLM(
     model=model_path,
     trust_remote_code=True,
-    num_speculative_tokens=1,
-    disable_log_stats=False
+    tensor_parallel_size=torch.cuda.device_count(),
+    gpu_memory_utilization=0.7
 )
-sampling_params = SamplingParams(temperature=0.2)
-
-conversation = [
-    {
-        "role": "system",
-        "content": ""
-    },
-    {
-        "role": "user",
-        "content": "Please help me write a snake game code.",
-    },
+messages = [
+    {"role": "user", "content": "请介绍一下快手"}
 ]
 
-outputs = llm.chat(conversation,
-                   sampling_params=sampling_params,
-                   use_tqdm=False)
+prompt = tokenizer.apply_chat_template(
+    messages,
+    tokenize=False,
+    add_generation_prompt=True
+)
+
+sampling_params = SamplingParams(
+    temperature=0.6, top_p=0.8, max_tokens=512
+)
+
+outputs = llm.generate([prompt], sampling_params)
 
-for idx, output in enumerate(outputs):
-    prompt = output.prompt
-    generated_text = output.outputs[0].text
-    print(f"==== Response #{idx} ====")
-    print(f"Prompt: {prompt}, Generated text: {generated_text}")
+print(outputs[0].outputs[0].text)
 
 ```