Update README.md

README.md

@@ -47,6 +47,7 @@ from transformers import AutoTokenizer
 
 model_id = "neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8"
 number_gpus = 1
+max_model_len = 8192
 
 sampling_params = SamplingParams(temperature=0.6, top_p=0.9, max_tokens=256)
 
@@ -59,7 +60,7 @@ messages = [
 
 prompts = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
 
-llm = LLM(model=model_id, tensor_parallel_size=number_gpus)
+llm = LLM(model=model_id, tensor_parallel_size=number_gpus, max_model_len=max_model_len)
 
 outputs = llm.generate(prompts, sampling_params)