kaividlabs/qwen3-4b-awq

Inference

pip install vllm

import os
from vllm import LLM, SamplingParams

os.environ["LD_LIBRARY_PATH"] = "/usr/local/lib/python3.12/dist-packages/nvidia/cu13/lib:" + os.environ.get("LD_LIBRARY_PATH", "")
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"

llm = LLM(model="kaividlabs/qwen3-4b-awq", quantization="awq_marlin")
sampling_params = SamplingParams(temperature=0.7, top_p=0.8, max_tokens=512)

messages = [{"role": "user", "content": "Give me a short introduction to Formula-1"}]
prompt = llm.get_tokenizer().apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=False)

print(outputs[0].outputs[0].text)