File size: 653 Bytes
ec0c6ff |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 |
# Step 1: Install required package
# !pip install openvino-genai
import openvino_genai as ov_genai
# Step 2: Define local model path
model_path = "/home/anish/Desktop/Anish/Openvino/Gemma-3-1b-it-ov-sym-int4/"
# Step 3: Initialize pipeline
device = "CPU" # or "GPU" if supported
pipe = ov_genai.LLMPipeline(model_path, device)
# Step 4: Set chat template (important for Qwen)
tokenizer = pipe.get_tokenizer()
tokenizer.set_chat_template(tokenizer.chat_template)
# Step 5: Run inference
prompt = "Capital of Australia ?"
response = pipe.generate(prompt, max_length=1024, temperature=0.7, top_p=0.9)
print("\n🧾 Model Response:")
print(response)
|