| # Step 1: Install required package | |
| # !pip install openvino-genai | |
| import openvino_genai as ov_genai | |
| # Step 2: Define local model path | |
| model_path = "/home/anish/Desktop/Anish/Openvino/Gemma-3-1b-it-ov-sym-int4/" | |
| # Step 3: Initialize pipeline | |
| device = "CPU" # or "GPU" if supported | |
| pipe = ov_genai.LLMPipeline(model_path, device) | |
| # Step 4: Set chat template (important for Qwen) | |
| tokenizer = pipe.get_tokenizer() | |
| tokenizer.set_chat_template(tokenizer.chat_template) | |
| # Step 5: Run inference | |
| prompt = "Capital of Australia ?" | |
| response = pipe.generate(prompt, max_length=1024, temperature=0.7, top_p=0.9) | |
| print("\n🧾 Model Response:") | |
| print(response) | |