| import gradio as gr |
| from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig |
| from peft import PeftModel |
| import torch |
|
|
| bnb_config = BitsAndBytesConfig( |
| load_in_4bit=True, |
| bnb_4bit_quant_type="nf4", |
| bnb_4bit_compute_dtype=torch.bfloat16 |
| ) |
|
|
| tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.3") |
|
|
| base_model = AutoModelForCausalLM.from_pretrained( |
| "mistralai/Mistral-7B-Instruct-v0.3", |
| quantization_config=bnb_config, |
| device_map="auto" |
| ) |
|
|
| model = PeftModel.from_pretrained(base_model, "turuncgil/mistral-tenderbot") |
|
|
| def chat(message): |
| inputs = tokenizer(f"[INST] {message} [/INST]", return_tensors="pt").to(model.device) |
| outputs = model.generate(**inputs, max_new_tokens=200, temperature=0.7) |
| response = tokenizer.decode(outputs[0], skip_special_tokens=True) |
| return response.split("[/INST]")[-1].strip() |
|
|
| gr.Interface( |
| fn=chat, |
| inputs=gr.Textbox(label="Your question"), |
| outputs=gr.Textbox(label="Tenderbot response"), |
| title="Tenderbot - Open Energy Ontology Assistant" |
| ).launch() |
