File size: 1,358 Bytes
05adf37 a240210 05adf37 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
import gradio as gr
from peft import AutoPeftModelForCausalLM
from transformers import AutoTokenizer, GPTQConfig, GenerationConfig
from peft import AutoPeftModelForCausalLM
from transformers import GenerationConfig
from transformers import AutoTokenizer, GPTQConfig
import torch
gptq_config = GPTQConfig(bits=4, disable_exllama=True)
model = AutoPeftModelForCausalLM.from_pretrained(
"Aneeth/zephyr_10k",
return_dict=True,
torch_dtype=torch.float32,
trust_remote_code=True,
quantization_config=gptq_config
)
tokenizer = AutoTokenizer.from_pretrained("Aneeth/zephyr_10k")
generation_config = GenerationConfig(
do_sample=True,
top_k=1,
temperature=0.5,
max_new_tokens=5000,
pad_token_id=tokenizer.eos_token_id,
)
def process_data_sample(example):
processed_example = "\n Generate an authentic job description using the given input.\n\n" + example["instruction"] + "\n\n"
return processed_example
def generate_text(prompt):
inp_str = process_data_sample({"instruction": prompt})
inputs = tokenizer(inp_str, return_tensors="pt").to("cpu")
outputs = model.generate(**inputs, generation_config=generation_config)
response = tokenizer.decode(outputs[0], skip_special_tokens=True)
return response
iface = gr.Interface(fn=generate_text, inputs="text", outputs="text", live=True)
iface.launch()
|