Create app.py

app.py

@@ -0,0 +1,35 @@
+import gradio as gr
+from peft import AutoPeftModelForCausalLM
+from transformers import AutoTokenizer, GPTQConfig, GenerationConfig
+
+gptq_config = GPTQConfig(bits=4, disable_exllama=True)
+model = AutoPeftModelForCausalLM.from_pretrained(
+    "Aneeth/zephyr_10k",
+    return_dict=True,
+    torch_dtype=torch.float32,
+    trust_remote_code=True,
+    quantization_config=gptq_config
+)
+
+tokenizer = AutoTokenizer.from_pretrained("Aneeth/zephyr_10k")
+generation_config = GenerationConfig(
+    do_sample=True,
+    top_k=1,
+    temperature=0.5,
+    max_new_tokens=5000,
+    pad_token_id=tokenizer.eos_token_id,
+)
+
+def process_data_sample(example):
+    processed_example = "\n Generate an authentic job description using the given input.\n\n" + example["instruction"] + "\n\n"
+    return processed_example
+
+def generate_text(prompt):
+    inp_str = process_data_sample({"instruction": prompt})
+    inputs = tokenizer(inp_str, return_tensors="pt").to("cpu")
+    outputs = model.generate(**inputs, generation_config=generation_config)
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return response
+
+iface = gr.Interface(fn=generate_text, inputs="text", outputs="text", live=True)
+iface.launch()