Update app.py

app.py

@@ -2,6 +2,9 @@ from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 from transformers import pipeline
 import torch
 import gradio as gr
+from huggingface_hub import InferenceClient
+
+client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
 # chatgpt-gpt4-prompts-bart-large-cnn-samsum
 tokenizer = AutoTokenizer.from_pretrained(
@@ -10,7 +13,7 @@ model = AutoModelForSeq2SeqLM.from_pretrained(
     "Kaludi/chatgpt-gpt4-prompts-bart-large-cnn-samsum", from_tf=True)
 
 # zephyr
-    pipe = pipeline("text-generation", model="HuggingFaceH4/zephyr-7b-alpha",torch_dtype=torch.bfloat16, device_map="auto")
+    # pipe = pipeline("text-generation", model="HuggingFaceH4/zephyr-7b-alpha",torch_dtype=torch.bfloat16, device_map="auto")
 
 def generate(inputuno, inputdos, max_new_tokens=3556, top_p=0.95, repetition_penalty=1.0):
     top_p = float(top_p)
@@ -35,15 +38,15 @@ def generate(inputuno, inputdos, max_new_tokens=3556, top_p=0.95, repetition_pen
             "role": "user", "content": str(promptdos)
         },
     ]
-
-    stream = pipe.tokenizer.apply_chat_template(messages, **generate_kwargs, stream=True, details=True, return_full_text=False)
+    
+    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
     output = ""
 
     for response in stream:
         output += response.token.text
         yield output
     return output
-
+    
 
 def generatePrompt(inputuno, inputdos):