min example

app.py

@@ -1,44 +1,42 @@
 import gradio as gr
 from huggingface_hub import InferenceClient
-from transformers import AutoModelForCausalLM, AutoTokenizer
+# from transformers import AutoModelForCausalLM, AutoTokenizer
 # import spaces
 
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
 # client = InferenceClient("BatsResearch/bonito-v1")
-model = AutoModelForCausalLM.from_pretrained("BatsResearch/bonito-v1")
-tokenizer = AutoTokenizer.from_pretrained("BatsResearch/bonito-v1")
+# model = AutoModelForCausalLM.from_pretrained("BatsResearch/bonito-v1")
+# tokenizer = AutoTokenizer.from_pretrained("BatsResearch/bonito-v1")
 
 # move to cuda
-model.to("cuda")
+# model.to("cuda")
 
 def respond(
     context: str,
-    task_type: str,
-    max_tokens: int = 256,
-    temperature: float = 0.5,
-    top_p: float = 0.95,
 ):
-    task_type = "extractive question answering"
-    input_text = "<|tasktype|>\n" + task_type.strip()
-    input_text += (
-                "\n<|context|>\n" + context.strip() + "\n<|task|>\n"
-    )
-
-    input_ids = tokenizer.encode(input_text, return_tensors="pt").to("cuda")
-    outputs = model.generate(
-        input_ids,
-        max_new_tokens=max_tokens,
-        temperature=temperature,
-        do_sample=True,
-        top_p=top_p,
-    )
-    pred_start = int(input_ids.shape[-1])
-    pred = tokenizer.decode(outputs[pred_start:], skip_special_tokens=True)
 
-    # replace the context
 
+    # task_type = "extractive question answering"
+    # input_text = "<|tasktype|>\n" + task_type.strip()
+    # input_text += (
+    #             "\n<|context|>\n" + context.strip() + "\n<|task|>\n"
+    # )
+
+    # input_ids = tokenizer.encode(input_text, return_tensors="pt").to("cuda")
+    # outputs = model.generate(
+    #     input_ids,
+    #     max_new_tokens=max_tokens,
+    #     temperature=temperature,
+    #     do_sample=True,
+    #     top_p=top_p,
+    # )
+    # pred_start = int(input_ids.shape[-1])
+    # pred = tokenizer.decode(outputs[pred_start:], skip_special_tokens=True)
+
+    # replace the context
+    pred = ""
     return pred
 
 
@@ -87,7 +85,8 @@ examples = [
 
 demo = gr.Interface(
     respond,
-    inputs=[gr.Textbox(lines=5, label="Enter context here"), gr.Dropdown(["extractive question answering"], label="Task Type")],
+    # inputs=[gr.Textbox(lines=5, label="Enter context here"), gr.Dropdown(["extractive question answering"], label="Task Type")],
+    inputs=gr.Textbox(lines=5, label="Enter context here"),
     outputs=gr.Textbox(lines=20, label="Generated Instruction-Response Pairs"),
     additional_inputs=[
         # gr.Textbox(value="You are a friendly Chatbot.", label="System message"),

requirements.txt

@@ -1,3 +1,3 @@
 huggingface_hub==0.22.2
-transformers
-accelerate
+# transformers
+# accelerate