test

app.py

@@ -7,112 +7,97 @@ model_name = "lmsys/vicuna-7b-v1.3"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 model = AutoModelForCausalLM.from_pretrained(model_name)
 
-## Task 1
-# msg = template_all.format(text)
-template_all = "Output the <Noun, Verb, Adjective, Adverb, Preposition/Subord, Coordinating Conjunction, Cardinal Number, Determiner, Noun Phrase, Verb Phrase, Adjective Phrase, Adverb Phrase, Preposition Phrase, Conjunction Phrase, Coordinate Phrase, Quantitave Phrase, Complex Nominal, Clause, Dependent Clause, Fragment Clause, T-unit, Complex T-unit, Fragment T-unit> in the following sentence without additional text in json format: "
-# msg = template_single.format(ents_prompt[eid], text)
-# template_single = "Output any <{}> in the following sentence one per line without additional text: "
+template_single = '''Please output any <{}> in the following sentence one per line without any additional text: "{}"'''
 
-## Task 2
-task2_pos = "POS tag the following sentence using Universal POS tag set without generating additional text: "
-task2_chunk = "Do sentence chunking for the following sentence as in CoNLL 2000 shared task without generating addtional text: "
-
-## Task 3
+linguistic_entities = [
+    "Noun",
+    "Determiner",
+    "Noun phrase",
+    "Verb phrase",
+    "Dependent Clause",
+    "T-units"
+]
 
 with gr.Blocks() as demo:
     gr.Markdown("# LLM Evaluator With Linguistic Scrutiny")
 
-    with gr.Tab("POS"):
-        gr.Markdown(" Description ")
-
-        prompt_POS = gr.Textbox(show_label=False, placeholder="Write a prompt and press enter")
-
-        gr.Markdown("Strategy 1 QA-Based Prompting")
-        with gr.Row():
-            vicuna_S1_chatbot_POS = gr.Chatbot(label="vicuna-7b")
-            llama_S1_chatbot_POS = gr.Chatbot(label="llama-7b")
-            gpt_S1_chatbot_POS = gr.Chatbot(label="gpt-3.5")
-        clear = gr.ClearButton([prompt_POS, vicuna_S1_chatbot_POS])
-        gr.Markdown("Strategy 2 Instruction-Based Prompting")
-        with gr.Row():
-            vicuna_S2_chatbot_POS = gr.Chatbot(label="vicuna-7b")
-            llama_S2_chatbot_POS = gr.Chatbot(label="llama-7b")
-            gpt_S2_chatbot_POS = gr.Chatbot(label="gpt-3.5")
-        clear = gr.ClearButton([prompt_POS, vicuna_S2_chatbot_POS])
-        gr.Markdown("Strategy 3 Structured Prompting")
-        with gr.Row():
-            vicuna_S3_chatbot_POS = gr.Chatbot(label="vicuna-7b")
-            llama_S3_chatbot_POS = gr.Chatbot(label="llama-7b")
-            gpt_S3_chatbot_POS = gr.Chatbot(label="gpt-3.5")
-        clear = gr.ClearButton([prompt_POS, vicuna_S3_chatbot_POS])
     
-    with gr.Tab("Chunk"):
-        gr.Markdown(" Description ")
+    gr.Markdown(" Description ")
 
-        prompt_CHUNK = gr.Textbox(show_label=False, placeholder="Write a prompt and press enter")
+    # Dropdown for linguistic entities
+    entity_dropdown = gr.Dropdown(linguistic_entities, label="Select Linguistic Entity")
 
-        gr.Markdown("Strategy 1 QA")
-        with gr.Row():
-            vicuna_S1_chatbot_CHUNK = gr.Chatbot(label="vicuna-7b")
-            llama_S1_chatbot_CHUNK = gr.Chatbot(label="llama-7b")
-            gpt_S1_chatbot_CHUNK = gr.Chatbot(label="gpt-3.5")
-        clear = gr.ClearButton([prompt_CHUNK, vicuna_S1_chatbot_CHUNK])
-        gr.Markdown("Strategy 2 Instruction")
-        with gr.Row():
-            vicuna_S2_chatbot_CHUNK = gr.Chatbot(label="vicuna-7b")
-            llama_S2_chatbot_CHUNK = gr.Chatbot(label="llama-7b")
-            gpt_S2_chatbot_CHUNK = gr.Chatbot(label="gpt-3.5")
-        clear = gr.ClearButton([prompt_CHUNK, vicuna_S2_chatbot_CHUNK])
-        gr.Markdown("Strategy 3 Structured Prompting")
-        with gr.Row():
-            vicuna_S3_chatbot_CHUNK = gr.Chatbot(label="vicuna-7b")
-            llama_S3_chatbot_CHUNK = gr.Chatbot(label="llama-7b")
-            gpt_S3_chatbot_CHUNK = gr.Chatbot(label="gpt-3.5")
-        clear = gr.ClearButton([prompt_CHUNK, vicuna_S3_chatbot_CHUNK])
+    prompt_POS = gr.Textbox(show_label=False, placeholder="Write a prompt and press enter")
 
-    def respond(message, chat_history):
-        input_ids = tokenizer.encode(message, return_tensors="pt")
-        output_ids = model.generate(input_ids, max_length=50, num_beams=5, no_repeat_ngram_size=2)
-        bot_message = tokenizer.decode(output_ids[0], skip_special_tokens=True)
-        
-        chat_history.append((message, bot_message))
-        time.sleep(2)
-        return "", chat_history
+    gr.Markdown("Strategy 1 QA-Based Prompting")
+    with gr.Row():
+        vicuna_S1_chatbot_POS = gr.Chatbot(label="vicuna-7b")
+        llama_S1_chatbot_POS = gr.Chatbot(label="llama-7b")
+        gpt_S1_chatbot_POS = gr.Chatbot(label="gpt-3.5")
+    clear = gr.ClearButton([prompt_POS, vicuna_S1_chatbot_POS])
+    gr.Markdown("Strategy 2 Instruction-Based Prompting")
+    with gr.Row():
+        vicuna_S2_chatbot_POS = gr.Chatbot(label="vicuna-7b")
+        llama_S2_chatbot_POS = gr.Chatbot(label="llama-7b")
+        gpt_S2_chatbot_POS = gr.Chatbot(label="gpt-3.5")
+    clear = gr.ClearButton([prompt_POS, vicuna_S2_chatbot_POS])
+    gr.Markdown("Strategy 3 Structured Prompting")
+    with gr.Row():
+        vicuna_S3_chatbot_POS = gr.Chatbot(label="vicuna-7b")
+        llama_S3_chatbot_POS = gr.Chatbot(label="llama-7b")
+        gpt_S3_chatbot_POS = gr.Chatbot(label="gpt-3.5")
+    clear = gr.ClearButton([prompt_POS, vicuna_S3_chatbot_POS])
     
-    def strategy1(message, chat_history):
-        input_ids = tokenizer.encode(template_all + message, return_tensors="pt")
-        output_ids = model.generate(input_ids, max_length=50, num_beams=5, no_repeat_ngram_size=2)
-        bot_message = tokenizer.decode(output_ids[0], skip_special_tokens=True)
-        
-        chat_history.append((template_all + message, bot_message))
-        time.sleep(2)
-        return "", chat_history
     
-    def strategy2_POS(message, chat_history):
-        input_ids = tokenizer.encode(task2_pos + message, return_tensors="pt")
-        output_ids = model.generate(input_ids, max_length=50, num_beams=5, no_repeat_ngram_size=2)
-        bot_message = tokenizer.decode(output_ids[0], skip_special_tokens=True)
+    gr.Markdown(" Description ")
+
+    prompt_CHUNK = gr.Textbox(show_label=False, placeholder="Write a prompt and press enter")
+
+    gr.Markdown("Strategy 1 QA")
+    with gr.Row():
+        vicuna_S1_chatbot_CHUNK = gr.Chatbot(label="vicuna-7b")
+        llama_S1_chatbot_CHUNK = gr.Chatbot(label="llama-7b")
+        gpt_S1_chatbot_CHUNK = gr.Chatbot(label="gpt-3.5")
+    clear = gr.ClearButton([prompt_CHUNK, vicuna_S1_chatbot_CHUNK])
+    gr.Markdown("Strategy 2 Instruction")
+    with gr.Row():
+        vicuna_S2_chatbot_CHUNK = gr.Chatbot(label="vicuna-7b")
+        llama_S2_chatbot_CHUNK = gr.Chatbot(label="llama-7b")
+        gpt_S2_chatbot_CHUNK = gr.Chatbot(label="gpt-3.5")
+    clear = gr.ClearButton([prompt_CHUNK, vicuna_S2_chatbot_CHUNK])
+    gr.Markdown("Strategy 3 Structured Prompting")
+    with gr.Row():
+        vicuna_S3_chatbot_CHUNK = gr.Chatbot(label="vicuna-7b")
+        llama_S3_chatbot_CHUNK = gr.Chatbot(label="llama-7b")
+        gpt_S3_chatbot_CHUNK = gr.Chatbot(label="gpt-3.5")
+    clear = gr.ClearButton([prompt_CHUNK, vicuna_S3_chatbot_CHUNK])
+
+    # def respond(message, chat_history):
+    #     input_ids = tokenizer.encode(message, return_tensors="pt")
+    #     output_ids = model.generate(input_ids, max_length=50, num_beams=5, no_repeat_ngram_size=2)
+    #     bot_message = tokenizer.decode(output_ids[0], skip_special_tokens=True)
         
-        chat_history.append((task2_pos + message, bot_message))
-        time.sleep(2)
-        return "", chat_history
+    #     chat_history.append((message, bot_message))
+    #     time.sleep(2)
+    #     return "", chat_history
     
-    def strategy2_CHUNK(message, chat_history):
-        input_ids = tokenizer.encode(task2_chunk + message, return_tensors="pt")
+    def respond_entities(message, chat_history):
+        entity = entity_dropdown.value
+        prompt = template_single.format(entity, message)
+        input_ids = tokenizer.encode(prompt, return_tensors="pt")
         output_ids = model.generate(input_ids, max_length=50, num_beams=5, no_repeat_ngram_size=2)
         bot_message = tokenizer.decode(output_ids[0], skip_special_tokens=True)
         
-        chat_history.append((task2_chunk + message, bot_message))
+        chat_history.append((message, bot_message))
         time.sleep(2)
         return "", chat_history
-    
 
-    prompt_POS.submit(strategy1, [prompt_POS, vicuna_S1_chatbot_POS], [prompt_POS, vicuna_S1_chatbot_POS])
-    prompt_POS.submit(strategy2_POS, [prompt_POS, vicuna_S2_chatbot_POS], [prompt_POS, vicuna_S2_chatbot_POS])
-    prompt_POS.submit(respond, [prompt_POS, vicuna_S3_chatbot_POS], [prompt_POS, vicuna_S3_chatbot_POS])
+    prompt_POS.submit(respond_entities, [prompt_POS, vicuna_S1_chatbot_POS], [prompt_POS, vicuna_S1_chatbot_POS])
+    prompt_POS.submit(respond_entities, [prompt_POS, vicuna_S2_chatbot_POS], [prompt_POS, vicuna_S2_chatbot_POS])
+    prompt_POS.submit(respond_entities, [prompt_POS, vicuna_S3_chatbot_POS], [prompt_POS, vicuna_S3_chatbot_POS])
 
-    prompt_CHUNK.submit(strategy1, [prompt_CHUNK, vicuna_S1_chatbot_CHUNK], [prompt_CHUNK, vicuna_S1_chatbot_CHUNK])
-    prompt_CHUNK.submit(strategy2_CHUNK, [prompt_CHUNK, vicuna_S2_chatbot_CHUNK], [prompt_CHUNK, vicuna_S2_chatbot_CHUNK])
-    prompt_CHUNK.submit(respond, [prompt_CHUNK, vicuna_S3_chatbot_CHUNK], [prompt_CHUNK, vicuna_S3_chatbot_CHUNK])
+    prompt_CHUNK.submit(respond_entities, [prompt_CHUNK, vicuna_S1_chatbot_CHUNK], [prompt_CHUNK, vicuna_S1_chatbot_CHUNK])
+    prompt_CHUNK.submit(respond_entities, [prompt_CHUNK, vicuna_S2_chatbot_CHUNK], [prompt_CHUNK, vicuna_S2_chatbot_CHUNK])
+    prompt_CHUNK.submit(respond_entities, [prompt_CHUNK, vicuna_S3_chatbot_CHUNK], [prompt_CHUNK, vicuna_S3_chatbot_CHUNK])
 
 demo.launch()