<ADD> update app

.gitignore

@@ -0,0 +1,6 @@
+.vscode
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class

app.py

@@ -15,7 +15,6 @@ from tasks.loader import TokenizedForMCRightPad
 
 DISPLAY_MAPPING = {
     "sst2": {"positive": "Pos", "negative": "Neg"},
-    "trec": {},
 }
 
 
@@ -78,13 +77,21 @@ def process_once(dataset_name, exemplar_str, forward_steps, raw_data):
         generated_info.extend(zipped_logprobs)
 
     all_predicted = []
+    num_correct = 0
     for idx, (data, choice_info) in enumerate(zip(processed_data, generated_info)):
         merged_choice_info = task_agent.merge_choice_info(choice_info)
         merged_predictions_idx = task_agent.choice_info_to_predictions(merged_choice_info)["lm_log_p"]
         predicted = task_agent.CHOICES[merged_predictions_idx]
         ground_truth = task_agent.CHOICES[data["answer_idx"]]
-        res = f"{DISPLAY_MAPPING[dataset_name][predicted]}{'✅' if predicted == ground_truth else '❌'}"
+
+        res = f"{DISPLAY_MAPPING[dataset_name][predicted]}"
+        if predicted == ground_truth:
+            res += " ✅"
+            num_correct += 1
+        else:
+            res += " ❌"
         all_predicted.append(res)
+    all_predicted.append(f"{100*num_correct / len(all_predicted):.2f}%")
     return all_predicted
 
 
@@ -102,7 +109,10 @@ def button_pressed(prev_state):
     current_output = process_once(dataset_name, exemplar_str, forward_steps, raw_data)
 
     t_prev = transpose(prev_table_data)
-    t_prev.append([f"T={forward_steps}"] + current_output)
+    if forward_steps == 1:
+        t_prev.append(["**ICL**"] + current_output)
+    else:
+        t_prev.append([f"**Step={forward_steps}**"] + current_output)
     updated_table_data = transpose(t_prev)
 
     ret = [
@@ -113,7 +123,7 @@ def button_pressed(prev_state):
             "step": forward_steps,
             "table_data": updated_table_data,
         },
-        f"Step + 2, Now: {forward_steps}",
+        f"Click here to train LLM ! Now Step: {forward_steps}",
         updated_table_data,
     ]
     return ret
@@ -138,37 +148,70 @@ if __name__ == "__main__":
     with task_root.joinpath("demos.txt").open("r") as f:
         demos = f.read()
     with task_root.joinpath("sample.pkl").open("r") as f:
-        data = json.load(f)
-        raw_data = [data[str(i)] for i in range(len(data))]
+        raw_data = json.load(f)
+
+    icl_result = process_once(dataset_name, demos, 1, raw_data)
+
+    text = """We utilize a Large Language Model (LLM) to perform in-context learning (ICL) for sentiment classification of movie reviews.
+    
+Taking the following two labeled examples as demonstrations, we predict the sentiment of the subsequent test input.
 
-    css = """ #the-table > div > div > div > table > thead {display: none}"""
+Directly employing ICL results in lower prediction accuracy. However, in our proposed approach, **Deep-Thinking**, we repeatedly apply **Forward Tuning**, leading to improved accuracy of the model."""
+
+    css = """
+#the-table { overflow: auto; }
+#the-table > div:nth-child(2) { margin: auto; width: fit-content; }
+#the-table > div > div > div > table { width: auto; margin: 0; white-space: normal; }
+#the-table > div > div > div > table > thead {display: none}
+#the-table > div > div > div > table > tbody > tr:last-child {background-color: beige}
+#the-table > div > div > div > table > tbody > tr:first-child {background-color: lightgray}
+#the-table > div > div > div > table > tbody > tr > td:first-child {min-width: 300px;}
+#the-table > div > div > div > table > tbody > tr > td:not(:first-child) {white-space: nowrap; padding: 0 2px; }
+#the-text { font-size: large; }
+    """
 
     title = "🤔 Iterative Forward Tuning Boosts In-context Learning in Language Models"
     demo = gr.Blocks(css=css, title="🤔Deep-Thinking")
     with demo:
         gr.Markdown(f"<h1 style='text-align: center; margin-bottom: 1rem'>{title}</h1>")
+        gr.Markdown(
+            """
+<h2 style='text-align: center; margin-bottom: 1rem'>
+<a href='https://arxiv.org/abs/2305.13016' target="_blank" style='text-decoration: none'>[Paper]</a>
+<a href='https://arxiv.org/abs/2305.13016' target="_blank" style='text-decoration: none'>[Code]</a>
+</h2>"""
+        )
+
+        gr.Markdown(text, elem_id="the-text")
         with gr.Tab("SST-2"):
             mapping = ["negative", "positive"]
 
-            init_columns = [[e["sentence"], f"*{DISPLAY_MAPPING['sst2'][mapping[e['label']]]}*"] for e in raw_data]
+            init_columns = [[e["sentence"]] for e in raw_data]
+
+            init_table_result = [["**Test Input**"], *init_columns, ["**Accuracy**"]]
+            init_table_result = transpose(init_table_result)
+            init_table_result.append(["**ICL**"] + icl_result)
+            init_table_result = transpose(init_table_result)
+
             state = gr.State(
                 {
                     "dataset_name": "sst2",
                     "exemplar_str": demos,
                     "raw_data": raw_data,
-                    "step": 0,
-                    "table_data": [["**Test Input**", "**Golden**"], *init_columns],
+                    "step": 1,
+                    "table_data": init_table_result,
                 }
             )
 
             prompt = gr.Textbox(label="Demonstrations (Prompt template formatted)", value=demos)
+            gr.Markdown("<h2 style='text-align: center; margin-bottom: 1rem'>👇 Run forward tuning once !</h2>")
+            step_button = gr.Button("Click here to train LLM ! Now Step: 1")
             big_table = gr.DataFrame(
-                value=[["**Test Input**", "**Golden**"], *init_columns],
+                value=init_table_result,
                 elem_id="the-table",
                 datatype=["markdown"] * 50,
                 headers=None,
             )
-            step_button = gr.Button("Step + 2, Now: 0")
             step_button.click(button_pressed, inputs=[state], outputs=[state, step_button, big_table])
 
     demo.launch(server_name="0.0.0.0")

example_sets/sst2/sample.pkl

@@ -1,10 +1,14 @@
-{
-    "0": { "sentence": "the cold turkey would 've been a far better title . ", "label": 0, "idx": 57 },
-    "1": { "sentence": "it 's a cookie-cutter movie , a cut-and-paste job . ", "label": 0, "idx": 28 },
-    "2": { "sentence": "a solid film ... but more conscientious than it is truly stirring . ", "label": 1, "idx": 143 },
-    "3": { "sentence": "it 's slow -- very , very slow . ", "label": 0, "idx": 4 },
-    "4": { "sentence": "filmmakers who can deftly change moods are treasures and even marvels . ", "label": 1, "idx": 679 },
-    "5": { "sentence": "it all adds up to good fun . ", "label": 1, "idx": 393 },
-    "6": { "sentence": "i am sorry that i was unable to get the full brunt of the comedy . ", "label": 0, "idx": 423 },
-    "7": { "sentence": "hilariously inept and ridiculous . ", "label": 1, "idx": 112 }
-}
+[
+    {"sentence":"... the movie is just a plain old monster . ","label":0,"idx":18},
+    {"sentence": "overall very good for what it 's trying to do . ", "label": 1, "idx": 150},
+    {"sentence": "it has all the excitement of eating oatmeal . ", "label": 0, "idx": 527},
+    {"sentence": "and when you 're talking about a slapstick comedy , that 's a pretty big problem . ", "label": 0, "idx": 748},
+    {"sentence": "and that 's a big part of why we go to the movies . ", "label": 1, "idx": 505},
+    {"sentence": "a good piece of work more often than not . ", "label": 1, "idx": 424},
+    {"sentence": "the cold turkey would 've been a far better title . ", "label": 0, "idx": 57},
+    {"sentence": "it 's slow -- very , very slow . ", "label": 0, "idx": 4},
+    {"sentence": "it 's a cookie-cutter movie , a cut-and-paste job . ", "label": 0, "idx": 28},
+    {"sentence": "i am sorry that i was unable to get the full brunt of the comedy . ", "label": 0, "idx": 423},
+    {"sentence": "filmmakers who can deftly change moods are treasures and even marvels . ", "label": 1, "idx": 679},
+    {"sentence": "a solid film ... but more conscientious than it is truly stirring . ", "label": 1, "idx": 143}
+]