chart for task summary

app.py

@@ -9,6 +9,7 @@ import matplotlib.pyplot as plt
 from sklearn.metrics import roc_curve, auc
 import pandas as pd
 from sklearn.metrics import roc_auc_score
+from matplotlib.figure import Figure
 # Define the function to process the input file and model selection
 
 def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
@@ -69,7 +70,7 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     indices = test_info[test_info[0].isin(random_schools)].index.tolist()
     high_indices = test_info[(test_info[0].isin(high_sample))].index.tolist()
     low_indices = test_info[(test_info[0].isin(low_sample))].index.tolist()
-
+    
     # Load the test file and select rows based on indices
     test = pd.read_csv(test_location, sep=',', header=None, engine='python')
     selected_rows_df2 = test.loc[indices]
@@ -80,7 +81,8 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     graduation_groups = [
     'high' if idx in high_indices else 'low' for idx in selected_rows_df2.index
     ]
-
+    # Group data by opt_task1 and opt_task2 based on test_info[6]
+    opt_task_groups = ['opt_task1' if test_info.loc[idx, 6] == 0 else 'opt_task2' for idx in selected_rows_df2.index]
     
     with open("roc_data2.pkl", 'rb') as file:
         data = pickle.load(file)
@@ -88,7 +90,7 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     p_label=data[1]
     # Step 1: Align graduation_group, t_label, and p_label
     aligned_labels = list(zip(graduation_groups, t_label, p_label))
-
+    opt_task_aligned = list(zip(opt_task_groups, t_label, p_label))
     # Step 2: Separate the labels for high and low groups
     high_t_labels = [t for grad, t, p in aligned_labels if grad == 'high']
     low_t_labels = [t for grad, t, p in aligned_labels if grad == 'low']
@@ -96,8 +98,18 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     high_p_labels = [p for grad, t, p in aligned_labels if grad == 'high']
     low_p_labels = [p for grad, t, p in aligned_labels if grad == 'low']
 
+    opt_task1_t_labels = [t for task, t, p in opt_task_aligned if task == 'opt_task1']
+    opt_task1_p_labels = [p for task, t, p in opt_task_aligned if task == 'opt_task1']
+
+    opt_task2_t_labels = [t for task, t, p in opt_task_aligned if task == 'opt_task2']
+    opt_task2_p_labels = [p for task, t, p in opt_task_aligned if task == 'opt_task2']
+
     high_roc_auc = roc_auc_score(high_t_labels, high_p_labels) if len(set(high_t_labels)) > 1 else None
     low_roc_auc = roc_auc_score(low_t_labels, low_p_labels) if len(set(low_t_labels)) > 1 else None
+
+    opt_task1_roc_auc = roc_auc_score(opt_task1_t_labels, opt_task1_p_labels) if len(set(opt_task1_t_labels)) > 1 else None
+    opt_task2_roc_auc = roc_auc_score(opt_task2_t_labels, opt_task2_p_labels) if len(set(opt_task2_t_labels)) > 1 else None
+
     # For demonstration purposes, we'll just return the content with the selected model name
 
     # print(checkpoint)
@@ -155,8 +167,8 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
 
     # Initialize counters
     task_counts = {
-        1: {"only_opt1": 0, "only_opt2": 0, "both": 0},
-        2: {"only_opt1": 0, "only_opt2": 0, "both": 0}
+    1: {"only_opt1": 0, "only_opt2": 0, "both": 0,"none":0},
+    2: {"only_opt1": 0, "only_opt2": 0, "both": 0,"none":0}
     }
 
     # Analyze rows
@@ -175,6 +187,8 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
                 task_counts[1]["only_opt2"] += 1
             elif opt1_done and opt2_done:
                 task_counts[1]["both"] += 1
+            else:
+                task_counts[1]["none"] +=1
         elif ideal_task == 1:
             if opt1_done and not opt2_done:
                 task_counts[2]["only_opt1"] += 1
@@ -182,32 +196,52 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
                 task_counts[2]["only_opt2"] += 1
             elif opt1_done and opt2_done:
                 task_counts[2]["both"] += 1
+            else:
+                task_counts[2]["none"] +=1
 
     # Create a string output for results
-    output_summary = "Task Analysis Summary:\n"
-    output_summary += "-----------------------\n"
+    # output_summary = "Task Analysis Summary:\n"
+    # output_summary += "-----------------------\n"
 
-    for ideal_task, counts in task_counts.items():
-        output_summary += f"Ideal Task = OptionalTask_{ideal_task}:\n"
-        output_summary += f"  Only OptionalTask_1 done: {counts['only_opt1']}\n"
-        output_summary += f"  Only OptionalTask_2 done: {counts['only_opt2']}\n"
-        output_summary += f"  Both done: {counts['both']}\n"
+    # for ideal_task, counts in task_counts.items():
+    #     output_summary += f"Ideal Task = OptionalTask_{ideal_task}:\n"
+    #     output_summary += f"  Only OptionalTask_1 done: {counts['only_opt1']}\n"
+    #     output_summary += f"  Only OptionalTask_2 done: {counts['only_opt2']}\n"
+    #     output_summary += f"  Both done: {counts['both']}\n"
+
+  # Generate pie chart for Task 1
+    task1_labels = list(task_counts[1].keys())
+    task1_values = list(task_counts[1].values())
+
+    fig_task1 = Figure()
+    ax1 = fig_task1.add_subplot(1, 1, 1)
+    ax1.pie(task1_values, labels=task1_labels, autopct='%1.1f%%', startangle=90)
+    ax1.set_title('Ideal Task 1 Distribution')
+
+    # Generate pie chart for Task 2
+    task2_labels = list(task_counts[2].keys())
+    task2_values = list(task_counts[2].values())
+
+    fig_task2 = Figure()
+    ax2 = fig_task2.add_subplot(1, 1, 1)
+    ax2.pie(task2_values, labels=task2_labels, autopct='%1.1f%%', startangle=90)
+    ax2.set_title('Ideal Task 2 Distribution')
 
     # print(output_summary)
 
     progress(0.2, desc="analysis done!! Executing models")
     print("finetuned task: ",finetune_task)
-    subprocess.run([
-        "python", "new_test_saved_finetuned_model.py",
-        "-workspace_name", "ratio_proportion_change3_2223/sch_largest_100-coded",
-        "-finetune_task", finetune_task,
-        "-test_dataset_path","../../../../selected_rows.txt",
-        # "-test_label_path","../../../../train_label.txt",
-        "-finetuned_bert_classifier_checkpoint", 
-        "ratio_proportion_change3_2223/sch_largest_100-coded/output/highGRschool10/bert_fine_tuned.model.ep42",
-        "-e",str(1),
-        "-b",str(1000)
-    ])
+    # subprocess.run([
+    #     "python", "new_test_saved_finetuned_model.py",
+    #     "-workspace_name", "ratio_proportion_change3_2223/sch_largest_100-coded",
+    #     "-finetune_task", finetune_task,
+    #     "-test_dataset_path","../../../../selected_rows.txt",
+    #     # "-test_label_path","../../../../train_label.txt",
+    #     "-finetuned_bert_classifier_checkpoint", 
+    #     "ratio_proportion_change3_2223/sch_largest_100-coded/output/highGRschool10/bert_fine_tuned.model.ep42",
+    #     "-e",str(1),
+    #     "-b",str(1000)
+    # ])
     progress(0.6,desc="Model execution completed")
     result = {}
     with open("result.txt", 'r') as file:
@@ -225,10 +259,14 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
         fpr, tpr, _ = pickle.load(f)
     # print(fpr,tpr)
     roc_auc = auc(fpr, tpr)
-    fig, ax = plt.subplots()
+
+
+#  Create a matplotlib figure
+    fig = Figure()
+    ax = fig.add_subplot(1, 1, 1)
     ax.plot(fpr, tpr, color='blue', lw=2, label=f'ROC curve (area = {roc_auc:.2f})')
     ax.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
-    ax.set(xlabel='False Positive Rate', ylabel='True Positive Rate', title=f'(Receiver Operating Curve) ROC')
+    ax.set(xlabel='False Positive Rate', ylabel='True Positive Rate', title=f'Receiver Operating Curve (ROC)')
     ax.legend(loc="lower right")
     ax.grid()
 
@@ -247,7 +285,6 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     text_output = f"""
     Model: {model_name}\n
     -----------------\n
-
     Time Taken: {result['time_taken_from_start']:.2f} seconds\n
     Total Schools in test: {len(unique_schools):.4f}\n
     Total number of instances having Schools with HGR : {len(high_sample):.4f}\n
@@ -255,9 +292,12 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
 
     ROC score of HGR: {high_roc_auc}\n
     ROC score of LGR: {low_roc_auc}\n
+
+    ROC score of opt1: {opt_task1_roc_auc}\n
+    ROC score of opt2: {opt_task2_roc_auc}\n
     -----------------\n
     """
-    return text_output,plot_path,output_summary
+    return text_output,fig,fig_task1,fig_task2
 
 # List of models for the dropdown menu
 
@@ -507,12 +547,16 @@ tbody.svelte-18wv37q>tr.svelte-18wv37q:nth-child(odd) {
     gr.Markdown("<p class='description'>Dashboard</p>")
     with gr.Row():
         output_text = gr.Textbox(label="")
-        output_image = gr.Image(label="ROC")
-        output_summary = gr.Textbox(label="Summary")
+        # output_image = gr.Image(label="ROC")
+        plot_output = gr.Plot(label="roc")
+    with gr.Row():
+        opt1_pie = gr.Plot(label="opt1")
+        opt2_pie = gr.Plot(label="opt2")
+        # output_summary = gr.Textbox(label="Summary")
 
     btn = gr.Button("Submit")
  
-    btn.click(fn=process_file, inputs=[model_dropdown,increment_slider], outputs=[output_text,output_image,output_summary])
+    btn.click(fn=process_file, inputs=[model_dropdown,increment_slider], outputs=[output_text,plot_output,opt1_pie,opt2_pie])
 
 
 # Launch the app

distinguish_high_low_label.ipynb

@@ -2,18 +2,19 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 27,
    "id": "960bac80-51c7-4e9f-ad2d-84cd6c710f98",
    "metadata": {},
    "outputs": [],
    "source": [
     "import pickle\n",
-    "import pandas as pd"
+    "import pandas as pd\n",
+    "from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, roc_curve, roc_auc_score,auc"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "id": "a34f21d0-0854-4a54-8f93-67718b2f969e",
    "metadata": {},
    "outputs": [],
@@ -31,7 +32,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 4,
    "id": "f9febed4-ce50-4e30-96ea-4b538ce2f9a1",
    "metadata": {},
    "outputs": [],
@@ -70,7 +71,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 5,
    "id": "fdfdf4b6-2752-4a21-9880-869af69f20cf",
    "metadata": {},
    "outputs": [],
@@ -81,7 +82,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 6,
    "id": "a79a4598-5702-4cc8-9f07-8e18fdda648b",
    "metadata": {},
    "outputs": [
@@ -91,7 +92,7 @@
        "997"
       ]
      },
-     "execution_count": 7,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -102,7 +103,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 7,
    "id": "4707f3e6-2f44-46d8-ad8c-b6c244f693af",
    "metadata": {},
    "outputs": [
@@ -197,7 +198,7 @@
        "[997 rows x 1 columns]"
       ]
      },
-     "execution_count": 8,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -211,20 +212,54 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 8,
    "id": "1d0c3d49-061f-486b-9c19-cf20945f3207",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "997"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "graduation_groups = [\n",
     "    'high' if idx in high_indices else 'low' for idx in selected_rows_df2.index\n",
     "]\n",
-    "# graduation_groups"
+    "# graduation_groups\n",
+    "len(graduation_groups)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "d2508a0f-e5ca-432e-b99b-481ea4536d4d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "997"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "opt_task_groups = ['opt_task1' if test_info.loc[idx, 6] == 0 else 'opt_task2' for idx in selected_rows_df2.index]\n",
+    "len(opt_task_groups)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 43,
+   "execution_count": 10,
    "id": "ad0ce4a1-27fa-4867-8061-4054dbb340df",
    "metadata": {},
    "outputs": [],
@@ -235,21 +270,51 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 47,
+   "execution_count": 12,
    "id": "a4f4a2b9-3134-42ac-871b-4e117098cd0e",
    "metadata": {},
    "outputs": [],
    "source": [
     "# Step 1: Align graduation_group, t_label, and p_label\n",
     "aligned_labels = list(zip(graduation_groups, t_label, p_label))\n",
-    "\n",
+    "opt_task_aligned = list(zip(opt_task_groups, t_label, p_label))\n",
     "# Step 2: Separate the labels for high and low groups\n",
     "high_t_labels = [t for grad, t, p in aligned_labels if grad == 'high']\n",
     "low_t_labels = [t for grad, t, p in aligned_labels if grad == 'low']\n",
     "\n",
     "high_p_labels = [p for grad, t, p in aligned_labels if grad == 'high']\n",
     "low_p_labels = [p for grad, t, p in aligned_labels if grad == 'low']\n",
-    "\n"
+    "\n",
+    "\n",
+    "opt_task1_t_labels = [t for task, t, p in opt_task_aligned if task == 'opt_task1']\n",
+    "opt_task1_p_labels = [p for task, t, p in opt_task_aligned if task == 'opt_task1']\n",
+    "\n",
+    "opt_task2_t_labels = [t for task, t, p in opt_task_aligned if task == 'opt_task2']\n",
+    "opt_task2_p_labels = [p for task, t, p in opt_task_aligned if task == 'opt_task2']\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "74cda932-ce98-4ad5-9c29-a54bdc4ee086",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "opt_task1 ROC-AUC: 0.7592686234399062\n",
+      "opt_task2 ROC-AUC: 0.7268598353289777\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "opt_task1_roc_auc = roc_auc_score(opt_task1_t_labels, opt_task1_p_labels) if len(set(opt_task1_t_labels)) > 1 else None\n",
+    "opt_task2_roc_auc = roc_auc_score(opt_task2_t_labels, opt_task2_p_labels) if len(set(opt_task2_t_labels)) > 1 else None\n",
+    "\n",
+    "print(f\"opt_task1 ROC-AUC: {opt_task1_roc_auc}\")\n",
+    "print(f\"opt_task2 ROC-AUC: {opt_task2_roc_auc}\")"
    ]
   },
   {
@@ -275,17 +340,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 51,
+   "execution_count": 13,
    "id": "c11050db-2636-4c50-9cd4-b9943e5cee83",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, roc_curve, roc_auc_score"
-   ]
+   "source": []
   },
   {
    "cell_type": "code",
-   "execution_count": 52,
+   "execution_count": 16,
    "id": "e1309e93-7063-4f48-bbc7-11a0d449c34e",
    "metadata": {},
    "outputs": [
@@ -308,7 +371,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 21,
    "id": "a99e7812-817d-4f9f-b6fa-1a58aa3a34dc",
    "metadata": {},
    "outputs": [
@@ -322,10 +385,12 @@
       "  Only OptionalTask_1 done: 22501\n",
       "  Only OptionalTask_2 done: 20014\n",
       "  Both done: 24854\n",
+      "  None done: 38\n",
       "Ideal Task = OptionalTask_2:\n",
       "  Only OptionalTask_1 done: 12588\n",
       "  Only OptionalTask_2 done: 18942\n",
       "  Both done: 15147\n",
+      "  None done: 78\n",
       "\n"
      ]
     }
@@ -377,8 +442,8 @@
     "\n",
     "# Initialize counters\n",
     "task_counts = {\n",
-    "    1: {\"only_opt1\": 0, \"only_opt2\": 0, \"both\": 0},\n",
-    "    2: {\"only_opt1\": 0, \"only_opt2\": 0, \"both\": 0}\n",
+    "    1: {\"only_opt1\": 0, \"only_opt2\": 0, \"both\": 0,\"none\":0},\n",
+    "    2: {\"only_opt1\": 0, \"only_opt2\": 0, \"both\": 0,\"none\":0}\n",
     "}\n",
     "\n",
     "# Analyze rows\n",
@@ -397,6 +462,8 @@
     "            task_counts[1][\"only_opt2\"] += 1\n",
     "        elif opt1_done and opt2_done:\n",
     "            task_counts[1][\"both\"] += 1\n",
+    "        else:\n",
+    "            task_counts[1][\"none\"] +=1\n",
     "    elif ideal_task == 1:\n",
     "        if opt1_done and not opt2_done:\n",
     "            task_counts[2][\"only_opt1\"] += 1\n",
@@ -404,6 +471,8 @@
     "            task_counts[2][\"only_opt2\"] += 1\n",
     "        elif opt1_done and opt2_done:\n",
     "            task_counts[2][\"both\"] += 1\n",
+    "        else:\n",
+    "            task_counts[2][\"none\"] +=1\n",
     "\n",
     "# Create a string output for results\n",
     "output_summary = \"Task Analysis Summary:\\n\"\n",
@@ -414,14 +483,47 @@
     "    output_summary += f\"  Only OptionalTask_1 done: {counts['only_opt1']}\\n\"\n",
     "    output_summary += f\"  Only OptionalTask_2 done: {counts['only_opt2']}\\n\"\n",
     "    output_summary += f\"  Both done: {counts['both']}\\n\"\n",
+    "    output_summary += f\"  None done: {counts['none']}\\n\"\n",
     "\n",
     "print(output_summary)\n"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "3630406c-859a-43ab-a569-67d577cc9bf6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import gradio as gr\n",
+    "from matplotlib.figure import Figure"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "99833638-882d-4c75-bcc3-031e39cfb5a7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"roc_data.pkl\", \"rb\") as f:\n",
+    "        fpr, tpr, _ = pickle.load(f)\n",
+    "roc_auc = auc(fpr, tpr)\n",
+    "\n",
+    "# Create a matplotlib figure\n",
+    "fig = Figure()\n",
+    "ax = fig.add_subplot(1, 1, 1)\n",
+    "ax.plot(fpr, tpr, color='blue', lw=2, label=f'ROC curve (area = {roc_auc:.2f})')\n",
+    "ax.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n",
+    "ax.set(xlabel='False Positive Rate', ylabel='True Positive Rate', title=f'Receiver Operating Curve (ROC)')\n",
+    "ax.legend(loc=\"lower right\")\n",
+    "ax.grid()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "65ad9383-741f-44eb-8e8f-853ee7bc52a2",
+   "id": "6eb3dece-5b33-4223-af9a-6b999bb2305b",
    "metadata": {},
    "outputs": [],
    "source": []

result.txt

@@ -3,5 +3,5 @@ total_acc: 69.00702106318957
 precisions: 0.7236623191454734
 recalls: 0.6900702106318957
 f1_scores: 0.6802420656474512
-time_taken_from_start: 21.604072332382202
+time_taken_from_start: 25.420082330703735
 auc_score: 0.7457100293916334