Hugging Face's logo

Spaces:
RVikas
/
MTBenchExplorer
Sleeping

App Files Community
RVikas commited on
Commit
94a9a1d
·
verified ·
1 Parent(s): 598e822

Delete data

Browse files
Files changed (2) hide show
  1. data/mt_bench_radar.ipynb +0 -714
  2. data/new_output_data.csv +0 -0
data/mt_bench_radar.ipynb DELETED
@@ -1,714 +0,0 @@
1
- {
2
- "nbformat": 4,
3
- "nbformat_minor": 0,
4
- "metadata": {
5
- "colab": {
6
- "provenance": []
7
- },
8
- "kernelspec": {
9
- "name": "python3",
10
- "display_name": "Python 3"
11
- },
12
- "language_info": {
13
- "name": "python"
14
- }
15
- },
16
- "cells": [
17
- {
18
- "cell_type": "code",
19
- "execution_count": null,
20
- "metadata": {
21
- "colab": {
22
- "base_uri": "https://localhost:8080/"
23
- },
24
- "id": "Vp5YVvaTpIiX",
25
- "outputId": "3c5b2a63-1fb4-430d-a986-4092ee8d4891"
26
- },
27
- "outputs": [
28
- {
29
- "output_type": "stream",
30
- "name": "stdout",
31
- "text": [
32
- "--2023-11-28 19:52:23-- https://huggingface.co/spaces/lmsys/mt-bench/resolve/main/data/mt_bench/model_judgment/gpt-4_single.jsonl\n",
33
- "Resolving huggingface.co (huggingface.co)... 18.164.174.55, 18.164.174.23, 18.164.174.118, ...\n",
34
- "Connecting to huggingface.co (huggingface.co)|18.164.174.55|:443... connected.\n",
35
- "HTTP request sent, awaiting response... 302 Found\n",
36
- "Location: https://cdn-lfs.huggingface.co/repos/12/2b/122bd8e9eccbb3acc98acf73e0ecef3c96f24dcdb5f6639074ed304eb19f9cd4/76c55033c6b2b1cc3f62513458f84748a23352495fd42b1062a7401de5ff9bd9?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27gpt-4_single.jsonl%3B+filename%3D%22gpt-4_single.jsonl%22%3B&Expires=1701460343&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMTQ2MDM0M319LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8xMi8yYi8xMjJiZDhlOWVjY2JiM2FjYzk4YWNmNzNlMGVjZWYzYzk2ZjI0ZGNkYjVmNjYzOTA3NGVkMzA0ZWIxOWY5Y2Q0Lzc2YzU1MDMzYzZiMmIxY2MzZjYyNTEzNDU4Zjg0NzQ4YTIzMzUyNDk1ZmQ0MmIxMDYyYTc0MDFkZTVmZjliZDk%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=hwTqBVlLz755xHaQaN6cSDP2FxoPBAXFcOE2uvFAYzg0Y90kGkY3A74Fj2wAkToA-dN1WJeMc%7Ef2XarD%7EbAw%7E4v2JCw9kphUxL-pcRF1uNBI2pzS-3Joff-m%7Ee3GVq5%7E8QabDfK60nWuA10CodvlaRDqVpuYEAvF2n5tY3Adf6-V-YdcaxE2DTlHXm65oJsJwWJTGiQYzTtn4rEVWKgQHVYp7CqX0IdyaILr966agOZvdUGDUZfkZtG6E9A6zKOgOBfdpJn1tjmMKEkDscDvLJvg8r9QJY7yttPHOMNVruzVtoLjpg1lFb-tXco3h%7EFZVKiOIZL%7E597WbaDu8hdZOQ__&Key-Pair-Id=KVTP0A1DKRTAX [following]\n",
37
- "--2023-11-28 19:52:23-- https://cdn-lfs.huggingface.co/repos/12/2b/122bd8e9eccbb3acc98acf73e0ecef3c96f24dcdb5f6639074ed304eb19f9cd4/76c55033c6b2b1cc3f62513458f84748a23352495fd42b1062a7401de5ff9bd9?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27gpt-4_single.jsonl%3B+filename%3D%22gpt-4_single.jsonl%22%3B&Expires=1701460343&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMTQ2MDM0M319LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8xMi8yYi8xMjJiZDhlOWVjY2JiM2FjYzk4YWNmNzNlMGVjZWYzYzk2ZjI0ZGNkYjVmNjYzOTA3NGVkMzA0ZWIxOWY5Y2Q0Lzc2YzU1MDMzYzZiMmIxY2MzZjYyNTEzNDU4Zjg0NzQ4YTIzMzUyNDk1ZmQ0MmIxMDYyYTc0MDFkZTVmZjliZDk%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=hwTqBVlLz755xHaQaN6cSDP2FxoPBAXFcOE2uvFAYzg0Y90kGkY3A74Fj2wAkToA-dN1WJeMc%7Ef2XarD%7EbAw%7E4v2JCw9kphUxL-pcRF1uNBI2pzS-3Joff-m%7Ee3GVq5%7E8QabDfK60nWuA10CodvlaRDqVpuYEAvF2n5tY3Adf6-V-YdcaxE2DTlHXm65oJsJwWJTGiQYzTtn4rEVWKgQHVYp7CqX0IdyaILr966agOZvdUGDUZfkZtG6E9A6zKOgOBfdpJn1tjmMKEkDscDvLJvg8r9QJY7yttPHOMNVruzVtoLjpg1lFb-tXco3h%7EFZVKiOIZL%7E597WbaDu8hdZOQ__&Key-Pair-Id=KVTP0A1DKRTAX\n",
38
- "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 18.65.25.40, 18.65.25.122, 18.65.25.124, ...\n",
39
- "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|18.65.25.40|:443... connected.\n",
40
- "HTTP request sent, awaiting response... 200 OK\n",
41
- "Length: 20113128 (19M) [text/plain]\n",
42
- "Saving to: ‘gpt-4_single.jsonl’\n",
43
- "\n",
44
- "gpt-4_single.jsonl 100%[===================>] 19.18M 25.8MB/s in 0.7s \n",
45
- "\n",
46
- "2023-11-28 19:52:25 (25.8 MB/s) - ‘gpt-4_single.jsonl’ saved [20113128/20113128]\n",
47
- "\n",
48
- "--2023-11-28 19:52:25-- https://huggingface.co/spaces/lmsys/mt-bench/resolve/main/data/mt_bench/model_judgment/gpt-4_pair.jsonl\n",
49
- "Resolving huggingface.co (huggingface.co)... 18.164.174.55, 18.164.174.23, 18.164.174.118, ...\n",
50
- "Connecting to huggingface.co (huggingface.co)|18.164.174.55|:443... connected.\n",
51
- "HTTP request sent, awaiting response... 302 Found\n",
52
- "Location: https://cdn-lfs.huggingface.co/repos/12/2b/122bd8e9eccbb3acc98acf73e0ecef3c96f24dcdb5f6639074ed304eb19f9cd4/d662c0b7d1d297f0494fcb4cc09fe8f054fa22d75deb4754a483a921984bc585?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27gpt-4_pair.jsonl%3B+filename%3D%22gpt-4_pair.jsonl%22%3B&Expires=1701460345&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMTQ2MDM0NX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8xMi8yYi8xMjJiZDhlOWVjY2JiM2FjYzk4YWNmNzNlMGVjZWYzYzk2ZjI0ZGNkYjVmNjYzOTA3NGVkMzA0ZWIxOWY5Y2Q0L2Q2NjJjMGI3ZDFkMjk3ZjA0OTRmY2I0Y2MwOWZlOGYwNTRmYTIyZDc1ZGViNDc1NGE0ODNhOTIxOTg0YmM1ODU%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=RcHQsWboSyCegZB6o-k6-9fsGpTmhArmdubGyrc7VTT2cc9FKMoPc4vHW0RtMgS%7EkYWm2eA9sfex%7EWN%7E5A0i1CBBWP3EDq365Jt52BdOw4BbOtezicyT2eLPzNkgrw3RuLMZTApHUr6md1TVm0W15rmSaUpoQT5sKcVwq%7EvmmLXr6AFOV6vWho6vEHSadzT8GJkK%7El9xOtBGhCE-pWOsEU6siX9sw0HwZBmg1mcXJzMj2du%7Em5AmG3lXsJm2fFY0ZmhSZjm7FH%7EBxF38wTuuf3gBUeJUU%7Ecx0Lv935FSAmmdzqrXO4CiGq%7EQSTp7uga8mUJikosX6DlfLMZudAIVzg__&Key-Pair-Id=KVTP0A1DKRTAX [following]\n",
53
- "--2023-11-28 19:52:25-- https://cdn-lfs.huggingface.co/repos/12/2b/122bd8e9eccbb3acc98acf73e0ecef3c96f24dcdb5f6639074ed304eb19f9cd4/d662c0b7d1d297f0494fcb4cc09fe8f054fa22d75deb4754a483a921984bc585?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27gpt-4_pair.jsonl%3B+filename%3D%22gpt-4_pair.jsonl%22%3B&Expires=1701460345&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMTQ2MDM0NX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8xMi8yYi8xMjJiZDhlOWVjY2JiM2FjYzk4YWNmNzNlMGVjZWYzYzk2ZjI0ZGNkYjVmNjYzOTA3NGVkMzA0ZWIxOWY5Y2Q0L2Q2NjJjMGI3ZDFkMjk3ZjA0OTRmY2I0Y2MwOWZlOGYwNTRmYTIyZDc1ZGViNDc1NGE0ODNhOTIxOTg0YmM1ODU%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=RcHQsWboSyCegZB6o-k6-9fsGpTmhArmdubGyrc7VTT2cc9FKMoPc4vHW0RtMgS%7EkYWm2eA9sfex%7EWN%7E5A0i1CBBWP3EDq365Jt52BdOw4BbOtezicyT2eLPzNkgrw3RuLMZTApHUr6md1TVm0W15rmSaUpoQT5sKcVwq%7EvmmLXr6AFOV6vWho6vEHSadzT8GJkK%7El9xOtBGhCE-pWOsEU6siX9sw0HwZBmg1mcXJzMj2du%7Em5AmG3lXsJm2fFY0ZmhSZjm7FH%7EBxF38wTuuf3gBUeJUU%7Ecx0Lv935FSAmmdzqrXO4CiGq%7EQSTp7uga8mUJikosX6DlfLMZudAIVzg__&Key-Pair-Id=KVTP0A1DKRTAX\n",
54
- "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 18.65.25.40, 18.65.25.122, 18.65.25.124, ...\n",
55
- "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|18.65.25.40|:443... connected.\n",
56
- "HTTP request sent, awaiting response... 200 OK\n",
57
- "Length: 48043462 (46M) [binary/octet-stream]\n",
58
- "Saving to: ‘gpt-4_pair.jsonl’\n",
59
- "\n",
60
- "gpt-4_pair.jsonl 100%[===================>] 45.82M 36.0MB/s in 1.3s \n",
61
- "\n",
62
- "2023-11-28 19:52:27 (36.0 MB/s) - ‘gpt-4_pair.jsonl’ saved [48043462/48043462]\n",
63
- "\n"
64
- ]
65
- }
66
- ],
67
- "source": [
68
- "!wget https://huggingface.co/spaces/lmsys/mt-bench/resolve/main/data/mt_bench/model_judgment/gpt-4_single.jsonl\n",
69
- "!wget https://huggingface.co/spaces/lmsys/mt-bench/resolve/main/data/mt_bench/model_judgment/gpt-4_pair.jsonl"
70
- ]
71
- },
72
- {
73
- "cell_type": "code",
74
- "source": [
75
- "!pip install -U plotly kaleido"
76
- ],
77
- "metadata": {
78
- "colab": {
79
- "base_uri": "https://localhost:8080/"
80
- },
81
- "id": "4eYlKr9RrPu2",
82
- "outputId": "b957d1f9-0024-4c5c-eb07-dcb1a0071081"
83
- },
84
- "execution_count": null,
85
- "outputs": [
86
- {
87
- "output_type": "stream",
88
- "name": "stdout",
89
- "text": [
90
- "Requirement already satisfied: plotly in /usr/local/lib/python3.10/dist-packages (5.15.0)\n",
91
- "Collecting plotly\n",
92
- " Downloading plotly-5.18.0-py3-none-any.whl (15.6 MB)\n",
93
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.6/15.6 MB\u001b[0m \u001b[31m27.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
94
- "\u001b[?25hCollecting kaleido\n",
95
- " Downloading kaleido-0.2.1-py2.py3-none-manylinux1_x86_64.whl (79.9 MB)\n",
96
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.9/79.9 MB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
97
- "\u001b[?25hRequirement already satisfied: tenacity>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from plotly) (8.2.3)\n",
98
- "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from plotly) (23.2)\n",
99
- "Installing collected packages: kaleido, plotly\n",
100
- " Attempting uninstall: plotly\n",
101
- " Found existing installation: plotly 5.15.0\n",
102
- " Uninstalling plotly-5.15.0:\n",
103
- " Successfully uninstalled plotly-5.15.0\n",
104
- "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
105
- "lida 0.0.10 requires fastapi, which is not installed.\n",
106
- "lida 0.0.10 requires python-multipart, which is not installed.\n",
107
- "lida 0.0.10 requires uvicorn, which is not installed.\u001b[0m\u001b[31m\n",
108
- "\u001b[0mSuccessfully installed kaleido-0.2.1 plotly-5.18.0\n"
109
- ]
110
- }
111
- ]
112
- },
113
- {
114
- "cell_type": "code",
115
- "source": [
116
- "import json\n",
117
- "import pandas as pd\n",
118
- "import plotly.express as px\n",
119
- "import plotly.graph_objects as go\n",
120
- "\n",
121
- "\n",
122
- "CATEGORIES = [\"Writing\", \"Roleplay\", \"Reasoning\", \"Math\", \"Coding\", \"Extraction\", \"STEM\", \"Humanities\"]\n",
123
- "\n",
124
- "\n",
125
- "def get_model_df():\n",
126
- " cnt = 0\n",
127
- " q2result = []\n",
128
- " fin = open(\"gpt-4_single.jsonl\", \"r\")\n",
129
- " for line in fin:\n",
130
- " obj = json.loads(line)\n",
131
- " obj[\"category\"] = CATEGORIES[(obj[\"question_id\"]-81)//10]\n",
132
- " q2result.append(obj)\n",
133
- " df = pd.DataFrame(q2result)\n",
134
- " return df\n",
135
- "\n",
136
- "def toggle(res_str):\n",
137
- " if res_str == \"win\":\n",
138
- " return \"loss\"\n",
139
- " elif res_str == \"loss\":\n",
140
- " return \"win\"\n",
141
- " return \"tie\"\n",
142
- "\n",
143
- "def get_model_df_pair():\n",
144
- " fin = open(\"gpt-4_pair.jsonl\", \"r\")\n",
145
- " cnt = 0\n",
146
- " q2result = []\n",
147
- " for line in fin:\n",
148
- " obj = json.loads(line)\n",
149
- "\n",
150
- " result = {}\n",
151
- " result[\"qid\"] = str(obj[\"question_id\"])\n",
152
- " result[\"turn\"] = str(obj[\"turn\"])\n",
153
- " if obj[\"g1_winner\"] == \"model_1\" and obj[\"g2_winner\"] == \"model_1\":\n",
154
- " result[\"result\"] = \"win\"\n",
155
- " elif obj[\"g1_winner\"] == \"model_2\" and obj[\"g2_winner\"] == \"model_2\":\n",
156
- " result[\"result\"] = \"loss\"\n",
157
- " else:\n",
158
- " result[\"result\"] = \"tie\"\n",
159
- " result[\"category\"] = CATEGORIES[(obj[\"question_id\"]-81)//10]\n",
160
- " result[\"model\"] = obj[\"model_1\"]\n",
161
- " q2result.append(result)\n",
162
- "\n",
163
- " df = pd.DataFrame(q2result)\n",
164
- "\n",
165
- " return df\n",
166
- "\n",
167
- "df = get_model_df()\n",
168
- "df_pair = get_model_df_pair()"
169
- ],
170
- "metadata": {
171
- "id": "m2tG_vDyqWZw"
172
- },
173
- "execution_count": null,
174
- "outputs": []
175
- },
176
- {
177
- "cell_type": "code",
178
- "source": [
179
- "df_pair"
180
- ],
181
- "metadata": {
182
- "colab": {
183
- "base_uri": "https://localhost:8080/",
184
- "height": 423
185
- },
186
- "id": "wUw1sxfmaGuK",
187
- "outputId": "21365f64-c2fa-47c7-9ad4-ca114eac6533"
188
- },
189
- "execution_count": null,
190
- "outputs": [
191
- {
192
- "output_type": "execute_result",
193
- "data": {
194
- "text/plain": [
195
- " qid turn result category model\n",
196
- "0 81 1 loss Writing alpaca-13b\n",
197
- "1 81 2 loss Writing alpaca-13b\n",
198
- "2 82 1 loss Writing alpaca-13b\n",
199
- "3 82 2 loss Writing alpaca-13b\n",
200
- "4 83 1 loss Writing alpaca-13b\n",
201
- "... ... ... ... ... ...\n",
202
- "4795 158 2 tie Humanities wizardlm-30b\n",
203
- "4796 159 1 loss Humanities wizardlm-30b\n",
204
- "4797 159 2 win Humanities wizardlm-30b\n",
205
- "4798 160 1 loss Humanities wizardlm-30b\n",
206
- "4799 160 2 tie Humanities wizardlm-30b\n",
207
- "\n",
208
- "[4800 rows x 5 columns]"
209
- ],
210
- "text/html": [
211
- "\n",
212
- " <div id=\"df-28f6ebcf-4509-4c45-8fe3-2d64ca044a2f\" class=\"colab-df-container\">\n",
213
- " <div>\n",
214
- "<style scoped>\n",
215
- " .dataframe tbody tr th:only-of-type {\n",
216
- " vertical-align: middle;\n",
217
- " }\n",
218
- "\n",
219
- " .dataframe tbody tr th {\n",
220
- " vertical-align: top;\n",
221
- " }\n",
222
- "\n",
223
- " .dataframe thead th {\n",
224
- " text-align: right;\n",
225
- " }\n",
226
- "</style>\n",
227
- "<table border=\"1\" class=\"dataframe\">\n",
228
- " <thead>\n",
229
- " <tr style=\"text-align: right;\">\n",
230
- " <th></th>\n",
231
- " <th>qid</th>\n",
232
- " <th>turn</th>\n",
233
- " <th>result</th>\n",
234
- " <th>category</th>\n",
235
- " <th>model</th>\n",
236
- " </tr>\n",
237
- " </thead>\n",
238
- " <tbody>\n",
239
- " <tr>\n",
240
- " <th>0</th>\n",
241
- " <td>81</td>\n",
242
- " <td>1</td>\n",
243
- " <td>loss</td>\n",
244
- " <td>Writing</td>\n",
245
- " <td>alpaca-13b</td>\n",
246
- " </tr>\n",
247
- " <tr>\n",
248
- " <th>1</th>\n",
249
- " <td>81</td>\n",
250
- " <td>2</td>\n",
251
- " <td>loss</td>\n",
252
- " <td>Writing</td>\n",
253
- " <td>alpaca-13b</td>\n",
254
- " </tr>\n",
255
- " <tr>\n",
256
- " <th>2</th>\n",
257
- " <td>82</td>\n",
258
- " <td>1</td>\n",
259
- " <td>loss</td>\n",
260
- " <td>Writing</td>\n",
261
- " <td>alpaca-13b</td>\n",
262
- " </tr>\n",
263
- " <tr>\n",
264
- " <th>3</th>\n",
265
- " <td>82</td>\n",
266
- " <td>2</td>\n",
267
- " <td>loss</td>\n",
268
- " <td>Writing</td>\n",
269
- " <td>alpaca-13b</td>\n",
270
- " </tr>\n",
271
- " <tr>\n",
272
- " <th>4</th>\n",
273
- " <td>83</td>\n",
274
- " <td>1</td>\n",
275
- " <td>loss</td>\n",
276
- " <td>Writing</td>\n",
277
- " <td>alpaca-13b</td>\n",
278
- " </tr>\n",
279
- " <tr>\n",
280
- " <th>...</th>\n",
281
- " <td>...</td>\n",
282
- " <td>...</td>\n",
283
- " <td>...</td>\n",
284
- " <td>...</td>\n",
285
- " <td>...</td>\n",
286
- " </tr>\n",
287
- " <tr>\n",
288
- " <th>4795</th>\n",
289
- " <td>158</td>\n",
290
- " <td>2</td>\n",
291
- " <td>tie</td>\n",
292
- " <td>Humanities</td>\n",
293
- " <td>wizardlm-30b</td>\n",
294
- " </tr>\n",
295
- " <tr>\n",
296
- " <th>4796</th>\n",
297
- " <td>159</td>\n",
298
- " <td>1</td>\n",
299
- " <td>loss</td>\n",
300
- " <td>Humanities</td>\n",
301
- " <td>wizardlm-30b</td>\n",
302
- " </tr>\n",
303
- " <tr>\n",
304
- " <th>4797</th>\n",
305
- " <td>159</td>\n",
306
- " <td>2</td>\n",
307
- " <td>win</td>\n",
308
- " <td>Humanities</td>\n",
309
- " <td>wizardlm-30b</td>\n",
310
- " </tr>\n",
311
- " <tr>\n",
312
- " <th>4798</th>\n",
313
- " <td>160</td>\n",
314
- " <td>1</td>\n",
315
- " <td>loss</td>\n",
316
- " <td>Humanities</td>\n",
317
- " <td>wizardlm-30b</td>\n",
318
- " </tr>\n",
319
- " <tr>\n",
320
- " <th>4799</th>\n",
321
- " <td>160</td>\n",
322
- " <td>2</td>\n",
323
- " <td>tie</td>\n",
324
- " <td>Humanities</td>\n",
325
- " <td>wizardlm-30b</td>\n",
326
- " </tr>\n",
327
- " </tbody>\n",
328
- "</table>\n",
329
- "<p>4800 rows × 5 columns</p>\n",
330
- "</div>\n",
331
- " <div class=\"colab-df-buttons\">\n",
332
- "\n",
333
- " <div class=\"colab-df-container\">\n",
334
- " <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-28f6ebcf-4509-4c45-8fe3-2d64ca044a2f')\"\n",
335
- " title=\"Convert this dataframe to an interactive table.\"\n",
336
- " style=\"display:none;\">\n",
337
- "\n",
338
- " <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
339
- " <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
340
- " </svg>\n",
341
- " </button>\n",
342
- "\n",
343
- " <style>\n",
344
- " .colab-df-container {\n",
345
- " display:flex;\n",
346
- " gap: 12px;\n",
347
- " }\n",
348
- "\n",
349
- " .colab-df-convert {\n",
350
- " background-color: #E8F0FE;\n",
351
- " border: none;\n",
352
- " border-radius: 50%;\n",
353
- " cursor: pointer;\n",
354
- " display: none;\n",
355
- " fill: #1967D2;\n",
356
- " height: 32px;\n",
357
- " padding: 0 0 0 0;\n",
358
- " width: 32px;\n",
359
- " }\n",
360
- "\n",
361
- " .colab-df-convert:hover {\n",
362
- " background-color: #E2EBFA;\n",
363
- " box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
364
- " fill: #174EA6;\n",
365
- " }\n",
366
- "\n",
367
- " .colab-df-buttons div {\n",
368
- " margin-bottom: 4px;\n",
369
- " }\n",
370
- "\n",
371
- " [theme=dark] .colab-df-convert {\n",
372
- " background-color: #3B4455;\n",
373
- " fill: #D2E3FC;\n",
374
- " }\n",
375
- "\n",
376
- " [theme=dark] .colab-df-convert:hover {\n",
377
- " background-color: #434B5C;\n",
378
- " box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
379
- " filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
380
- " fill: #FFFFFF;\n",
381
- " }\n",
382
- " </style>\n",
383
- "\n",
384
- " <script>\n",
385
- " const buttonEl =\n",
386
- " document.querySelector('#df-28f6ebcf-4509-4c45-8fe3-2d64ca044a2f button.colab-df-convert');\n",
387
- " buttonEl.style.display =\n",
388
- " google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
389
- "\n",
390
- " async function convertToInteractive(key) {\n",
391
- " const element = document.querySelector('#df-28f6ebcf-4509-4c45-8fe3-2d64ca044a2f');\n",
392
- " const dataTable =\n",
393
- " await google.colab.kernel.invokeFunction('convertToInteractive',\n",
394
- " [key], {});\n",
395
- " if (!dataTable) return;\n",
396
- "\n",
397
- " const docLinkHtml = 'Like what you see? Visit the ' +\n",
398
- " '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
399
- " + ' to learn more about interactive tables.';\n",
400
- " element.innerHTML = '';\n",
401
- " dataTable['output_type'] = 'display_data';\n",
402
- " await google.colab.output.renderOutput(dataTable, element);\n",
403
- " const docLink = document.createElement('div');\n",
404
- " docLink.innerHTML = docLinkHtml;\n",
405
- " element.appendChild(docLink);\n",
406
- " }\n",
407
- " </script>\n",
408
- " </div>\n",
409
- "\n",
410
- "\n",
411
- "<div id=\"df-02d34fc9-f4ca-40d7-b50a-bd1ffede5665\">\n",
412
- " <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-02d34fc9-f4ca-40d7-b50a-bd1ffede5665')\"\n",
413
- " title=\"Suggest charts\"\n",
414
- " style=\"display:none;\">\n",
415
- "\n",
416
- "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
417
- " width=\"24px\">\n",
418
- " <g>\n",
419
- " <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
420
- " </g>\n",
421
- "</svg>\n",
422
- " </button>\n",
423
- "\n",
424
- "<style>\n",
425
- " .colab-df-quickchart {\n",
426
- " --bg-color: #E8F0FE;\n",
427
- " --fill-color: #1967D2;\n",
428
- " --hover-bg-color: #E2EBFA;\n",
429
- " --hover-fill-color: #174EA6;\n",
430
- " --disabled-fill-color: #AAA;\n",
431
- " --disabled-bg-color: #DDD;\n",
432
- " }\n",
433
- "\n",
434
- " [theme=dark] .colab-df-quickchart {\n",
435
- " --bg-color: #3B4455;\n",
436
- " --fill-color: #D2E3FC;\n",
437
- " --hover-bg-color: #434B5C;\n",
438
- " --hover-fill-color: #FFFFFF;\n",
439
- " --disabled-bg-color: #3B4455;\n",
440
- " --disabled-fill-color: #666;\n",
441
- " }\n",
442
- "\n",
443
- " .colab-df-quickchart {\n",
444
- " background-color: var(--bg-color);\n",
445
- " border: none;\n",
446
- " border-radius: 50%;\n",
447
- " cursor: pointer;\n",
448
- " display: none;\n",
449
- " fill: var(--fill-color);\n",
450
- " height: 32px;\n",
451
- " padding: 0;\n",
452
- " width: 32px;\n",
453
- " }\n",
454
- "\n",
455
- " .colab-df-quickchart:hover {\n",
456
- " background-color: var(--hover-bg-color);\n",
457
- " box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
458
- " fill: var(--button-hover-fill-color);\n",
459
- " }\n",
460
- "\n",
461
- " .colab-df-quickchart-complete:disabled,\n",
462
- " .colab-df-quickchart-complete:disabled:hover {\n",
463
- " background-color: var(--disabled-bg-color);\n",
464
- " fill: var(--disabled-fill-color);\n",
465
- " box-shadow: none;\n",
466
- " }\n",
467
- "\n",
468
- " .colab-df-spinner {\n",
469
- " border: 2px solid var(--fill-color);\n",
470
- " border-color: transparent;\n",
471
- " border-bottom-color: var(--fill-color);\n",
472
- " animation:\n",
473
- " spin 1s steps(1) infinite;\n",
474
- " }\n",
475
- "\n",
476
- " @keyframes spin {\n",
477
- " 0% {\n",
478
- " border-color: transparent;\n",
479
- " border-bottom-color: var(--fill-color);\n",
480
- " border-left-color: var(--fill-color);\n",
481
- " }\n",
482
- " 20% {\n",
483
- " border-color: transparent;\n",
484
- " border-left-color: var(--fill-color);\n",
485
- " border-top-color: var(--fill-color);\n",
486
- " }\n",
487
- " 30% {\n",
488
- " border-color: transparent;\n",
489
- " border-left-color: var(--fill-color);\n",
490
- " border-top-color: var(--fill-color);\n",
491
- " border-right-color: var(--fill-color);\n",
492
- " }\n",
493
- " 40% {\n",
494
- " border-color: transparent;\n",
495
- " border-right-color: var(--fill-color);\n",
496
- " border-top-color: var(--fill-color);\n",
497
- " }\n",
498
- " 60% {\n",
499
- " border-color: transparent;\n",
500
- " border-right-color: var(--fill-color);\n",
501
- " }\n",
502
- " 80% {\n",
503
- " border-color: transparent;\n",
504
- " border-right-color: var(--fill-color);\n",
505
- " border-bottom-color: var(--fill-color);\n",
506
- " }\n",
507
- " 90% {\n",
508
- " border-color: transparent;\n",
509
- " border-bottom-color: var(--fill-color);\n",
510
- " }\n",
511
- " }\n",
512
- "</style>\n",
513
- "\n",
514
- " <script>\n",
515
- " async function quickchart(key) {\n",
516
- " const quickchartButtonEl =\n",
517
- " document.querySelector('#' + key + ' button');\n",
518
- " quickchartButtonEl.disabled = true; // To prevent multiple clicks.\n",
519
- " quickchartButtonEl.classList.add('colab-df-spinner');\n",
520
- " try {\n",
521
- " const charts = await google.colab.kernel.invokeFunction(\n",
522
- " 'suggestCharts', [key], {});\n",
523
- " } catch (error) {\n",
524
- " console.error('Error during call to suggestCharts:', error);\n",
525
- " }\n",
526
- " quickchartButtonEl.classList.remove('colab-df-spinner');\n",
527
- " quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
528
- " }\n",
529
- " (() => {\n",
530
- " let quickchartButtonEl =\n",
531
- " document.querySelector('#df-02d34fc9-f4ca-40d7-b50a-bd1ffede5665 button');\n",
532
- " quickchartButtonEl.style.display =\n",
533
- " google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
534
- " })();\n",
535
- " </script>\n",
536
- "</div>\n",
537
- " </div>\n",
538
- " </div>\n"
539
- ]
540
- },
541
- "metadata": {},
542
- "execution_count": 4
543
- }
544
- ]
545
- },
546
- {
547
- "cell_type": "code",
548
- "source": [
549
- "all_models = df[\"model\"].unique()\n",
550
- "print(all_models)\n",
551
- "scores_all = []\n",
552
- "for model in all_models:\n",
553
- " for cat in CATEGORIES:\n",
554
- " # filter category/model, and score format error (<1% case)\n",
555
- " res = df[(df[\"category\"]==cat) & (df[\"model\"]==model) & (df[\"score\"] >= 0)]\n",
556
- " score = res[\"score\"].mean()\n",
557
- "\n",
558
- " # # pairwise result\n",
559
- " # res_pair = df_pair[(df_pair[\"category\"]==cat) & (df_pair[\"model\"]==model)][\"result\"].value_counts()\n",
560
- " # wincnt = res_pair[\"win\"] if \"win\" in res_pair.index else 0\n",
561
- " # tiecnt = res_pair[\"tie\"] if \"tie\" in res_pair.index else 0\n",
562
- " # winrate = wincnt/res_pair.sum()\n",
563
- " # winrate_adjusted = (wincnt + tiecnt)/res_pair.sum()\n",
564
- " # # print(winrate_adjusted)\n",
565
- "\n",
566
- " # scores_all.append({\"model\": model, \"category\": cat, \"score\": score, \"winrate\": winrate, \"wtrate\": winrate_adjusted})\n",
567
- " scores_all.append({\"model\": model, \"category\": cat, \"score\": score})"
568
- ],
569
- "metadata": {
570
- "colab": {
571
- "base_uri": "https://localhost:8080/"
572
- },
573
- "id": "MpBKLuVmqZ7O",
574
- "outputId": "f7ea476f-dde8-4b7c-fb69-5d7d33999caf"
575
- },
576
- "execution_count": null,
577
- "outputs": [
578
- {
579
- "output_type": "stream",
580
- "name": "stdout",
581
- "text": [
582
- "['alpaca-13b' 'baize-v2-13b' 'chatglm-6b' 'claude-instant-v1' 'claude-v1'\n",
583
- " 'dolly-v2-12b' 'falcon-40b-instruct' 'fastchat-t5-3b' 'gpt-3.5-turbo'\n",
584
- " 'gpt-4' 'gpt4all-13b-snoozy' 'guanaco-33b' 'guanaco-65b'\n",
585
- " 'h2ogpt-oasst-open-llama-13b' 'koala-13b' 'llama-13b' 'mpt-30b-chat'\n",
586
- " 'mpt-30b-instruct' 'mpt-7b-chat' 'nous-hermes-13b'\n",
587
- " 'oasst-sft-4-pythia-12b' 'oasst-sft-7-llama-30b' 'palm-2-chat-bison-001'\n",
588
- " 'rwkv-4-raven-14b' 'stablelm-tuned-alpha-7b' 'tulu-30b' 'vicuna-13b-v1.3'\n",
589
- " 'vicuna-33b-v1.3' 'vicuna-7b-v1.3' 'wizardlm-13b' 'wizardlm-30b'\n",
590
- " 'Llama-2-7b-chat' 'Llama-2-13b-chat' 'Llama-2-70b-chat']\n"
591
- ]
592
- }
593
- ]
594
- },
595
- {
596
- "cell_type": "code",
597
- "source": [
598
- "target_models = [\"Llama-2-7b-chat\", \"Llama-2-13b-chat\", \"Llama-2-70b-chat\", \"gpt-3.5-turbo\", \"claude-v1\", \"gpt-4\"]\n",
599
- "\n",
600
- "scores_target = [scores_all[i] for i in range(len(scores_all)) if scores_all[i][\"model\"] in target_models]\n",
601
- "\n",
602
- "# sort by target_models\n",
603
- "scores_target = sorted(scores_target, key=lambda x: target_models.index(x[\"model\"]), reverse=True)\n",
604
- "\n",
605
- "df_score = pd.DataFrame(scores_target)\n",
606
- "df_score = df_score[df_score[\"model\"].isin(target_models)]\n",
607
- "\n",
608
- "rename_map = {\"llama-13b\": \"LLaMA-13B\",\n",
609
- " \"alpaca-13b\": \"Alpaca-13B\",\n",
610
- " \"vicuna-33b-v1.3\": \"Vicuna-33B\",\n",
611
- " \"vicuna-13b-v1.3\": \"Vicuna-13B\",\n",
612
- " \"gpt-3.5-turbo\": \"GPT-3.5-turbo\",\n",
613
- " \"claude-v1\": \"Claude-v1\",\n",
614
- " \"gpt-4\": \"GPT-4\"}\n",
615
- "\n",
616
- "for k, v in rename_map.items():\n",
617
- " df_score.replace(k, v, inplace=True)\n",
618
- "\n",
619
- "fig = px.line_polar(df_score, r = 'score', theta = 'category', line_close = True, category_orders = {\"category\": CATEGORIES},\n",
620
- " color = 'model', markers=True, color_discrete_sequence=px.colors.qualitative.Pastel)\n",
621
- "\n",
622
- "fig.show()"
623
- ],
624
- "metadata": {
625
- "colab": {
626
- "base_uri": "https://localhost:8080/",
627
- "height": 542
628
- },
629
- "id": "5i8R0l-XqkgO",
630
- "outputId": "10151ab6-cf3d-4162-a0cf-c510f2e3968a"
631
- },
632
- "execution_count": null,
633
- "outputs": [
634
- {
635
- "output_type": "display_data",
636
- "data": {
637
- "text/html": [
638
- "<html>\n",
639
- "<head><meta charset=\"utf-8\" /></head>\n",
640
- "<body>\n",
641
- " <div> <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script> <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
642
- " <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.27.0.min.js\"></script> <div id=\"7d62e79f-72ea-4c66-9a41-d7c4c2f7d741\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div> <script type=\"text/javascript\"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById(\"7d62e79f-72ea-4c66-9a41-d7c4c2f7d741\")) { Plotly.newPlot( \"7d62e79f-72ea-4c66-9a41-d7c4c2f7d741\", [{\"hovertemplate\":\"model=GPT-4\\u003cbr\\u003escore=%{r}\\u003cbr\\u003ecategory=%{theta}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"GPT-4\",\"line\":{\"color\":\"rgb(102, 197, 204)\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"markers+lines\",\"name\":\"GPT-4\",\"r\":[9.65,8.9,9.0,6.8,8.55,9.375,9.7,9.95,9.65],\"showlegend\":true,\"subplot\":\"polar\",\"theta\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\",\"Writing\"],\"type\":\"scatterpolar\"},{\"hovertemplate\":\"model=Claude-v1\\u003cbr\\u003escore=%{r}\\u003cbr\\u003ecategory=%{theta}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"Claude-v1\",\"line\":{\"color\":\"rgb(246, 207, 113)\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"markers+lines\",\"name\":\"Claude-v1\",\"r\":[9.5,8.5,5.95,4.8,6.25,8.8,9.7,9.7,9.5],\"showlegend\":true,\"subplot\":\"polar\",\"theta\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\",\"Writing\"],\"type\":\"scatterpolar\"},{\"hovertemplate\":\"model=GPT-3.5-turbo\\u003cbr\\u003escore=%{r}\\u003cbr\\u003ecategory=%{theta}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"GPT-3.5-turbo\",\"line\":{\"color\":\"rgb(248, 156, 116)\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"markers+lines\",\"name\":\"GPT-3.5-turbo\",\"r\":[9.2,8.4,5.65,6.3,6.9,8.85,8.7,9.55,9.2],\"showlegend\":true,\"subplot\":\"polar\",\"theta\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\",\"Writing\"],\"type\":\"scatterpolar\"},{\"hovertemplate\":\"model=Llama-2-70b-chat\\u003cbr\\u003escore=%{r}\\u003cbr\\u003ecategory=%{theta}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"Llama-2-70b-chat\",\"line\":{\"color\":\"rgb(220, 176, 242)\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"markers+lines\",\"name\":\"Llama-2-70b-chat\",\"r\":[9.3,7.5,5.8,3.3,3.15,7.25,8.925,9.625,9.3],\"showlegend\":true,\"subplot\":\"polar\",\"theta\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\",\"Writing\"],\"type\":\"scatterpolar\"},{\"hovertemplate\":\"model=Llama-2-13b-chat\\u003cbr\\u003escore=%{r}\\u003cbr\\u003ecategory=%{theta}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"Llama-2-13b-chat\",\"line\":{\"color\":\"rgb(135, 197, 95)\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"markers+lines\",\"name\":\"Llama-2-13b-chat\",\"r\":[8.85,7.5,5.1,3.45,3.0,6.925,8.625,9.75,8.85],\"showlegend\":true,\"subplot\":\"polar\",\"theta\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\",\"Writing\"],\"type\":\"scatterpolar\"},{\"hovertemplate\":\"model=Llama-2-7b-chat\\u003cbr\\u003escore=%{r}\\u003cbr\\u003ecategory=%{theta}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"Llama-2-7b-chat\",\"line\":{\"color\":\"rgb(158, 185, 243)\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"markers+lines\",\"name\":\"Llama-2-7b-chat\",\"r\":[8.9,7.7,4.25,2.4,3.0,6.5,8.65,8.75,8.9],\"showlegend\":true,\"subplot\":\"polar\",\"theta\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\",\"Writing\"],\"type\":\"scatterpolar\"}], {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"polar\":{\"domain\":{\"x\":[0.0,1.0],\"y\":[0.0,1.0]},\"angularaxis\":{\"direction\":\"clockwise\",\"rotation\":90,\"categoryorder\":\"array\",\"categoryarray\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\"]}},\"legend\":{\"title\":{\"text\":\"model\"},\"tracegroupgap\":0},\"margin\":{\"t\":60}}, {\"responsive\": true} ).then(function(){\n",
643
- " \n",
644
- "var gd = document.getElementById('7d62e79f-72ea-4c66-9a41-d7c4c2f7d741');\n",
645
- "var x = new MutationObserver(function (mutations, observer) {{\n",
646
- " var display = window.getComputedStyle(gd).display;\n",
647
- " if (!display || display === 'none') {{\n",
648
- " console.log([gd, 'removed!']);\n",
649
- " Plotly.purge(gd);\n",
650
- " observer.disconnect();\n",
651
- " }}\n",
652
- "}});\n",
653
- "\n",
654
- "// Listen for the removal of the full notebook cells\n",
655
- "var notebookContainer = gd.closest('#notebook-container');\n",
656
- "if (notebookContainer) {{\n",
657
- " x.observe(notebookContainer, {childList: true});\n",
658
- "}}\n",
659
- "\n",
660
- "// Listen for the clearing of the current output cell\n",
661
- "var outputEl = gd.closest('.output');\n",
662
- "if (outputEl) {{\n",
663
- " x.observe(outputEl, {childList: true});\n",
664
- "}}\n",
665
- "\n",
666
- " }) }; </script> </div>\n",
667
- "</body>\n",
668
- "</html>"
669
- ]
670
- },
671
- "metadata": {}
672
- }
673
- ]
674
- },
675
- {
676
- "cell_type": "code",
677
- "source": [
678
- "# fig = px.line_polar(df_score, r = 'wtrate', theta = 'category', line_close = True, category_orders = {\"category\": CATEGORIES},\n",
679
- "# color = 'model', markers=True, color_discrete_sequence=px.colors.qualitative.Pastel)\n",
680
- "# fig.show()"
681
- ],
682
- "metadata": {
683
- "id": "MaBaUN4IqvJI"
684
- },
685
- "execution_count": null,
686
- "outputs": []
687
- },
688
- {
689
- "cell_type": "code",
690
- "source": [
691
- "fig.update_layout(\n",
692
- " font=dict(\n",
693
- " size=18,\n",
694
- " ),\n",
695
- ")\n",
696
- "fig.write_image(\"fig.png\", width=800, height=600, scale=2)"
697
- ],
698
- "metadata": {
699
- "id": "4l1bzYM2bgDW"
700
- },
701
- "execution_count": null,
702
- "outputs": []
703
- },
704
- {
705
- "cell_type": "code",
706
- "source": [],
707
- "metadata": {
708
- "id": "nfpERnxFANhV"
709
- },
710
- "execution_count": null,
711
- "outputs": []
712
- }
713
- ]
714
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
data/new_output_data.csv DELETED
The diff for this file is too large to render. See raw diff