yoshinakachi commited on
Commit
5b4f4bd
·
verified ·
1 Parent(s): 1e8671b

Update src/App.js

Browse files
Files changed (1) hide show
  1. src/App.js +113 -112
src/App.js CHANGED
@@ -1,93 +1,93 @@
1
  import React, { useState, useEffect } from 'react';
2
 
3
  const BenchmarkChart = () => {
4
- // Real data from your ASR calculations - sorted by highest achievable ASR
5
  const benchmarkData = [
6
  {
7
- model: "Grok 4",
8
- baseline: 68.67,
9
  methods: {
10
- keyword_objective_combined: 85.15
 
11
  }
12
  },
13
  {
14
- model: "Deepseek R1-0528",
15
- baseline: 68.67,
16
  methods: {
17
- keyword_objective_combined: 83.76
 
18
  }
19
  },
20
  {
21
- model: "Llama 3.1 405B",
22
- baseline: 67.00,
23
  methods: {
24
- keyword_objective_combined: 80.75
25
  }
26
  },
27
  {
28
- model: "Gemini 2.5 Pro",
29
- baseline: 55.67,
30
  methods: {
31
- keyword_objective_combined: 74.14,
32
- root_problem: 67.19
33
  }
34
  },
35
  {
36
- model: "Llama 3 8B Instruct Reference",
37
- baseline: 58.33,
38
  methods: {
39
- keyword_objective_combined: 68.86
40
  }
41
  },
42
  {
43
- model: "Mixtral 8x22B",
44
- baseline: 48.00,
45
  methods: {
46
- keyword_objective_combined: 66.82
47
  }
48
  },
49
  {
50
- model: "Llama 4 Maverick Instruct",
51
- baseline: 45.00,
52
  methods: {
53
- keyword_objective_combined: 56.46
54
  }
55
  },
56
  {
57
- model: "GPT o3",
58
- baseline: 22.00,
59
  methods: {
60
- keyword_objective_combined: 30.53
 
61
  }
62
  },
63
  {
64
- model: "Claude 4 Sonnet",
65
- baseline: 26.33,
66
  methods: {
67
- keyword_objective_combined: 28.64
68
  }
69
  },
70
  {
71
- model: "Claude Opus 4.1",
72
- baseline: 20.67,
73
  methods: {
74
- keyword_objective_combined: 23.56
75
  }
76
  },
77
  {
78
- model: "GPT 5",
79
- baseline: 8.33,
80
  methods: {
81
- keyword_objective_combined: 11.68,
82
- root_problem: 12.46
83
  }
84
  },
85
  {
86
- model: "GPT 5 mini",
87
- baseline: 7.67,
88
  methods: {
89
- keyword_objective_combined: 11.28,
90
- root_problem: 10.44
91
  }
92
  }
93
  ];
@@ -96,41 +96,11 @@ const BenchmarkChart = () => {
96
  const [currentMethodIndex, setCurrentMethodIndex] = useState(0);
97
 
98
  const synthesisMethodsOrder = ['keyword_objective_combined', 'root_problem'];
99
-
100
- const phases = [
101
- { key: 'baseline', label: 'Human Baseline ASR' },
102
- { key: 'additive_synthesis', label: 'Human + Transformation Methods ASR' }
103
- ];
104
-
105
- useEffect(() => {
106
- const interval = setInterval(() => {
107
- setCurrentPhase(prev => prev === 'baseline' ? 'additive_synthesis' : 'baseline');
108
- setCurrentMethodIndex(0);
109
- }, 8000);
110
-
111
- return () => clearInterval(interval);
112
- }, []);
113
-
114
- useEffect(() => {
115
- if (currentPhase === 'additive_synthesis') {
116
- const methodInterval = setInterval(() => {
117
- setCurrentMethodIndex(prev => {
118
- const nextIndex = prev + 1;
119
- if (nextIndex > synthesisMethodsOrder.length) {
120
- return synthesisMethodsOrder.length;
121
- }
122
- return nextIndex;
123
- });
124
- }, 2000);
125
- return () => clearInterval(methodInterval);
126
- }
127
- }, [currentPhase]);
128
 
129
  const getCurrentValue = (modelData, phase) => {
130
  if (phase === 'baseline') {
131
  return modelData.baseline;
132
  } else if (phase === 'additive_synthesis') {
133
- // Show the highest ASR achieved by any transformation method tried so far
134
  let maxASR = modelData.baseline;
135
 
136
  for (let i = 0; i < currentMethodIndex; i++) {
@@ -160,19 +130,25 @@ const BenchmarkChart = () => {
160
  if (availableMethods.length === 0) return 'Human Baseline';
161
 
162
  const lastMethod = availableMethods[availableMethods.length - 1];
163
- if (lastMethod === 'keyword_objective_combined') return 'Keyword/Objective Transformation';
164
- if (lastMethod === 'root_problem') return 'Root Problem Transformation';
 
 
165
 
166
- return lastMethod.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase());
167
  };
168
 
169
- const getBarColor = (modelData, phase) => {
170
- if (phase === 'baseline') {
171
- return 'from-blue-500 to-blue-600';
172
- } else if (phase === 'additive_synthesis' && currentMethodIndex > 0) {
173
- return 'from-green-500 to-green-600';
 
 
 
174
  } else {
175
- return 'from-blue-500 to-blue-600';
 
176
  }
177
  };
178
 
@@ -182,43 +158,40 @@ const BenchmarkChart = () => {
182
  {/* Header */}
183
  <div className="text-center mb-6">
184
  <h1 className="text-3xl font-bold text-white mb-3">
185
- LLM Attack Success Rate with Transformation Methods
186
  </h1>
187
  <p className="text-slate-300">
188
- SafetyBench Aug 2025 - Real ASR Calculations
189
  </p>
190
-
191
- {/* Methodology Note */}
192
- <div className="mt-4 p-3 bg-yellow-900/30 border border-yellow-500/30 rounded-lg max-w-4xl mx-auto">
193
- <div className="flex items-start space-x-3">
194
- <div className="text-yellow-400 mt-1">⚠️</div>
195
- <div className="text-left">
196
- <p className="text-yellow-200 font-semibold mb-2">Methodology Note</p>
197
- <p className="text-yellow-100 text-sm leading-relaxed">
198
- <strong>Additive Visualization:</strong> This chart shows cumulative impact by progressively adding each transformation method's individual attack success rate.
199
- Values >100% represent transformation of multiple conversations off one failed, human seed conversation.
200
- Results are based on HarmBench Grading methodology and should be interpreted as relative performance indicators.
201
- </p>
202
- </div>
203
- </div>
204
  </div>
205
  </div>
206
 
207
- {/* Chart Container - Scrollable Box */}
208
  <div className="bg-white rounded-xl shadow-2xl p-4">
209
- <div className="h-96 overflow-y-auto pr-2">
210
  <div className="space-y-2">
211
  {benchmarkData.map((modelData, index) => {
212
  const currentValue = getCurrentValue(modelData, currentPhase);
213
  const baselineValue = modelData.baseline;
214
  const maxValue = 90;
215
- const barWidth = (currentValue / maxValue) * 100;
 
 
216
  const currentMethod = getCurrentMethod(modelData, currentPhase);
217
  const gain = currentValue - baselineValue;
218
 
219
  return (
220
  <div key={modelData.model} className="relative">
221
- {/* Model Name and Value */}
222
  <div className="flex items-center justify-between mb-1">
223
  <div>
224
  <h3 className="font-semibold text-gray-800 text-sm">
@@ -228,24 +201,37 @@ const BenchmarkChart = () => {
228
  {currentMethod}
229
  </p>
230
  </div>
231
- <div className="text-right">
 
 
 
 
 
232
  <span className="text-lg font-bold text-gray-700">
233
  {currentValue.toFixed(1)}%
234
  </span>
235
- {gain > 0 && (
236
- <div className="text-xs font-semibold text-green-600">
237
- +{gain.toFixed(1)} points
238
- </div>
239
- )}
240
  </div>
241
  </div>
242
 
243
- {/* Progress Bar */}
244
  <div className="relative h-6 bg-gray-200 rounded-full overflow-hidden">
 
 
 
 
 
 
 
245
  <div
246
- className={`h-full bg-gradient-to-r ${getBarColor(modelData, currentPhase)} rounded-full transition-all duration-2000 ease-out flex items-center justify-end pr-2`}
247
- style={{ width: `${Math.max(barWidth, 5)}%` }}
248
- >
 
 
 
 
 
 
249
  <div className="text-white font-semibold text-xs">
250
  {currentValue > 8 ? `${currentValue.toFixed(1)}%` : ''}
251
  </div>
@@ -265,7 +251,22 @@ const BenchmarkChart = () => {
265
  </div>
266
  <div className="flex items-center space-x-2">
267
  <div className="w-3 h-3 bg-gradient-to-r from-green-500 to-green-600 rounded"></div>
268
- <span className="text-gray-700">With Transformation Methods</span>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
269
  </div>
270
  </div>
271
  </div>
@@ -273,10 +274,10 @@ const BenchmarkChart = () => {
273
  {/* Footer */}
274
  <div className="mt-4 text-center text-slate-400 space-y-1">
275
  <p className="text-sm">
276
- Top performers: Grok 4 (85.15%), Deepseek R1-0528 (83.76%), Llama 3.1 405B (80.75%)
277
  </p>
278
  <p className="text-xs">
279
- Shows highest ASR achieved when combining human attempts with transformation methods
280
  </p>
281
  </div>
282
  </div>
 
1
  import React, { useState, useEffect } from 'react';
2
 
3
  const BenchmarkChart = () => {
4
+ // Real data sorted by Human Baseline (ascending)
5
  const benchmarkData = [
6
  {
7
+ model: "GPT 5 mini",
8
+ baseline: 7.67,
9
  methods: {
10
+ keyword_objective_combined: 11.28,
11
+ root_problem: 10.44
12
  }
13
  },
14
  {
15
+ model: "GPT 5",
16
+ baseline: 8.33,
17
  methods: {
18
+ keyword_objective_combined: 11.68,
19
+ root_problem: 12.46
20
  }
21
  },
22
  {
23
+ model: "Claude Opus 4.1",
24
+ baseline: 20.67,
25
  methods: {
26
+ keyword_objective_combined: 23.56
27
  }
28
  },
29
  {
30
+ model: "GPT o3",
31
+ baseline: 22.00,
32
  methods: {
33
+ keyword_objective_combined: 30.53
 
34
  }
35
  },
36
  {
37
+ model: "Claude 4 Sonnet",
38
+ baseline: 26.33,
39
  methods: {
40
+ keyword_objective_combined: 28.64
41
  }
42
  },
43
  {
44
+ model: "Llama 4 Maverick Instruct",
45
+ baseline: 45.00,
46
  methods: {
47
+ keyword_objective_combined: 56.46
48
  }
49
  },
50
  {
51
+ model: "Mixtral 8x22B",
52
+ baseline: 48.00,
53
  methods: {
54
+ keyword_objective_combined: 66.82
55
  }
56
  },
57
  {
58
+ model: "Gemini 2.5 Pro",
59
+ baseline: 55.67,
60
  methods: {
61
+ keyword_objective_combined: 74.14,
62
+ root_problem: 67.19
63
  }
64
  },
65
  {
66
+ model: "Llama 3 8B Instruct Reference",
67
+ baseline: 58.33,
68
  methods: {
69
+ keyword_objective_combined: 68.86
70
  }
71
  },
72
  {
73
+ model: "Llama 3.1 405B",
74
+ baseline: 67.00,
75
  methods: {
76
+ keyword_objective_combined: 80.75
77
  }
78
  },
79
  {
80
+ model: "Deepseek R1-0528",
81
+ baseline: 68.67,
82
  methods: {
83
+ keyword_objective_combined: 83.76
 
84
  }
85
  },
86
  {
87
+ model: "Grok 4",
88
+ baseline: 68.67,
89
  methods: {
90
+ keyword_objective_combined: 85.15
 
91
  }
92
  }
93
  ];
 
96
  const [currentMethodIndex, setCurrentMethodIndex] = useState(0);
97
 
98
  const synthesisMethodsOrder = ['keyword_objective_combined', 'root_problem'];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
99
 
100
  const getCurrentValue = (modelData, phase) => {
101
  if (phase === 'baseline') {
102
  return modelData.baseline;
103
  } else if (phase === 'additive_synthesis') {
 
104
  let maxASR = modelData.baseline;
105
 
106
  for (let i = 0; i < currentMethodIndex; i++) {
 
130
  if (availableMethods.length === 0) return 'Human Baseline';
131
 
132
  const lastMethod = availableMethods[availableMethods.length - 1];
133
+ let transformationName;
134
+ if (lastMethod === 'keyword_objective_combined') transformationName = 'Keyword/Objective Transformation';
135
+ else if (lastMethod === 'root_problem') transformationName = 'Root Problem Transformation';
136
+ else transformationName = lastMethod.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase());
137
 
138
+ return `Human Baseline + ${transformationName}`;
139
  };
140
 
141
+ const handleManualAnimate = () => {
142
+ if (currentPhase === 'baseline') {
143
+ setCurrentPhase('additive_synthesis');
144
+ setCurrentMethodIndex(0);
145
+
146
+ // Manually step through methods
147
+ setTimeout(() => setCurrentMethodIndex(1), 500);
148
+ setTimeout(() => setCurrentMethodIndex(2), 2500);
149
  } else {
150
+ setCurrentPhase('baseline');
151
+ setCurrentMethodIndex(0);
152
  }
153
  };
154
 
 
158
  {/* Header */}
159
  <div className="text-center mb-6">
160
  <h1 className="text-3xl font-bold text-white mb-3">
161
+ JulyAI Safety Benchmark For Frontier Models
162
  </h1>
163
  <p className="text-slate-300">
164
+ Attack Success Rate Analysis with Transformation Methods
165
  </p>
166
+
167
+ {/* Control Button */}
168
+ <div className="mt-4 flex justify-center">
169
+ <button
170
+ onClick={handleManualAnimate}
171
+ className="px-6 py-2 bg-blue-600 hover:bg-blue-700 text-white rounded-lg font-semibold transition-colors"
172
+ >
173
+ {currentPhase === 'baseline' ? 'Show Transformation Impact' : 'Reset to Baseline'}
174
+ </button>
 
 
 
 
 
175
  </div>
176
  </div>
177
 
178
+ {/* Chart Container - Taller */}
179
  <div className="bg-white rounded-xl shadow-2xl p-4">
180
+ <div className="h-[600px] overflow-y-auto pr-2">
181
  <div className="space-y-2">
182
  {benchmarkData.map((modelData, index) => {
183
  const currentValue = getCurrentValue(modelData, currentPhase);
184
  const baselineValue = modelData.baseline;
185
  const maxValue = 90;
186
+ const baselineWidth = (baselineValue / maxValue) * 100;
187
+ const totalWidth = (currentValue / maxValue) * 100;
188
+ const extensionWidth = totalWidth - baselineWidth;
189
  const currentMethod = getCurrentMethod(modelData, currentPhase);
190
  const gain = currentValue - baselineValue;
191
 
192
  return (
193
  <div key={modelData.model} className="relative">
194
+ {/* Model Name and Value - Fixed Height Layout */}
195
  <div className="flex items-center justify-between mb-1">
196
  <div>
197
  <h3 className="font-semibold text-gray-800 text-sm">
 
201
  {currentMethod}
202
  </p>
203
  </div>
204
+ <div className="text-right flex items-center space-x-2">
205
+ {gain > 0 && (
206
+ <span className="text-xs font-semibold text-green-600">
207
+ +{gain.toFixed(1)}
208
+ </span>
209
+ )}
210
  <span className="text-lg font-bold text-gray-700">
211
  {currentValue.toFixed(1)}%
212
  </span>
 
 
 
 
 
213
  </div>
214
  </div>
215
 
216
+ {/* Progress Bar - Shows Growth from Baseline */}
217
  <div className="relative h-6 bg-gray-200 rounded-full overflow-hidden">
218
+ {/* Baseline Bar (Blue) - Flat, no rounding for seamless extension */}
219
+ <div
220
+ className="absolute left-0 top-0 h-full bg-gradient-to-r from-blue-500 to-blue-600"
221
+ style={{ width: `${Math.max(baselineWidth, 5)}%` }}
222
+ />
223
+
224
+ {/* Extension Bar (Green) - Only rounded at the end */}
225
  <div
226
+ className="absolute top-0 h-full bg-gradient-to-r from-green-500 to-green-600 rounded-r-full transition-all duration-2000 ease-out"
227
+ style={{
228
+ left: `${baselineWidth}%`,
229
+ width: currentPhase === 'additive_synthesis' && currentMethodIndex > 0 ? `${extensionWidth}%` : '0%'
230
+ }}
231
+ />
232
+
233
+ {/* Percentage Label */}
234
+ <div className="absolute right-2 top-0 h-full flex items-center">
235
  <div className="text-white font-semibold text-xs">
236
  {currentValue > 8 ? `${currentValue.toFixed(1)}%` : ''}
237
  </div>
 
251
  </div>
252
  <div className="flex items-center space-x-2">
253
  <div className="w-3 h-3 bg-gradient-to-r from-green-500 to-green-600 rounded"></div>
254
+ <span className="text-gray-700">Transformation Extension</span>
255
+ </div>
256
+ </div>
257
+ </div>
258
+
259
+ {/* Methodology Note - Moved Below Chart */}
260
+ <div className="mt-6 p-3 bg-yellow-900/30 border border-yellow-500/30 rounded-lg max-w-4xl mx-auto">
261
+ <div className="flex items-start space-x-3">
262
+ <div className="text-yellow-400 mt-1">⚠️</div>
263
+ <div className="text-left">
264
+ <p className="text-yellow-200 font-semibold mb-2">Methodology Note</p>
265
+ <p className="text-yellow-100 text-sm leading-relaxed">
266
+ <strong>Additive Visualization:</strong> This chart shows cumulative impact by progressively adding each transformation method's individual attack success rate.
267
+ Values >100% represent transformation of multiple conversations off one failed, human seed conversation.
268
+ Results are based on HarmBench Grading methodology and should be interpreted as relative performance indicators.
269
+ </p>
270
  </div>
271
  </div>
272
  </div>
 
274
  {/* Footer */}
275
  <div className="mt-4 text-center text-slate-400 space-y-1">
276
  <p className="text-sm">
277
+ Sorted by Human Baseline ASR Click button above for manual control
278
  </p>
279
  <p className="text-xs">
280
+ Bars extend from baseline to show transformation method impact
281
  </p>
282
  </div>
283
  </div>