{ "attack_breakdown": { "safety_scores": { "Code/Encryption Prompt": { "Fanar": 0.3225806451612903, "Jais 30B": 0.27419354838709675, "GPT-4o": 0.3870967741935484, "ALLaM 7B": 0.5161290322580645, "CR-7B-Arabic": 0.0967741935483871, "Mistral-Saba": 0.11290322580645161, "Claude-3.7-Sonnet": 0.7096774193548387 }, "Direct Prompting": { "Fanar": 0.3409090909090909, "Jais 30B": 0.26136363636363635, "GPT-4o": 0.29545454545454547, "ALLaM 7B": 0.45454545454545453, "CR-7B-Arabic": 0.23863636363636365, "Mistral-Saba": 0.26136363636363635, "Claude-3.7-Sonnet": 0.5227272727272727 }, "False Premise": { "Fanar": 0.3698630136986301, "Jais 30B": 0.3561643835616438, "GPT-4o": 0.4794520547945205, "ALLaM 7B": 0.4246575342465753, "CR-7B-Arabic": 0.3150684931506849, "Mistral-Saba": 0.4383561643835616, "Claude-3.7-Sonnet": 0.6575342465753424 }, "Hypothetical Testing": { "Fanar": 0.3055555555555556, "Jais 30B": 0.4027777777777778, "GPT-4o": 0.4444444444444444, "ALLaM 7B": 0.4305555555555556, "CR-7B-Arabic": 0.25, "Mistral-Saba": 0.2916666666666667, "Claude-3.7-Sonnet": 0.6388888888888888 }, "Out of Context": { "Fanar": 0.7513227513227513, "Jais 30B": 0.7619047619047619, "GPT-4o": 0.746031746031746, "ALLaM 7B": 0.8306878306878307, "CR-7B-Arabic": 0.708994708994709, "Mistral-Saba": 0.7037037037037037, "Claude-3.7-Sonnet": 0.8624338624338624 }, "Persona Emulation": { "Fanar": 0.373134328358209, "Jais 30B": 0.34328358208955223, "GPT-4o": 0.3582089552238806, "ALLaM 7B": 0.47761194029850745, "CR-7B-Arabic": 0.23880597014925373, "Mistral-Saba": 0.2537313432835821, "Claude-3.7-Sonnet": 0.4925373134328358 }, "Step-by-Step Escalation": { "Fanar": 0.6222222222222222, "Jais 30B": 0.6388888888888888, "GPT-4o": 0.6555555555555556, "ALLaM 7B": 0.6666666666666666, "CR-7B-Arabic": 0.6, "Mistral-Saba": 0.6277777777777778, "Claude-3.7-Sonnet": 0.7277777777777777 }, "Storytelling": { "Fanar": 0.34285714285714286, "Jais 30B": 0.2714285714285714, "GPT-4o": 0.34285714285714286, "ALLaM 7B": 0.3, "CR-7B-Arabic": 0.21428571428571427, "Mistral-Saba": 0.2, "Claude-3.7-Sonnet": 0.5 } }, "slightly_unsafe_scores": { "Code/Encryption Prompt": { "Fanar": 0.12903225806451613, "Jais 30B": 0.16129032258064516, "GPT-4o": 0.06451612903225806, "ALLaM 7B": 0.06451612903225806, "CR-7B-Arabic": 0.1774193548387097, "Mistral-Saba": 0.3387096774193548, "Claude-3.7-Sonnet": 0.04838709677419355 }, "Direct Prompting": { "Fanar": 0.06818181818181818, "Jais 30B": 0.07954545454545454, "GPT-4o": 0.17045454545454544, "ALLaM 7B": 0.11363636363636363, "CR-7B-Arabic": 0.14772727272727273, "Mistral-Saba": 0.13636363636363635, "Claude-3.7-Sonnet": 0.125 }, "False Premise": { "Fanar": 0.0273972602739726, "Jais 30B": 0.0821917808219178, "GPT-4o": 0.0958904109589041, "ALLaM 7B": 0.1506849315068493, "CR-7B-Arabic": 0.2191780821917808, "Mistral-Saba": 0.2054794520547945, "Claude-3.7-Sonnet": 0.1506849315068493 }, "Hypothetical Testing": { "Fanar": 0.16666666666666666, "Jais 30B": 0.08333333333333333, "GPT-4o": 0.08333333333333333, "ALLaM 7B": 0.05555555555555555, "CR-7B-Arabic": 0.125, "Mistral-Saba": 0.125, "Claude-3.7-Sonnet": 0.05555555555555555 }, "Out of Context": { "Fanar": 0.0582010582010582, "Jais 30B": 0.021164021164021163, "GPT-4o": 0.047619047619047616, "ALLaM 7B": 0.047619047619047616, "CR-7B-Arabic": 0.09523809523809523, "Mistral-Saba": 0.07936507936507936, "Claude-3.7-Sonnet": 0.042328042328042326 }, "Persona Emulation": { "Fanar": 0.14925373134328357, "Jais 30B": 0.14925373134328357, "GPT-4o": 0.14925373134328357, "ALLaM 7B": 0.07462686567164178, "CR-7B-Arabic": 0.1791044776119403, "Mistral-Saba": 0.16417910447761194, "Claude-3.7-Sonnet": 0.13432835820895522 }, "Step-by-Step Escalation": { "Fanar": 0.08333333333333333, "Jais 30B": 0.08333333333333333, "GPT-4o": 0.07222222222222222, "ALLaM 7B": 0.08888888888888889, "CR-7B-Arabic": 0.07222222222222222, "Mistral-Saba": 0.07222222222222222, "Claude-3.7-Sonnet": 0.08333333333333333 }, "Storytelling": { "Fanar": 0.14285714285714285, "Jais 30B": 0.22857142857142856, "GPT-4o": 0.08571428571428572, "ALLaM 7B": 0.2, "CR-7B-Arabic": 0.18571428571428572, "Mistral-Saba": 0.14285714285714285, "Claude-3.7-Sonnet": 0.14285714285714285 } }, "moderately_unsafe_scores": { "Code/Encryption Prompt": { "Fanar": 0.08064516129032258, "Jais 30B": 0.16129032258064516, "GPT-4o": 0.08064516129032258, "ALLaM 7B": 0.16129032258064516, "CR-7B-Arabic": 0.1774193548387097, "Mistral-Saba": 0.16129032258064516, "Claude-3.7-Sonnet": 0.08064516129032258 }, "Direct Prompting": { "Fanar": 0.14772727272727273, "Jais 30B": 0.1590909090909091, "GPT-4o": 0.10227272727272728, "ALLaM 7B": 0.125, "CR-7B-Arabic": 0.18181818181818182, "Mistral-Saba": 0.1590909090909091, "Claude-3.7-Sonnet": 0.10227272727272728 }, "False Premise": { "Fanar": 0.1232876712328767, "Jais 30B": 0.1232876712328767, "GPT-4o": 0.0958904109589041, "ALLaM 7B": 0.1506849315068493, "CR-7B-Arabic": 0.1506849315068493, "Mistral-Saba": 0.0821917808219178, "Claude-3.7-Sonnet": 0.0136986301369863 }, "Hypothetical Testing": { "Fanar": 0.1527777777777778, "Jais 30B": 0.1388888888888889, "GPT-4o": 0.125, "ALLaM 7B": 0.1388888888888889, "CR-7B-Arabic": 0.20833333333333334, "Mistral-Saba": 0.1527777777777778, "Claude-3.7-Sonnet": 0.06944444444444445 }, "Out of Context": { "Fanar": 0.037037037037037035, "Jais 30B": 0.05291005291005291, "GPT-4o": 0.047619047619047616, "ALLaM 7B": 0.037037037037037035, "CR-7B-Arabic": 0.0582010582010582, "Mistral-Saba": 0.07407407407407407, "Claude-3.7-Sonnet": 0.037037037037037035 }, "Persona Emulation": { "Fanar": 0.08955223880597014, "Jais 30B": 0.07462686567164178, "GPT-4o": 0.04477611940298507, "ALLaM 7B": 0.029850746268656716, "CR-7B-Arabic": 0.14925373134328357, "Mistral-Saba": 0.16417910447761194, "Claude-3.7-Sonnet": 0.07462686567164178 }, "Step-by-Step Escalation": { "Fanar": 0.1, "Jais 30B": 0.08888888888888889, "GPT-4o": 0.09444444444444444, "ALLaM 7B": 0.1, "CR-7B-Arabic": 0.12777777777777777, "Mistral-Saba": 0.10555555555555556, "Claude-3.7-Sonnet": 0.08333333333333333 }, "Storytelling": { "Fanar": 0.2, "Jais 30B": 0.18571428571428572, "GPT-4o": 0.18571428571428572, "ALLaM 7B": 0.21428571428571427, "CR-7B-Arabic": 0.17142857142857143, "Mistral-Saba": 0.18571428571428572, "Claude-3.7-Sonnet": 0.08571428571428572 } }, "extremely_unsafe_scores": { "Code/Encryption Prompt": { "Fanar": 0.46774193548387094, "Jais 30B": 0.4032258064516129, "GPT-4o": 0.46774193548387094, "ALLaM 7B": 0.25806451612903225, "CR-7B-Arabic": 0.5483870967741935, "Mistral-Saba": 0.3870967741935484, "Claude-3.7-Sonnet": 0.16129032258064516 }, "Direct Prompting": { "Fanar": 0.4431818181818182, "Jais 30B": 0.5, "GPT-4o": 0.4318181818181818, "ALLaM 7B": 0.3068181818181818, "CR-7B-Arabic": 0.4318181818181818, "Mistral-Saba": 0.4431818181818182, "Claude-3.7-Sonnet": 0.25 }, "False Premise": { "Fanar": 0.4794520547945205, "Jais 30B": 0.4383561643835616, "GPT-4o": 0.3287671232876712, "ALLaM 7B": 0.273972602739726, "CR-7B-Arabic": 0.3150684931506849, "Mistral-Saba": 0.273972602739726, "Claude-3.7-Sonnet": 0.1780821917808219 }, "Hypothetical Testing": { "Fanar": 0.375, "Jais 30B": 0.375, "GPT-4o": 0.3472222222222222, "ALLaM 7B": 0.375, "CR-7B-Arabic": 0.4166666666666667, "Mistral-Saba": 0.4305555555555556, "Claude-3.7-Sonnet": 0.2361111111111111 }, "Out of Context": { "Fanar": 0.15343915343915343, "Jais 30B": 0.164021164021164, "GPT-4o": 0.15873015873015872, "ALLaM 7B": 0.08465608465608465, "CR-7B-Arabic": 0.13756613756613756, "Mistral-Saba": 0.14285714285714285, "Claude-3.7-Sonnet": 0.0582010582010582 }, "Persona Emulation": { "Fanar": 0.3880597014925373, "Jais 30B": 0.43283582089552236, "GPT-4o": 0.44776119402985076, "ALLaM 7B": 0.417910447761194, "CR-7B-Arabic": 0.43283582089552236, "Mistral-Saba": 0.417910447761194, "Claude-3.7-Sonnet": 0.29850746268656714 }, "Step-by-Step Escalation": { "Fanar": 0.19444444444444445, "Jais 30B": 0.18888888888888888, "GPT-4o": 0.17777777777777778, "ALLaM 7B": 0.14444444444444443, "CR-7B-Arabic": 0.2, "Mistral-Saba": 0.19444444444444445, "Claude-3.7-Sonnet": 0.10555555555555556 }, "Storytelling": { "Fanar": 0.3142857142857143, "Jais 30B": 0.3142857142857143, "GPT-4o": 0.38571428571428573, "ALLaM 7B": 0.2857142857142857, "CR-7B-Arabic": 0.42857142857142855, "Mistral-Saba": 0.4714285714285714, "Claude-3.7-Sonnet": 0.2714285714285714 } } } }