Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -5,7 +5,19 @@ pipe = JailbreakPipeline(consider_output=False)
|
|
| 5 |
|
| 6 |
def analyze(prompt: str):
|
| 7 |
r = pipe.process(prompt)
|
| 8 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
|
| 10 |
with gr.Blocks() as demo:
|
| 11 |
gr.Markdown("# JailBreakDefense – Prompt Jailbreak Detector")
|
|
|
|
| 5 |
|
| 6 |
def analyze(prompt: str):
|
| 7 |
r = pipe.process(prompt)
|
| 8 |
+
fired = r.get("fired_rules", [])
|
| 9 |
+
|
| 10 |
+
# fired_rules can be ["rule1", "rule2"] OR [{"name": "rule1"}, ...]
|
| 11 |
+
if fired and isinstance(fired[0], dict):
|
| 12 |
+
fired_text = ", ".join(
|
| 13 |
+
str(item.get("name") or item.get("rule") or item.get("id") or item)
|
| 14 |
+
for item in fired
|
| 15 |
+
)
|
| 16 |
+
else:
|
| 17 |
+
fired_text = ", ".join(str(x) for x in fired)
|
| 18 |
+
|
| 19 |
+
return r["risk_score"], fired_text, r["safe_output"]
|
| 20 |
+
|
| 21 |
|
| 22 |
with gr.Blocks() as demo:
|
| 23 |
gr.Markdown("# JailBreakDefense – Prompt Jailbreak Detector")
|