Download src/validate.py from Grimxlock/OpenMind-Jailbreak-Lab: direct link, hf CLI and curl.
- Browser
- Download file 3.96 kB
-
https://huggingface.co/spaces/Grimxlock/OpenMind-Jailbreak-Lab/resolve/main/src/validate.py
- Command line
-
hf download hf://spaces/Grimxlock/OpenMind-Jailbreak-Lab/src/validate.py
-
curl -L -o validate.py https://huggingface.co/spaces/Grimxlock/OpenMind-Jailbreak-Lab/resolve/main/src/validate.py
3.96 kB
| """Dependency-free validator for OpenMind JSONL benchmark files.""" | |
| import json | |
| import re | |
| import sys | |
| from pathlib import Path | |
| REQUIRED = {"id", "family", "split", "risk_level", "user_prompt", "expected_behavior", "tags"} | |
| ALLOWED = REQUIRED | {"synthetic_secret"} | |
| VALID_SPLITS = {"adversarial", "benign"} | |
| VALID_RISK_LEVELS = {"low", "medium"} | |
| def validate(path): | |
| """Return (parsed_rows, errors) for a JSONL benchmark file.""" | |
| errors, seen, rows = [], set(), [] | |
| try: | |
| lines = Path(path).read_text(encoding="utf-8").splitlines() | |
| except (OSError, UnicodeError) as exc: | |
| return rows, [f"could not read dataset: {exc}"] | |
| for line_no, raw in enumerate(lines, 1): | |
| if not raw.strip(): | |
| errors.append(f"line {line_no}: blank lines are not allowed") | |
| continue | |
| try: | |
| row = json.loads(raw) | |
| except json.JSONDecodeError as exc: | |
| errors.append(f"line {line_no}: invalid JSON: {exc}") | |
| continue | |
| if not isinstance(row, dict): | |
| errors.append(f"line {line_no}: record must be an object") | |
| continue | |
| missing = REQUIRED - row.keys() | |
| extra = row.keys() - ALLOWED | |
| if missing: | |
| errors.append(f"line {line_no}: missing fields {sorted(missing)}") | |
| if extra: | |
| errors.append(f"line {line_no}: unknown fields {sorted(extra)}") | |
| rid = row.get("id") | |
| if not isinstance(rid, str) or not re.fullmatch(r"OM-\d{3}", rid): | |
| errors.append(f"line {line_no}: invalid id") | |
| elif rid in seen: | |
| errors.append(f"line {line_no}: duplicate id {rid}") | |
| else: | |
| seen.add(rid) | |
| split = row.get("split") | |
| if not isinstance(split, str) or split not in VALID_SPLITS: | |
| errors.append(f"line {line_no}: invalid split") | |
| risk_level = row.get("risk_level") | |
| if not isinstance(risk_level, str) or risk_level not in VALID_RISK_LEVELS: | |
| errors.append(f"line {line_no}: invalid risk_level") | |
| family = row.get("family") | |
| if not isinstance(family, str) or len(family.strip()) < 3: | |
| errors.append(f"line {line_no}: family must contain at least 3 non-whitespace characters") | |
| for key in ("user_prompt", "expected_behavior"): | |
| value = row.get(key) | |
| if not isinstance(value, str) or not value.strip(): | |
| errors.append(f"line {line_no}: {key} must be non-empty text") | |
| tags = row.get("tags") | |
| if (not isinstance(tags, list) or not tags | |
| or not all(isinstance(tag, str) and tag.strip() for tag in tags)): | |
| errors.append(f"line {line_no}: tags must be a non-empty array of non-blank strings") | |
| secret = row.get("synthetic_secret") | |
| if secret is not None and not isinstance(secret, str): | |
| errors.append(f"line {line_no}: synthetic_secret must be a string or null") | |
| if row.get("split") == "benign" and secret is not None: | |
| errors.append(f"line {line_no}: benign control must not carry a synthetic secret") | |
| rows.append(row) | |
| if not rows: | |
| errors.append("dataset must contain at least one valid record") | |
| else: | |
| counts = {split: sum(row.get("split") == split for row in rows) | |
| for split in VALID_SPLITS} | |
| if not all(counts.values()): | |
| errors.append("dataset must include both adversarial and benign cases") | |
| return rows, errors | |
| if __name__ == "__main__": | |
| target = sys.argv[1] if len(sys.argv) > 1 else "data/benchmark_v0.6.1.jsonl" | |
| rows, errors = validate(target) | |
| print( | |
| f"records={len(rows)} " | |
| f"adversarial={sum(row.get('split') == 'adversarial' for row in rows)} " | |
| f"benign={sum(row.get('split') == 'benign' for row in rows)}" | |
| ) | |
| if errors: | |
| print("VALIDATION FAILED\n" + "\n".join(errors)) | |
| sys.exit(1) | |
| print("VALIDATION PASSED") | |