Buckets:
| { | |
| "task_id": "juliet_vulnerability_analyzer", | |
| "name": "Juliet Vulnerability Analyzer", | |
| "category": "Systems & Software Engineering", | |
| "base_image": "python", | |
| "platform": "linux/amd64", | |
| "internet": false, | |
| "cwd": "/home/workspace/juliet-static-analyzer/agent-start", | |
| "submit_paths": [ | |
| "." | |
| ], | |
| "submit_exclude": [ | |
| "out/", | |
| "__pycache__/", | |
| ".pytest_cache/", | |
| "*.pyc", | |
| ".DS_Store" | |
| ], | |
| "work": { | |
| "image_tag": "bae3ab85c5f5", | |
| "specs_dir": "/home/workspace/juliet-static-analyzer", | |
| "agent_query": "Implement a deterministic static analyzer in agent-start/analyzer.py for the provided Juliet facts schema. Report findings for CWE-121, CWE-122, CWE-190, CWE-476, CWE-416, and CWE-78. Modify analyzer.py and files you create under agent-start only. Do not use prohibited external analyzers, network access, hidden evaluator files, or hard-coded answers." | |
| }, | |
| "judge": { | |
| "image_tag": "0c4f2c0057c2", | |
| "eval_cmd": "python -c 'import json,subprocess; proc=subprocess.run([\"bash\",\"/root/sebench_private/juliet-static-analyzer-evaluator-v3/evaluator-hidden/score_hidden.sh\",\"/home/workspace/juliet-static-analyzer/agent-start\"], text=True, capture_output=True, timeout=120); output=(proc.stdout or \"\")+(proc.stderr or \"\"); print(output,end=\"\"); lines=output.splitlines(); score=float(next((x.split(\"=\",1)[1] for x in lines if x.startswith(\"SCORE=\")),\"0\")); status=next((x.split(\"=\",1)[1] for x in lines if x.startswith(\"SCORE_STATUS=\")),\"\"); valid=(status==\"OK\"); passed=1 if valid and score>0.0 else 0; print(\">>>>> Start Structured Result\"); print(json.dumps({\"valid\":valid,\"score\":score,\"pass_rate\":1.0 if passed else 0.0,\"total_tests\":1,\"passed\":passed,\"failed\":0 if passed else 1,\"errors\":0 if valid else 1,\"summary\":\"Score: {:.2f}\".format(score),\"details\":[{\"name\":\"hidden_score\",\"status\":\"PASSED\" if passed else \"FAILED\",\"score\":score,\"message\":output[-2000:]}],\"metrics\":{\"score\":score,\"runner_returncode\":proc.returncode}}, ensure_ascii=False)); print(\">>>>> End Structured Result\")'", | |
| "eval_timeout": 180, | |
| "parser": "structured_json", | |
| "score_direction": "maximize", | |
| "selection": "score_first", | |
| "rescale": { | |
| "kind": "linear", | |
| "lower": 0.0, | |
| "upper": 100.0 | |
| } | |
| } | |
| } | |
Xet Storage Details
- Size:
- 2.33 kB
- Xet hash:
- 850fa26bc0b930ae2896ca9447d5fefcba6e56be34be550ce8eb1f6e45157963
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.