Shubham Shah commited on
Commit ·
378972b
1
Parent(s): 1b7b2a4
P2: V0 commit
Browse files- .idea/.gitignore +5 -0
- .idea/inspectionProfiles/Project_Default.xml +15 -0
- .idea/meta-rl-dsa-solver.iml +9 -0
- .idea/misc.xml +12 -0
- .idea/modules.xml +8 -0
- .idea/vcs.xml +6 -0
- scripts/test_verifier.py +16 -0
- verifier/__init__.py +0 -0
- verifier/metrics.py +12 -0
- verifier/sandbox.py +31 -0
- verifier/verifier.py +23 -0
.idea/.gitignore
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Default ignored files
|
| 2 |
+
/shelf/
|
| 3 |
+
/workspace.xml
|
| 4 |
+
# Editor-based HTTP Client requests
|
| 5 |
+
/httpRequests/
|
.idea/inspectionProfiles/Project_Default.xml
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<component name="InspectionProjectProfileManager">
|
| 2 |
+
<profile version="1.0">
|
| 3 |
+
<option name="myName" value="Project Default" />
|
| 4 |
+
<inspection_tool class="PyCompatibilityInspection" enabled="true" level="WARNING" enabled_by_default="true">
|
| 5 |
+
<option name="ourVersions">
|
| 6 |
+
<value>
|
| 7 |
+
<list size="2">
|
| 8 |
+
<item index="0" class="java.lang.String" itemvalue="2.7" />
|
| 9 |
+
<item index="1" class="java.lang.String" itemvalue="3.14" />
|
| 10 |
+
</list>
|
| 11 |
+
</value>
|
| 12 |
+
</option>
|
| 13 |
+
</inspection_tool>
|
| 14 |
+
</profile>
|
| 15 |
+
</component>
|
.idea/meta-rl-dsa-solver.iml
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<?xml version="1.0" encoding="UTF-8"?>
|
| 2 |
+
<module type="JAVA_MODULE" version="4">
|
| 3 |
+
<component name="NewModuleRootManager" inherit-compiler-output="true">
|
| 4 |
+
<exclude-output />
|
| 5 |
+
<content url="file://$MODULE_DIR$" />
|
| 6 |
+
<orderEntry type="inheritedJdk" />
|
| 7 |
+
<orderEntry type="sourceFolder" forTests="false" />
|
| 8 |
+
</component>
|
| 9 |
+
</module>
|
.idea/misc.xml
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<?xml version="1.0" encoding="UTF-8"?>
|
| 2 |
+
<project version="4">
|
| 3 |
+
<component name="Black">
|
| 4 |
+
<option name="sdkName" value="Python 3.9" />
|
| 5 |
+
</component>
|
| 6 |
+
<component name="ProjectRootManager" version="2" languageLevel="JDK_24" default="true" project-jdk-name="Python 3.9" project-jdk-type="Python SDK">
|
| 7 |
+
<output url="file://$PROJECT_DIR$/out" />
|
| 8 |
+
</component>
|
| 9 |
+
<component name="PythonCompatibilityInspectionAdvertiser">
|
| 10 |
+
<option name="version" value="3" />
|
| 11 |
+
</component>
|
| 12 |
+
</project>
|
.idea/modules.xml
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<?xml version="1.0" encoding="UTF-8"?>
|
| 2 |
+
<project version="4">
|
| 3 |
+
<component name="ProjectModuleManager">
|
| 4 |
+
<modules>
|
| 5 |
+
<module fileurl="file://$PROJECT_DIR$/.idea/meta-rl-dsa-solver.iml" filepath="$PROJECT_DIR$/.idea/meta-rl-dsa-solver.iml" />
|
| 6 |
+
</modules>
|
| 7 |
+
</component>
|
| 8 |
+
</project>
|
.idea/vcs.xml
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<?xml version="1.0" encoding="UTF-8"?>
|
| 2 |
+
<project version="4">
|
| 3 |
+
<component name="VcsDirectoryMappings">
|
| 4 |
+
<mapping directory="" vcs="Git" />
|
| 5 |
+
</component>
|
| 6 |
+
</project>
|
scripts/test_verifier.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from verifier.verifier import verify
|
| 2 |
+
test_cases = [
|
| 3 |
+
("5\n", "10"),
|
| 4 |
+
("0\n", "0"),
|
| 5 |
+
("-3\n", "-6"),
|
| 6 |
+
]
|
| 7 |
+
|
| 8 |
+
code = """
|
| 9 |
+
n = int(input())
|
| 10 |
+
print(n * 2)
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
reward, info = verify(code, test_cases)
|
| 14 |
+
|
| 15 |
+
print("Reward:", reward)
|
| 16 |
+
print(info)
|
verifier/__init__.py
ADDED
|
File without changes
|
verifier/metrics.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
def compute_pass_rate(results):
|
| 2 |
+
total = len(results)
|
| 3 |
+
passed = sum(1 for r in results if r["passed"])
|
| 4 |
+
|
| 5 |
+
pass_rate = passed / total if total > 0 else 0.0
|
| 6 |
+
reward = 1.0 if pass_rate == 1.0 else 0.0
|
| 7 |
+
|
| 8 |
+
return reward, {
|
| 9 |
+
"passed": passed,
|
| 10 |
+
"total": total,
|
| 11 |
+
"pass_rate": pass_rate,
|
| 12 |
+
}
|
verifier/sandbox.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import subprocess
|
| 2 |
+
import tempfile
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def run_code(code: str, stdin: str, timeout: int = 2):
|
| 7 |
+
try:
|
| 8 |
+
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f:
|
| 9 |
+
f.write(code)
|
| 10 |
+
path = f.name
|
| 11 |
+
|
| 12 |
+
result = subprocess.run(
|
| 13 |
+
["python3", path],
|
| 14 |
+
input=stdin,
|
| 15 |
+
text=True,
|
| 16 |
+
capture_output=True,
|
| 17 |
+
timeout=timeout,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
os.remove(path)
|
| 21 |
+
|
| 22 |
+
if result.returncode != 0:
|
| 23 |
+
return False, result.stderr.strip()
|
| 24 |
+
|
| 25 |
+
return True, result.stdout.strip()
|
| 26 |
+
|
| 27 |
+
except subprocess.TimeoutExpired:
|
| 28 |
+
return False, "TIMEOUT"
|
| 29 |
+
|
| 30 |
+
except Exception as e:
|
| 31 |
+
return False, f"ERROR: {e}"
|
verifier/verifier.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from verifier.sandbox import run_code
|
| 2 |
+
from verifier.metrics import compute_pass_rate
|
| 3 |
+
|
| 4 |
+
def verify(code: str, test_cases):
|
| 5 |
+
results = []
|
| 6 |
+
|
| 7 |
+
for stdin, expected in test_cases:
|
| 8 |
+
ok, output = run_code(code, stdin)
|
| 9 |
+
passed = ok and output.strip() == expected.strip()
|
| 10 |
+
|
| 11 |
+
results.append({
|
| 12 |
+
"input": stdin.strip(),
|
| 13 |
+
"expected": expected.strip(),
|
| 14 |
+
"output": output.strip(),
|
| 15 |
+
"passed": passed,
|
| 16 |
+
})
|
| 17 |
+
|
| 18 |
+
reward, metrics = compute_pass_rate(results)
|
| 19 |
+
|
| 20 |
+
return reward, {
|
| 21 |
+
**metrics,
|
| 22 |
+
"results": results,
|
| 23 |
+
}
|