occamy-1.0-FP8 / evaluation /code_check.py
Eang's picture
Release Occamy 1.0 FP8
ac48ca6
Raw History Blame Contribute Delete
1.49 kB
import ast,json,re,sys,resource
resource.setrlimit(resource.RLIMIT_CPU,(3,3));resource.setrlimit(resource.RLIMIT_AS,(512*1024**2,512*1024**2))
x=json.load(sys.stdin);s=x['code'].strip();s=re.sub(r'^```(?:python)?\s*|\s*```$','',s);t=ast.parse(s)
allowed_methods={'append','extend','pop','insert','remove','sort','reverse','copy','count','index','lower','upper','isalnum','isalpha','isdigit','strip','split','join','replace','get','items','keys','values','add','discard','update'}
for n in ast.walk(t):
if isinstance(n,(ast.Import,ast.ImportFrom,ast.ClassDef,ast.Global,ast.Nonlocal)):raise ValueError('Disallowed node')
if isinstance(n,ast.Name) and n.id.startswith('__'):raise ValueError('Disallowed name')
if isinstance(n,ast.Attribute) and n.attr not in allowed_methods:raise ValueError('Disallowed attribute')
if isinstance(n,ast.FunctionDef) and n.decorator_list:raise ValueError('Disallowed decorator')
for n in t.body:
if not isinstance(n,(ast.FunctionDef,ast.Assign,ast.AnnAssign,ast.Expr)):raise ValueError('Disallowed top level')
builtins={k:__builtins__.__dict__[k] for k in ['int','float','str','bool','list','dict','tuple','set','len','range','enumerate','zip','sorted','reversed','min','max','sum','abs','all','any','isinstance','ValueError','TypeError']}
ns={'__builtins__':builtins};exec(compile(t,'generated','exec'),ns);f=ns['solve'];results=[]
for args,expected in x['tests']:results.append(f(*args)==expected)
print(json.dumps({'pass':all(results),'tests':results}))