File size: 1,621 Bytes
4dc5bc6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 | #!/usr/bin/env python3
from __future__ import annotations
import argparse, json, os
from pathlib import Path
from shieldstral_policy import discover_model, score_policy
def main():
root=Path(__file__).resolve().parents[1]
p=argparse.ArgumentParser(description='Score an AXONVERTEX deployment policy outside the Appendix B taxonomy.')
p.add_argument('--policy',required=True)
p.add_argument('--document',required=True)
p.add_argument('--document-type',choices=('prompt','response'),default='prompt')
p.add_argument('--instruct',default='Evaluate whether the document matches the query criteria.')
p.add_argument('--threshold',type=float,default=0.5)
p.add_argument('--base-url',default=os.getenv('BASE_URL','http://127.0.0.1:18190/v1'))
p.add_argument('--model',default=None)
a=p.parse_args()
registry=json.loads((root/'taxonomy/deployment_policies.json').read_text(encoding='utf-8'))
policies={x['id']:x for x in registry['policies']}
pid=a.policy.upper()
if pid not in policies: raise SystemExit(f'Unknown deployment policy: {pid}')
policy=policies[pid]; query=policy[f'{a.document_type}_query']; model=discover_model(a.base_url,a.model)
score,_=score_policy(base_url=a.base_url,model=model,instruct=a.instruct,query=query,document=a.document,threshold=a.threshold)
print(json.dumps({'classification_mode':'deployment_policy_binary','registry_disclosure':registry['disclosure'],'model':model,'document_type':a.document_type,'policy':{'id':pid,'name':policy['name'],'query':query,**score}},indent=2,ensure_ascii=False))
if __name__=='__main__': main()
|