File size: 2,468 Bytes
3ad1000 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | """Select a function with Nev and display an application-built call without executing it."""
import argparse
import json
from pathlib import Path
from huggingface_hub import snapshot_download
import torch
from decision_model import DecisionModel
from hf_decide import REPO, REVISION
from run_reference_examples import proposed_call
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--variant', choices=['v2', 'v3'], default='v3')
parser.add_argument('--language', choices=['en', 'it', 'both'], default='both')
parser.add_argument('--example', choices=['order', 'document', 'guide', 'payments'], default='document')
args = parser.parse_args()
if not torch.cuda.is_available():
parser.error('This A100 demonstration requires a CUDA-compatible GPU')
free, total = torch.cuda.mem_get_info()
cap = 4 * 1024**3
if free < cap:
parser.error('At least 4 GiB of free GPU memory is required')
torch.cuda.set_per_process_memory_fraction(min(cap / total, 1.0))
cases = json.loads(Path(__file__).with_name('reference_candidates.json').read_text())['cases']
cases = [c for c in cases if c['pair'] == f'function_selection/{args.example}'
and (args.language == 'both' or c['language'] == args.language)]
folder = Path(snapshot_download(REPO, revision=REVISION,
allow_patterns=[f'{args.variant}/**'], local_files_only=True)) / args.variant
model = DecisionModel.from_pretrained(folder, device='cuda')
results = []
for case in cases:
with torch.inference_mode():
response = model.evaluate(**case['request'])
answer = response['answers']['decision']
results.append({
'language': case['language'], 'input': case['request']['state'],
'model_selected_function': answer['choice'],
'probabilities': answer['probabilities'],
'expected_function': case['expected'],
'correct': answer['choice'] == case['expected'],
'application_built_call': proposed_call(answer['choice'], case['application_context']),
})
print(json.dumps({'repository': REPO, 'revision': REVISION, 'variant': args.variant,
'scope': 'Function-name selection only. Arguments come from the application; no tool is executed.',
'results': results}, ensure_ascii=False, indent=2))
if __name__ == '__main__':
main()
|