| |
| """Send one inference request with a fixed repetition penalty of 1.05.""" |
| import argparse |
| import json |
| import urllib.request |
| from pathlib import Path |
|
|
|
|
| def main(): |
| p = argparse.ArgumentParser(description=__doc__) |
| p.add_argument('--prompt', required=True) |
| p.add_argument('--base-url', default='http://127.0.0.1:28080/v1') |
| p.add_argument('--model', default='Safin-1.5') |
| p.add_argument('--max-tokens', type=int, default=65536) |
| p.add_argument('--output') |
| a = p.parse_args() |
| body = { |
| 'model': a.model, 'messages': [{'role': 'user', 'content': a.prompt}], |
| 'temperature': 1.0, 'top_p': 0.95, 'seed': 42, |
| 'max_tokens': a.max_tokens, 'presence_penalty': 0.0, |
| 'repetition_penalty': 1.05, |
| 'chat_template_kwargs': {'enable_thinking': True}, 'stream': False, |
| } |
| opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) |
| req = urllib.request.Request( |
| a.base_url.rstrip('/') + '/chat/completions', |
| data=json.dumps(body).encode(), headers={'Content-Type': 'application/json'}, |
| ) |
| with opener.open(req, timeout=3600) as response: |
| raw = json.load(response) |
| if a.output: |
| Path(a.output).write_text(json.dumps({'request': body, 'response': raw}, ensure_ascii=False, indent=2) + '\n') |
| print(raw['choices'][0]['message'].get('content') or '') |
|
|
|
|
| if __name__ == '__main__': |
| main() |
|
|
