File size: 1,980 Bytes
47dda17 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | """Answer one request from the command line, or serve the model over HTTP (server.py).
python -m decisionnode_edge path/to/decisionnode-edge request.json # or "-" to read the request from stdin
python -m decisionnode_edge serve path/to/decisionnode-edge --port 8000
"""
from __future__ import annotations
import argparse
import json
import sys
from . import EdgeModel, RequestError
def main(argv: list[str] | None = None) -> int:
argv = sys.argv[1:] if argv is None else argv
if argv[:1] == ["serve"]:
from .server import main as serve
return serve(argv[1:])
parser = argparse.ArgumentParser(prog="python -m decisionnode_edge", description="Answer a /v1/decide request with a local DecisionNode Edge model.")
parser.add_argument("model", help="the model directory")
parser.add_argument("request", help="a JSON file with the request body, or - for stdin")
parser.add_argument("--device", default=None, help="cuda or cpu (default: cuda when present)")
parser.add_argument("--dtype", default=None, help="bfloat16 or float32 (default: bfloat16 on cuda, float32 on cpu)")
parser.add_argument("--backend", default=None, help="auto, torch, vllm, openvino, mlx or llamacpp (default: auto, the fastest installed)")
parser.add_argument("--artifacts", default=None, help="the directory python -m decisionnode_edge.build wrote (openvino, llamacpp)")
args = parser.parse_args(argv)
body = json.load(sys.stdin if args.request == "-" else open(args.request, encoding="utf-8"))
model = EdgeModel.load(args.model, device=args.device, dtype=args.dtype, backend=args.backend, artifacts=args.artifacts)
try:
response = model.decide(body)
except RequestError as error:
print(json.dumps({"error": str(error), "status": error.status}), file=sys.stderr)
return 1
print(json.dumps(response, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
sys.exit(main())
|