| """Answer one request from the command line, or serve the model over HTTP (server.py). |
| |
| python -m decisionnode_edge path/to/decisionnode-edge request.json # or "-" to read the request from stdin |
| python -m decisionnode_edge serve path/to/decisionnode-edge --port 8000 |
| """ |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import sys |
|
|
| from . import EdgeModel, RequestError |
|
|
|
|
| def main(argv: list[str] | None = None) -> int: |
| argv = sys.argv[1:] if argv is None else argv |
| if argv[:1] == ["serve"]: |
| from .server import main as serve |
|
|
| return serve(argv[1:]) |
| parser = argparse.ArgumentParser(prog="python -m decisionnode_edge", description="Answer a /v1/decide request with a local DecisionNode Edge model.") |
| parser.add_argument("model", help="the model directory") |
| parser.add_argument("request", help="a JSON file with the request body, or - for stdin") |
| parser.add_argument("--device", default=None, help="cuda or cpu (default: cuda when present)") |
| parser.add_argument("--dtype", default=None, help="bfloat16 or float32 (default: bfloat16 on cuda, float32 on cpu)") |
| parser.add_argument("--backend", default=None, help="auto, torch, vllm, openvino, mlx or llamacpp (default: auto, the fastest installed)") |
| parser.add_argument("--artifacts", default=None, help="the directory python -m decisionnode_edge.build wrote (openvino, llamacpp)") |
| args = parser.parse_args(argv) |
| body = json.load(sys.stdin if args.request == "-" else open(args.request, encoding="utf-8")) |
| model = EdgeModel.load(args.model, device=args.device, dtype=args.dtype, backend=args.backend, artifacts=args.artifacts) |
| try: |
| response = model.decide(body) |
| except RequestError as error: |
| print(json.dumps({"error": str(error), "status": error.status}), file=sys.stderr) |
| return 1 |
| print(json.dumps(response, ensure_ascii=False, indent=2)) |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| sys.exit(main()) |
|
|
