File size: 1,351 Bytes
1ac2be1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
"""Run the published Modilify Mk2 model with Apple Silicon MLX."""

from __future__ import annotations

import argparse
from pathlib import Path


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--model", type=Path, default=Path(__file__).resolve().parent)
    parser.add_argument("--prompt", default="你好,请介绍一下自己。")
    parser.add_argument("--max-new-tokens", type=int, default=256)
    parser.add_argument("--max-denoising-steps", type=int, default=256)
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--no-thinking", action="store_true")
    parser.add_argument("--stream", action="store_true")
    args = parser.parse_args()
    if min(args.max_new_tokens, args.max_denoising_steps) < 1:
        parser.error("Generation budgets must be positive.")

    from modilify_mk2_mlx import load

    runtime = load(args.model)
    result = runtime.generate(
        [{"role": "user", "content": args.prompt}],
        enable_thinking=not args.no_thinking, seed=args.seed,
        max_new_tokens=args.max_new_tokens, max_denoising_steps=args.max_denoising_steps,
        on_text=(lambda chunk: print(chunk, end="", flush=True)) if args.stream else None,
    )
    print("" if args.stream else result.text)


if __name__ == "__main__":
    main()