Modilify-Mk2-mlx / inference.py
ydy9038074's picture
Publish Modilify Mk2 MLX step 4400
1ac2be1 verified
Raw History Blame Contribute Delete
1.35 kB
"""Run the published Modilify Mk2 model with Apple Silicon MLX."""
from __future__ import annotations
import argparse
from pathlib import Path
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", type=Path, default=Path(__file__).resolve().parent)
parser.add_argument("--prompt", default="你好,请介绍一下自己。")
parser.add_argument("--max-new-tokens", type=int, default=256)
parser.add_argument("--max-denoising-steps", type=int, default=256)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--no-thinking", action="store_true")
parser.add_argument("--stream", action="store_true")
args = parser.parse_args()
if min(args.max_new_tokens, args.max_denoising_steps) < 1:
parser.error("Generation budgets must be positive.")
from modilify_mk2_mlx import load
runtime = load(args.model)
result = runtime.generate(
[{"role": "user", "content": args.prompt}],
enable_thinking=not args.no_thinking, seed=args.seed,
max_new_tokens=args.max_new_tokens, max_denoising_steps=args.max_denoising_steps,
on_text=(lambda chunk: print(chunk, end="", flush=True)) if args.stream else None,
)
print("" if args.stream else result.text)
if __name__ == "__main__":
main()