File size: 908 Bytes
db50de9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | import sys
from llama_cpp import Llama # pip install llama-cpp-python
from tools import parse_call, run_tool
llm = Llama(model_path=sys.argv[1], n_ctx=2048, verbose=False)
msgs = []
while True:
q = input("you> ").strip()
if not q: break
msgs.append({"role": "user", "content": q})
for _ in range(3):
out = llm.create_chat_completion(messages=msgs, max_tokens=700, temperature=0.6, top_p=0.95)["choices"][0]["message"]["content"]
if "<think>" not in out: out = "<think>\n" + out # the chat template already opened the think block
msgs.append({"role": "assistant", "content": out})
call = parse_call(out)
if not call: break
try: res = run_tool(call["name"], call["arguments"])
except Exception as e: res = "error: " + str(e)
msgs.append({"role": "tool", "content": res})
print(out.split("</think>")[-1].strip())
|