Tetu-Code-2 / chat.py
tetu-AI's picture
Initial release
2309cb1 verified
Raw History Blame Contribute Delete
1.67 kB
"""Chat with Têtu Code 2 in the terminal (no tools). For the agent in OpenCode: python -m tetu_code.serve
python chat.py (GPU) python chat.py --cpu
"""
import argparse
from pathlib import Path
import re
import torch
from tetu_code import fmt
from tetu_code.generate import Decoder, load
ROOT = Path(__file__).resolve().parent
def main():
p = argparse.ArgumentParser()
p.add_argument("--cpu", action="store_true")
p.add_argument("--temp", type=float, default=0.2)
p.add_argument("--max_new", type=int, default=1024)
a = p.parse_args()
dev = "cpu" if a.cpu or not torch.cuda.is_available() else "cuda"
model, tok = load(str(ROOT), device=dev, dtype=torch.float32 if dev == "cpu" else torch.bfloat16)
dec = Decoder(model, tok, max_len=16384, graph=dev == "cuda")
stop = {tok.token_to_id(t) for t in ("<|end|>", "<|eos|>", "<|user|>", "<|/call|>")}
text = f"<|system|>{fmt.system_text()}"
print("Têtu Code 2 — /reset to clear, Ctrl+C to quit")
while True:
q = input("\n> ").strip()
if q == "/reset":
text = f"<|system|>{fmt.system_text()}"
continue
text += f"<|user|>{q}<|assistant|>"
ids = tok.encode(text, add_special_tokens=False).ids
out = []
for t in dec.generate(ids, max_new=a.max_new, temp=a.temp, stop=stop):
if t in stop:
break
out.append(t)
print(tok.decode([t], skip_special_tokens=False), end="", flush=True)
reply = tok.decode(out, skip_special_tokens=False)
text += reply + "<|end|>"
print()
if __name__ == "__main__":
main()