Tetu-Code-2 / tetu_code /serve.py
tetu-AI's picture
Initial release
2309cb1 verified
Raw History Blame Contribute Delete
23.7 kB
# Têtu Code 2 — inference code (tetu-AI). License: Apache 2.0, see README.md.
import argparse
import collections
import hashlib
import ntpath
import os
import posixpath
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import re
import threading
import time
import uuid
from tetu_code import fmt
def project_instructions(system):
out = []
for m in re.finditer('Instructions from: ([^\\n]+)\\n(.*?)(?=\\nInstructions from: |\\n<available_skills>|\\Z)', system, re.S):
out.append(f'Instructions ({m.group(1).strip()}) :\n{m.group(2).strip()[:4000]}')
return '\n\n'.join(out)
def split_think(s):
if not s.startswith('<|think|>'):
return ('', s)
inner, sep, rest = s[len('<|think|>'):].partition('<|/think|>')
return (inner, rest if sep else '')
def last_call(body):
for m in reversed(body.get('messages', [])):
if m.get('role') == 'assistant' and m.get('tool_calls'):
f = m['tool_calls'][-1].get('function', {})
try:
return (f.get('name'), json.loads(f.get('arguments') or '{}'))
except ValueError:
return None
return None
def degenerate(toks, max_period=256):
n = len(toks)
for p in range(1, max_period + 1):
L = max(4 * p, 160)
if L > n:
break
if toks[-1] == toks[-1 - p] and toks[n - L + p:] == toks[n - L:n - p]:
return n - L + p
return 0
def recent_calls(body, k=4):
out = []
for m in reversed(body.get('messages', [])):
for c in reversed(m.get('tool_calls') or []) if m.get('role') == 'assistant' else []:
f = c.get('function', {})
try:
out.append((f.get('name'), json.loads(f.get('arguments') or '{}')))
except ValueError:
pass
if len(out) >= k:
return out
return out
def core_command(cmd):
c = re.sub('\\s*(\\|\\||;|&&)\\s*(true|echo\\b[^|;&]*|:)\\s*', ' ', cmd or '')
c = re.sub('\\s*\\d?>&\\d|\\s*2>\\s*/dev/null', ' ', c)
return ' '.join(c.split())
CODE_START = re.compile('\\s*(?:from \\w[\\w.]* import|import \\w|def \\w|class \\w|#!/usr/bin/env python|if __name__)')
def misplaced(call, body):
if call is None or call[0] not in ('write', 'edit'):
return False
path = str(call[1].get('filePath') or '')
text = str(call[1].get('content') if call[0] == 'write' else call[1].get('newString') or '')
if not re.search('\\.(md|txt|rst)$', path, re.I) or not CODE_START.match(text):
return False
users = ' '.join((fmt.content_text(m.get('content')) for m in body.get('messages', []) if m.get('role') == 'user'))
return re.split('[\\\\/]', path)[-1].lower() not in users.lower()
def workdir(body):
msgs = body.get('messages', [])
return fmt.env_from_system('\n'.join((fmt.content_text(m.get('content')) for m in msgs if m.get('role') == 'system'))).get('Working directory', '')
def rebase(call, body):
wd = workdir(body)
if call is None or not wd:
return call
pm = ntpath if re.match('[A-Za-z]:[\\\\/]', wd) else posixpath
root = pm.normpath(wd)
name, args = call
new = dict(args)
for k in ('filePath', 'path'):
p = args.get(k)
if not isinstance(p, str) or not pm.isabs(p):
continue
q = pm.normpath(p)
if pm.normcase(q) == pm.normcase(root) or pm.normcase(q).startswith(pm.normcase(root) + pm.sep):
continue
anc = root
for _ in range(2):
if pm.dirname(anc) == anc:
break
anc = pm.dirname(anc)
if pm.normcase(q) == pm.normcase(anc):
new[k] = wd
break
if pm.normcase(q).startswith(pm.normcase(anc).rstrip(pm.sep) + pm.sep):
if not os.path.exists(p):
new[k] = pm.join(root, q[len(anc.rstrip(pm.sep)) + 1:])
break
return call if new == args else (name, new)
def missing_file(call, body):
if call is None or call[0] not in ('read', 'edit'):
return False
path = str(call[1].get('filePath') or '')
wd = workdir(body)
if not path or not wd or (not os.path.isdir(wd)):
return False
return not os.path.exists(path if os.path.isabs(path) else os.path.join(wd, path))
def looping(call, prev, recent=()):
if call is None:
return False
name, args = call
if name == 'edit' and args.get('oldString') == args.get('newString'):
return True
if prev is not None and (name, args) == tuple(prev):
return True
if name == 'bash':
core = core_command(args.get('command'))
return sum((n == 'bash' and core_command(a.get('command')) == core for n, a in recent)) >= 2
return False
TASK_HINT = re.compile('(?i)\\b(?:cr[ée]{1,2}r?|cr[ée]{1,2}s|[ée]cri[srt]?|[ée]crire|fai[st]|fais-moi|ajout\\w*|modifi\\w*|corrig\\w*|renomm\\w*|lance\\w*|teste[rz]?|code[rz]?|programm\\w*|g[ée]n[èe]re\\w*|supprim\\w*|enl[èe]v\\w*|chang\\w*|remplac\\w*|mets|met|install\\w*|r[ée]par\\w*|impl[ée]ment\\w*|transform\\w*|traduis|convertis|create|write|make|build|add|edit|fix|rename|run|implement|refactor|delete|remove|site|page|script|programme|appli\\w*|app|jeu|outil|todo|calculatrice|convertisseur|compteur|classe|fonction|fichier|projet|api|serveur|bot|formulaire|bouton|menu|readme|tests|html|css|python|javascript|js|flask)\\b|\\w\\.(?:py|js|ts|html|css|json|md|txt|sh|java|go|rs|c|cpp)\\b')
DANGER = re.compile('(?i)bombes?\\b(?! de bain)|explosi|\\barmes?\\b|keylogger|espion|pirat|hack|phishing|fausse page de connexion|vol(?:er)? (?:des|les) (?:mots de passe|identifiants)|porn|sex|prompt syst|system prompt|ignore (?:tes|les) consignes')
NO_TOOL_HINT = re.compile("(?i)^\\W*(?:explique|c'est quoi|qu'est[- ]ce|quel(?:le)?s?\\b|pourquoi|comment\\b|quelle est la diff|diff[ée]rence|r[ée]sume|d[ée]finis|que (?:veut|signifie)|est-ce que|peux-tu m'expliquer|tu peux m'expliquer)|\\b(?:donne|affiche|montre|r[ée]ponds|envoie)(?:-moi)? (?:uniquement |seulement |juste |directement )?(?:avec )?(?:le |du |que le )?code\\b|\\b(?:uniquement|seulement|juste) le code\\b|\\bsans (?:outils?|fichiers?|cr[ée]er de fichier)|\\bdans (?:ta|la) r[ée]ponse\\b|\\br[ée]ponds en\\b|\\ben \\d+ phrases?\\b|\\ben (?:une|deux|trois) phrases?\\b")
FILE_NAME = re.compile('\\w\\.(?:py|js|ts|html|css|json|md|txt|sh|java|go|rs|c|cpp)\\b')
def wants_tools(text):
text = (text or '').strip()
if not text or DANGER.search(text):
return False
if NO_TOOL_HINT.search(text) and (not FILE_NAME.search(text)):
return False
words = len(text.split())
return bool(TASK_HINT.search(text)) and (words >= 3 or bool(re.search('\\w\\.\\w{1,4}\\b', text)))
def turn_calls(body):
out = []
for m in body.get('messages', []):
if m.get('role') == 'user':
out = []
for c in m.get('tool_calls') or [] if m.get('role') == 'assistant' else []:
f = c.get('function', {})
try:
out.append((f.get('name'), json.loads(f.get('arguments') or '{}')))
except ValueError:
out.append((f.get('name'), {}))
return out
CITED = re.compile('(?<![\\w/\\\\.-])((?:[\\w-]+[/\\\\])*[\\w-]+\\.(?:py|js|ts|html|css|json|md|txt|sh|csv))\\b')
TESTED = re.compile("(?i)\\bj'ai (?:bien )?(?:test[ée]|v[ée]rifi[ée]|lanc[ée]|ex[ée]cut[ée])|\\btests? (?:passent|r[ée]ussi|pass(?:ed|ing)?\\b)|\\btests (?:sont|ont) (?:ok|verts|r[ée]ussi|pass[ée])|\\btest[ée]e?s? et (?:[çc]a|il|elle|tout) (?:marche|fonctionne)|\\bi (?:have )?(?:tested|ran|run|verified)\\b|\\ball tests pass")
def dishonest(text, body):
calls = turn_calls(body)
if not calls or not (text or '').strip():
return ''
if TESTED.search(text) and (not any((n == 'bash' for n, _ in calls))):
return 'dit avoir testé sans commande'
seen = []
for m in body.get('messages', []):
if m.get('role') != 'system':
seen.append(fmt.content_text(m.get('content')))
for c in m.get('tool_calls') or []:
seen.append(str(c.get('function', {}).get('arguments') or ''))
seen = '\n'.join(seen).lower().replace('\\\\', '/').replace('\\', '/')
wd = workdir(body)
local = bool(wd) and os.path.isdir(wd)
for name in CITED.findall(text):
if re.split('[/\\\\]', name)[-1].lower() in seen:
continue
if local and os.path.exists(os.path.join(wd, name)):
continue
return f'cite {name}, absent du projet'
return ''
def last_user(body):
msgs = body.get('messages', [])
return fmt.content_text(msgs[-1].get('content')) if msgs and msgs[-1].get('role') == 'user' else ''
def finish_call(text, tools):
if '<|call|>' not in text:
return (text.strip(), None)
before, _, call = text.partition('<|call|>')
while call.startswith('<|call|>'):
call = call[len('<|call|>'):]
name, args = fmt.parse_call(call, tools)
names = {t['function']['name'] for t in tools or []}
if names and name not in names:
vals = ' '.join((str(v) for v in args.values()))
return (' '.join((x for x in (before.strip(), name, vals) if x)).strip(), None)
return (before.strip(), (name, args))
def is_title_request(body):
msgs = body.get('messages', [])
sys_ = fmt.content_text(msgs[0].get('content')) if msgs and msgs[0].get('role') == 'system' else ''
return not body.get('tools') and 'title' in sys_.lower() and ('generate' in sys_.lower())
def title_of(body):
users = [fmt.content_text(m.get('content')) for m in body.get('messages', []) if m.get('role') == 'user']
text = users[-1] if users else ''
text = re.sub('^\\s*Generate a title for this conversation:?\\s*', '', text, flags=re.I).strip().strip('"')
words = re.sub('[@`*#>]', '', text).split()
t = ' '.join(words[:6]) or 'Nouvelle session'
return t[:1].upper() + t[1:]
class Engine:
def __init__(self, a):
import torch
from tetu_code.generate import Decoder, load
dev = a.device if a.device != 'cuda' or torch.cuda.is_available() else 'cpu'
self.model, self.tok = load(a.model, device=dev)
memo = None
if getattr(a, 'memo', ''):
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lab'))
from tc_cles_exact import load_memo
memo = load_memo(a.memo, dev)
self.dec = Decoder(self.model, self.tok, max_len=a.max_len, graph=dev == 'cuda', compile=a.compile and dev == 'cuda', memo=memo)
self.lock = threading.Lock()
self.a = a
self.stop_ids = {self.tok.token_to_id(t) for t in ('<|end|>', '<|/call|>', '<|eos|>')}
self.call_id = self.tok.token_to_id('<|call|>')
self.thoughts = collections.OrderedDict()
d = os.environ.get('TETU_REP_LOG') or ('rep-log' if os.path.isdir('rep-log') else '')
self.rep_log = open(os.path.join(d, f'serve-{a.port}.jsonl'), 'a', encoding='utf-8') if d else None
def prompt_ids(self, body):
msgs = body.get('messages', [])
system = '\n'.join((fmt.content_text(m.get('content')) for m in msgs if m.get('role') == 'system'))
env = fmt.env_from_system(system)
sys_text = fmt.system_text(env)
extra = project_instructions(system)
if extra:
sys_text += '\n\n' + extra
tools = fmt.keep_tools(body.get('tools'))
msgs = [self.with_thought(m) for m in msgs if m.get('role') != 'system']
text = fmt.text_of(fmt.render(msgs, tools, system=sys_text))
ids = self.tok.encode(text + '<|assistant|>', add_special_tokens=False).ids
budget = self.a.max_len - self.a.max_new - 16
if len(ids) > budget:
head = ids[:2048]
ids = head + ids[-(budget - len(head)):]
return (ids, tools)
@staticmethod
def thought_key(m):
calls = m.get('tool_calls') or []
if calls and calls[0].get('id'):
return calls[0]['id']
return 'txt:' + hashlib.md5(fmt.content_text(m.get('content')).strip().encode()).hexdigest()
def with_thought(self, m):
if m.get('role') != 'assistant' or m.get('reasoning') or m.get('reasoning_content'):
return m
t = self.thoughts.get(self.thought_key(m))
return dict(m, reasoning=t) if t else m
def remember(self, key, think):
if think:
self.thoughts[key] = think
while len(self.thoughts) > 5000:
self.thoughts.popitem(last=False)
def generate(self, body, on_text=None, on_think=None, info=None):
tools_m, back = fmt.dialect(body.get('tools'))
body = dict(body, tools=tools_m, messages=fmt.to_model(body.get('messages', []), back))
calls = sum((len(m.get('tool_calls') or []) for m in body.get('messages', []) if m.get('role') == 'assistant'))
if self.a.max_calls and calls >= self.a.max_calls:
text = "Je m'arrête ici : trop d'étapes pour cette demande. Dis-moi comment continuer."
if on_text:
on_text(text)
return (text, None, 0, '')
ids, tools = self.prompt_ids(body)
if info is not None:
info['prompt'] = len(ids)
force = body.get('tool_choice') == 'required' or isinstance(body.get('tool_choice'), dict)
mode = getattr(self.a, 'force', 'smart')
if force and mode == 'smart':
force = wants_tools(last_user(body))
elif mode == 'never':
force = False
temp = float(body.get('temperature', self.a.temp) if self.a.respect_temp else self.a.temp)
max_new = min(int(body.get('max_tokens') or self.a.max_new), self.a.max_new)
prev, recent, total = (last_call(body), recent_calls(body), 0)
hold = bool(getattr(self.a, 'garde', 1)) and bool(turn_calls(body))
with self.lock:
for k in range(1 + self.a.retries):
t_k = temp if k == 0 else max(temp, (0.7, 1.0)[min(k, 2) - 1])
text, call, n, rep, think = self._sample(ids, tools, max_new, t_k, k, on_text if k == 0 and (not hold) else None, on_think if k == 0 else None, force=force)
fixed = rebase(call, body)
if fixed is not call and self.a.verbose:
print(f' path moved back into the project: {call[1]} → {fixed[1]}'[:300], flush=True)
call = fixed
total += n
if info is not None:
info['last'] = n
lie = dishonest(text, body) if hold and call is None and (k < self.a.retries) else ''
if not lie and (not rep) and (not looping(call, prev, recent)) and (not (k < self.a.retries and (misplaced(call, body) or missing_file(call, body)))):
break
if self.a.verbose:
why = f'anti-mensonge : {lie}' if lie else 'anti-boucle : ' + ('répétition en boucle' if rep else f'{call[0]} inutile')
print(f' {why}, resampling (temperature {t_k:.1f})', flush=True)
if hold and on_text and text:
on_text(text)
return (text, fmt.to_client(call, back), total, think)
def _sample(self, ids, tools, max_new, temp, seed, on_text, on_think=None, force=False):
out, shown, shown_th, in_call, n, rep = ([], '', '', False, 0, False)
if force:
ids, out, in_call = (list(ids) + [self.call_id], [self.call_id], True)
for t in self.dec.generate(ids, max_new=max_new, temp=temp, top_p=self.a.top_p, stop=self.stop_ids, seed=seed):
n += 1
if t in self.stop_ids:
break
out.append(t)
if t == self.call_id:
in_call = True
if n % 32 == 0:
keep = degenerate(out)
if keep:
if self.rep_log:
self.rep_log.write(json.dumps({'ids': list(ids), 'gen': list(out), 'keep': keep, 'temp': temp}) + '\n')
self.rep_log.flush()
del out[keep:]
rep = True
break
if (on_text or on_think) and (not in_call):
s = self.tok.decode(out, skip_special_tokens=False)
if not s.endswith('�'):
th, vis = split_think(s)
if on_think and len(th) > len(shown_th):
on_think(th[len(shown_th):])
shown_th = th
if on_text and len(vis) > len(shown):
on_text(vis[len(shown):])
shown = vis
think, text = split_think(self.tok.decode(out, skip_special_tokens=False))
text, call = finish_call(text, tools)
return (text, call, n, rep, think.strip())
def make_handler(eng, a):
class H(BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.1'
def log_message(self, *args):
pass
def _json(self, code, obj):
b = json.dumps(obj).encode()
self.send_response(code)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(b)))
self.end_headers()
self.wfile.write(b)
def do_GET(self):
if self.path.rstrip('/').endswith('/models'):
self._json(200, {'object': 'list', 'data': [{'id': a.name, 'object': 'model', 'owned_by': 'tetu', 'context_length': a.max_len}]})
else:
self._json(404, {'error': 'not found'})
def do_POST(self):
n = int(self.headers.get('Content-Length', 0))
body = json.loads(self.rfile.read(n) or b'{}')
if not self.path.rstrip('/').endswith('/chat/completions'):
self._json(404, {'error': 'not found'})
return
cid, created = (f'chatcmpl-{uuid.uuid4().hex[:12]}', int(time.time()))
base = {'id': cid, 'created': created, 'model': a.name}
stream = bool(body.get('stream'))
t0, info = (time.time(), {})
if is_title_request(body):
text, call, ntok, think = (title_of(body), None, 0, '')
chunks = None
elif stream:
self.send_response(200)
self.send_header('Content-Type', 'text/event-stream')
self.send_header('Cache-Control', 'no-cache')
self.end_headers()
def send(obj):
self.wfile.write(f'data: {json.dumps(obj)}\n\n'.encode())
self.wfile.flush()
send(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {'role': 'assistant', 'content': ''}, 'finish_reason': None}]})
text, call, ntok, think = eng.generate(body, on_text=lambda s: send(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {'content': s}, 'finish_reason': None}]}), on_think=lambda s: send(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {'reasoning_content': s}, 'finish_reason': None}]}), info=info)
chunks = send
else:
text, call, ntok, think = eng.generate(body, info=info)
chunks = None
prompt, last = (info.get('prompt', 0), info.get('last', ntok))
usage = {'prompt_tokens': prompt, 'completion_tokens': last, 'total_tokens': prompt + last, 'context_window': a.max_len}
tool_calls = None
if call:
tool_calls = [{'index': 0, 'id': f'call_{uuid.uuid4().hex[:10]}', 'type': 'function', 'function': {'name': call[0], 'arguments': json.dumps(call[1], ensure_ascii=False)}}]
eng.remember(tool_calls[0]['id'] if tool_calls else 'txt:' + hashlib.md5(text.strip().encode()).hexdigest(), think)
fin = 'tool_calls' if call else 'stop'
if a.verbose:
print(f"[{time.strftime('%H:%M:%S')}] {ntok} tokens in {time.time() - t0:.1f} s · {('call ' + call[0] if call else 'answer')}", flush=True)
if chunks is not None:
if tool_calls:
chunks(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {'tool_calls': tool_calls}, 'finish_reason': None}]})
chunks(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {}, 'finish_reason': fin}], 'usage': usage})
self.wfile.write(b'data: [DONE]\n\n')
self.wfile.flush()
self.close_connection = True
return
msg = {'role': 'assistant', 'content': text if not call or text else None}
if think:
msg['reasoning_content'] = think
if tool_calls:
msg['tool_calls'] = [{k: v for k, v in tc.items() if k != 'index'} for tc in tool_calls]
if stream:
self.send_response(200)
self.send_header('Content-Type', 'text/event-stream')
self.end_headers()
for c in ({'role': 'assistant', 'content': text}, {}):
self.wfile.write(f"data: {json.dumps(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': c, 'finish_reason': None if c else 'stop'}]})}\n\n".encode())
self.wfile.write(b'data: [DONE]\n\n')
self.close_connection = True
return
self._json(200, base | {'object': 'chat.completion', 'usage': usage, 'choices': [{'index': 0, 'message': msg, 'finish_reason': fin}]})
return H
def main():
p = argparse.ArgumentParser()
p.add_argument('--model', required=True)
p.add_argument('--name', default='tetu-code')
p.add_argument('--host', default='127.0.0.1')
p.add_argument('--port', type=int, default=8070)
p.add_argument('--device', default='cuda')
p.add_argument('--max_len', type=int, default=65536)
p.add_argument('--max_new', type=int, default=4096)
p.add_argument('--temp', type=float, default=0.2)
p.add_argument('--top_p', type=float, default=0.95)
p.add_argument('--respect_temp', action='store_true', help='use the temperature sent by the client')
p.add_argument('--max_calls', type=int, default=40, help='max tool calls per conversation (0 = no limit)')
p.add_argument('--compile', action='store_true', help='torch.compile the decoding step (+40 %% speed, 2-8 min at startup)')
p.add_argument('--garde', type=int, default=1, help='1: check the final answer (cited file missing, "tested" without a command) and redraw if needed')
p.add_argument('--retries', type=int, default=2, help='resamples when a call loops (0 = never)')
p.add_argument('--force', choices=('smart', 'client', 'never'), default='smart', help='tool_choice "required": smart = only for requests that look like a task')
p.add_argument('--verbose', action='store_true')
p.add_argument('--memo', default='', help='optional add-on, not included in this release')
a = p.parse_args()
eng = Engine(a)
ThreadingHTTPServer.daemon_threads = True
srv = ThreadingHTTPServer((a.host, a.port), make_handler(eng, a))
print(f'Têtu Code ready: http://{a.host}:{a.port}/v1 (model "{a.name}")', flush=True)
srv.serve_forever()
if __name__ == '__main__':
main()