# Têtu Code 2 — inference code (tetu-AI). License: Apache 2.0, see README.md. import argparse import collections import hashlib import ntpath import os import posixpath from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import re import threading import time import uuid from tetu_code import fmt def project_instructions(system): out = [] for m in re.finditer('Instructions from: ([^\\n]+)\\n(.*?)(?=\\nInstructions from: |\\n|\\Z)', system, re.S): out.append(f'Instructions ({m.group(1).strip()}) :\n{m.group(2).strip()[:4000]}') return '\n\n'.join(out) def split_think(s): if not s.startswith('<|think|>'): return ('', s) inner, sep, rest = s[len('<|think|>'):].partition('<|/think|>') return (inner, rest if sep else '') def last_call(body): for m in reversed(body.get('messages', [])): if m.get('role') == 'assistant' and m.get('tool_calls'): f = m['tool_calls'][-1].get('function', {}) try: return (f.get('name'), json.loads(f.get('arguments') or '{}')) except ValueError: return None return None def degenerate(toks, max_period=256): n = len(toks) for p in range(1, max_period + 1): L = max(4 * p, 160) if L > n: break if toks[-1] == toks[-1 - p] and toks[n - L + p:] == toks[n - L:n - p]: return n - L + p return 0 def recent_calls(body, k=4): out = [] for m in reversed(body.get('messages', [])): for c in reversed(m.get('tool_calls') or []) if m.get('role') == 'assistant' else []: f = c.get('function', {}) try: out.append((f.get('name'), json.loads(f.get('arguments') or '{}'))) except ValueError: pass if len(out) >= k: return out return out def core_command(cmd): c = re.sub('\\s*(\\|\\||;|&&)\\s*(true|echo\\b[^|;&]*|:)\\s*', ' ', cmd or '') c = re.sub('\\s*\\d?>&\\d|\\s*2>\\s*/dev/null', ' ', c) return ' '.join(c.split()) CODE_START = re.compile('\\s*(?:from \\w[\\w.]* import|import \\w|def \\w|class \\w|#!/usr/bin/env python|if __name__)') def misplaced(call, body): if call is None or call[0] not in ('write', 'edit'): return False path = str(call[1].get('filePath') or '') text = str(call[1].get('content') if call[0] == 'write' else call[1].get('newString') or '') if not re.search('\\.(md|txt|rst)$', path, re.I) or not CODE_START.match(text): return False users = ' '.join((fmt.content_text(m.get('content')) for m in body.get('messages', []) if m.get('role') == 'user')) return re.split('[\\\\/]', path)[-1].lower() not in users.lower() def workdir(body): msgs = body.get('messages', []) return fmt.env_from_system('\n'.join((fmt.content_text(m.get('content')) for m in msgs if m.get('role') == 'system'))).get('Working directory', '') def rebase(call, body): wd = workdir(body) if call is None or not wd: return call pm = ntpath if re.match('[A-Za-z]:[\\\\/]', wd) else posixpath root = pm.normpath(wd) name, args = call new = dict(args) for k in ('filePath', 'path'): p = args.get(k) if not isinstance(p, str) or not pm.isabs(p): continue q = pm.normpath(p) if pm.normcase(q) == pm.normcase(root) or pm.normcase(q).startswith(pm.normcase(root) + pm.sep): continue anc = root for _ in range(2): if pm.dirname(anc) == anc: break anc = pm.dirname(anc) if pm.normcase(q) == pm.normcase(anc): new[k] = wd break if pm.normcase(q).startswith(pm.normcase(anc).rstrip(pm.sep) + pm.sep): if not os.path.exists(p): new[k] = pm.join(root, q[len(anc.rstrip(pm.sep)) + 1:]) break return call if new == args else (name, new) def missing_file(call, body): if call is None or call[0] not in ('read', 'edit'): return False path = str(call[1].get('filePath') or '') wd = workdir(body) if not path or not wd or (not os.path.isdir(wd)): return False return not os.path.exists(path if os.path.isabs(path) else os.path.join(wd, path)) def looping(call, prev, recent=()): if call is None: return False name, args = call if name == 'edit' and args.get('oldString') == args.get('newString'): return True if prev is not None and (name, args) == tuple(prev): return True if name == 'bash': core = core_command(args.get('command')) return sum((n == 'bash' and core_command(a.get('command')) == core for n, a in recent)) >= 2 return False TASK_HINT = re.compile('(?i)\\b(?:cr[ée]{1,2}r?|cr[ée]{1,2}s|[ée]cri[srt]?|[ée]crire|fai[st]|fais-moi|ajout\\w*|modifi\\w*|corrig\\w*|renomm\\w*|lance\\w*|teste[rz]?|code[rz]?|programm\\w*|g[ée]n[èe]re\\w*|supprim\\w*|enl[èe]v\\w*|chang\\w*|remplac\\w*|mets|met|install\\w*|r[ée]par\\w*|impl[ée]ment\\w*|transform\\w*|traduis|convertis|create|write|make|build|add|edit|fix|rename|run|implement|refactor|delete|remove|site|page|script|programme|appli\\w*|app|jeu|outil|todo|calculatrice|convertisseur|compteur|classe|fonction|fichier|projet|api|serveur|bot|formulaire|bouton|menu|readme|tests|html|css|python|javascript|js|flask)\\b|\\w\\.(?:py|js|ts|html|css|json|md|txt|sh|java|go|rs|c|cpp)\\b') DANGER = re.compile('(?i)bombes?\\b(?! de bain)|explosi|\\barmes?\\b|keylogger|espion|pirat|hack|phishing|fausse page de connexion|vol(?:er)? (?:des|les) (?:mots de passe|identifiants)|porn|sex|prompt syst|system prompt|ignore (?:tes|les) consignes') NO_TOOL_HINT = re.compile("(?i)^\\W*(?:explique|c'est quoi|qu'est[- ]ce|quel(?:le)?s?\\b|pourquoi|comment\\b|quelle est la diff|diff[ée]rence|r[ée]sume|d[ée]finis|que (?:veut|signifie)|est-ce que|peux-tu m'expliquer|tu peux m'expliquer)|\\b(?:donne|affiche|montre|r[ée]ponds|envoie)(?:-moi)? (?:uniquement |seulement |juste |directement )?(?:avec )?(?:le |du |que le )?code\\b|\\b(?:uniquement|seulement|juste) le code\\b|\\bsans (?:outils?|fichiers?|cr[ée]er de fichier)|\\bdans (?:ta|la) r[ée]ponse\\b|\\br[ée]ponds en\\b|\\ben \\d+ phrases?\\b|\\ben (?:une|deux|trois) phrases?\\b") FILE_NAME = re.compile('\\w\\.(?:py|js|ts|html|css|json|md|txt|sh|java|go|rs|c|cpp)\\b') def wants_tools(text): text = (text or '').strip() if not text or DANGER.search(text): return False if NO_TOOL_HINT.search(text) and (not FILE_NAME.search(text)): return False words = len(text.split()) return bool(TASK_HINT.search(text)) and (words >= 3 or bool(re.search('\\w\\.\\w{1,4}\\b', text))) def turn_calls(body): out = [] for m in body.get('messages', []): if m.get('role') == 'user': out = [] for c in m.get('tool_calls') or [] if m.get('role') == 'assistant' else []: f = c.get('function', {}) try: out.append((f.get('name'), json.loads(f.get('arguments') or '{}'))) except ValueError: out.append((f.get('name'), {})) return out CITED = re.compile('(?' not in text: return (text.strip(), None) before, _, call = text.partition('<|call|>') while call.startswith('<|call|>'): call = call[len('<|call|>'):] name, args = fmt.parse_call(call, tools) names = {t['function']['name'] for t in tools or []} if names and name not in names: vals = ' '.join((str(v) for v in args.values())) return (' '.join((x for x in (before.strip(), name, vals) if x)).strip(), None) return (before.strip(), (name, args)) def is_title_request(body): msgs = body.get('messages', []) sys_ = fmt.content_text(msgs[0].get('content')) if msgs and msgs[0].get('role') == 'system' else '' return not body.get('tools') and 'title' in sys_.lower() and ('generate' in sys_.lower()) def title_of(body): users = [fmt.content_text(m.get('content')) for m in body.get('messages', []) if m.get('role') == 'user'] text = users[-1] if users else '' text = re.sub('^\\s*Generate a title for this conversation:?\\s*', '', text, flags=re.I).strip().strip('"') words = re.sub('[@`*#>]', '', text).split() t = ' '.join(words[:6]) or 'Nouvelle session' return t[:1].upper() + t[1:] class Engine: def __init__(self, a): import torch from tetu_code.generate import Decoder, load dev = a.device if a.device != 'cuda' or torch.cuda.is_available() else 'cpu' self.model, self.tok = load(a.model, device=dev) memo = None if getattr(a, 'memo', ''): import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lab')) from tc_cles_exact import load_memo memo = load_memo(a.memo, dev) self.dec = Decoder(self.model, self.tok, max_len=a.max_len, graph=dev == 'cuda', compile=a.compile and dev == 'cuda', memo=memo) self.lock = threading.Lock() self.a = a self.stop_ids = {self.tok.token_to_id(t) for t in ('<|end|>', '<|/call|>', '<|eos|>')} self.call_id = self.tok.token_to_id('<|call|>') self.thoughts = collections.OrderedDict() d = os.environ.get('TETU_REP_LOG') or ('rep-log' if os.path.isdir('rep-log') else '') self.rep_log = open(os.path.join(d, f'serve-{a.port}.jsonl'), 'a', encoding='utf-8') if d else None def prompt_ids(self, body): msgs = body.get('messages', []) system = '\n'.join((fmt.content_text(m.get('content')) for m in msgs if m.get('role') == 'system')) env = fmt.env_from_system(system) sys_text = fmt.system_text(env) extra = project_instructions(system) if extra: sys_text += '\n\n' + extra tools = fmt.keep_tools(body.get('tools')) msgs = [self.with_thought(m) for m in msgs if m.get('role') != 'system'] text = fmt.text_of(fmt.render(msgs, tools, system=sys_text)) ids = self.tok.encode(text + '<|assistant|>', add_special_tokens=False).ids budget = self.a.max_len - self.a.max_new - 16 if len(ids) > budget: head = ids[:2048] ids = head + ids[-(budget - len(head)):] return (ids, tools) @staticmethod def thought_key(m): calls = m.get('tool_calls') or [] if calls and calls[0].get('id'): return calls[0]['id'] return 'txt:' + hashlib.md5(fmt.content_text(m.get('content')).strip().encode()).hexdigest() def with_thought(self, m): if m.get('role') != 'assistant' or m.get('reasoning') or m.get('reasoning_content'): return m t = self.thoughts.get(self.thought_key(m)) return dict(m, reasoning=t) if t else m def remember(self, key, think): if think: self.thoughts[key] = think while len(self.thoughts) > 5000: self.thoughts.popitem(last=False) def generate(self, body, on_text=None, on_think=None, info=None): tools_m, back = fmt.dialect(body.get('tools')) body = dict(body, tools=tools_m, messages=fmt.to_model(body.get('messages', []), back)) calls = sum((len(m.get('tool_calls') or []) for m in body.get('messages', []) if m.get('role') == 'assistant')) if self.a.max_calls and calls >= self.a.max_calls: text = "Je m'arrête ici : trop d'étapes pour cette demande. Dis-moi comment continuer." if on_text: on_text(text) return (text, None, 0, '') ids, tools = self.prompt_ids(body) if info is not None: info['prompt'] = len(ids) force = body.get('tool_choice') == 'required' or isinstance(body.get('tool_choice'), dict) mode = getattr(self.a, 'force', 'smart') if force and mode == 'smart': force = wants_tools(last_user(body)) elif mode == 'never': force = False temp = float(body.get('temperature', self.a.temp) if self.a.respect_temp else self.a.temp) max_new = min(int(body.get('max_tokens') or self.a.max_new), self.a.max_new) prev, recent, total = (last_call(body), recent_calls(body), 0) hold = bool(getattr(self.a, 'garde', 1)) and bool(turn_calls(body)) with self.lock: for k in range(1 + self.a.retries): t_k = temp if k == 0 else max(temp, (0.7, 1.0)[min(k, 2) - 1]) text, call, n, rep, think = self._sample(ids, tools, max_new, t_k, k, on_text if k == 0 and (not hold) else None, on_think if k == 0 else None, force=force) fixed = rebase(call, body) if fixed is not call and self.a.verbose: print(f' path moved back into the project: {call[1]} → {fixed[1]}'[:300], flush=True) call = fixed total += n if info is not None: info['last'] = n lie = dishonest(text, body) if hold and call is None and (k < self.a.retries) else '' if not lie and (not rep) and (not looping(call, prev, recent)) and (not (k < self.a.retries and (misplaced(call, body) or missing_file(call, body)))): break if self.a.verbose: why = f'anti-mensonge : {lie}' if lie else 'anti-boucle : ' + ('répétition en boucle' if rep else f'{call[0]} inutile') print(f' {why}, resampling (temperature {t_k:.1f})', flush=True) if hold and on_text and text: on_text(text) return (text, fmt.to_client(call, back), total, think) def _sample(self, ids, tools, max_new, temp, seed, on_text, on_think=None, force=False): out, shown, shown_th, in_call, n, rep = ([], '', '', False, 0, False) if force: ids, out, in_call = (list(ids) + [self.call_id], [self.call_id], True) for t in self.dec.generate(ids, max_new=max_new, temp=temp, top_p=self.a.top_p, stop=self.stop_ids, seed=seed): n += 1 if t in self.stop_ids: break out.append(t) if t == self.call_id: in_call = True if n % 32 == 0: keep = degenerate(out) if keep: if self.rep_log: self.rep_log.write(json.dumps({'ids': list(ids), 'gen': list(out), 'keep': keep, 'temp': temp}) + '\n') self.rep_log.flush() del out[keep:] rep = True break if (on_text or on_think) and (not in_call): s = self.tok.decode(out, skip_special_tokens=False) if not s.endswith('�'): th, vis = split_think(s) if on_think and len(th) > len(shown_th): on_think(th[len(shown_th):]) shown_th = th if on_text and len(vis) > len(shown): on_text(vis[len(shown):]) shown = vis think, text = split_think(self.tok.decode(out, skip_special_tokens=False)) text, call = finish_call(text, tools) return (text, call, n, rep, think.strip()) def make_handler(eng, a): class H(BaseHTTPRequestHandler): protocol_version = 'HTTP/1.1' def log_message(self, *args): pass def _json(self, code, obj): b = json.dumps(obj).encode() self.send_response(code) self.send_header('Content-Type', 'application/json') self.send_header('Content-Length', str(len(b))) self.end_headers() self.wfile.write(b) def do_GET(self): if self.path.rstrip('/').endswith('/models'): self._json(200, {'object': 'list', 'data': [{'id': a.name, 'object': 'model', 'owned_by': 'tetu', 'context_length': a.max_len}]}) else: self._json(404, {'error': 'not found'}) def do_POST(self): n = int(self.headers.get('Content-Length', 0)) body = json.loads(self.rfile.read(n) or b'{}') if not self.path.rstrip('/').endswith('/chat/completions'): self._json(404, {'error': 'not found'}) return cid, created = (f'chatcmpl-{uuid.uuid4().hex[:12]}', int(time.time())) base = {'id': cid, 'created': created, 'model': a.name} stream = bool(body.get('stream')) t0, info = (time.time(), {}) if is_title_request(body): text, call, ntok, think = (title_of(body), None, 0, '') chunks = None elif stream: self.send_response(200) self.send_header('Content-Type', 'text/event-stream') self.send_header('Cache-Control', 'no-cache') self.end_headers() def send(obj): self.wfile.write(f'data: {json.dumps(obj)}\n\n'.encode()) self.wfile.flush() send(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {'role': 'assistant', 'content': ''}, 'finish_reason': None}]}) text, call, ntok, think = eng.generate(body, on_text=lambda s: send(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {'content': s}, 'finish_reason': None}]}), on_think=lambda s: send(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {'reasoning_content': s}, 'finish_reason': None}]}), info=info) chunks = send else: text, call, ntok, think = eng.generate(body, info=info) chunks = None prompt, last = (info.get('prompt', 0), info.get('last', ntok)) usage = {'prompt_tokens': prompt, 'completion_tokens': last, 'total_tokens': prompt + last, 'context_window': a.max_len} tool_calls = None if call: tool_calls = [{'index': 0, 'id': f'call_{uuid.uuid4().hex[:10]}', 'type': 'function', 'function': {'name': call[0], 'arguments': json.dumps(call[1], ensure_ascii=False)}}] eng.remember(tool_calls[0]['id'] if tool_calls else 'txt:' + hashlib.md5(text.strip().encode()).hexdigest(), think) fin = 'tool_calls' if call else 'stop' if a.verbose: print(f"[{time.strftime('%H:%M:%S')}] {ntok} tokens in {time.time() - t0:.1f} s · {('call ' + call[0] if call else 'answer')}", flush=True) if chunks is not None: if tool_calls: chunks(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {'tool_calls': tool_calls}, 'finish_reason': None}]}) chunks(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {}, 'finish_reason': fin}], 'usage': usage}) self.wfile.write(b'data: [DONE]\n\n') self.wfile.flush() self.close_connection = True return msg = {'role': 'assistant', 'content': text if not call or text else None} if think: msg['reasoning_content'] = think if tool_calls: msg['tool_calls'] = [{k: v for k, v in tc.items() if k != 'index'} for tc in tool_calls] if stream: self.send_response(200) self.send_header('Content-Type', 'text/event-stream') self.end_headers() for c in ({'role': 'assistant', 'content': text}, {}): self.wfile.write(f"data: {json.dumps(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': c, 'finish_reason': None if c else 'stop'}]})}\n\n".encode()) self.wfile.write(b'data: [DONE]\n\n') self.close_connection = True return self._json(200, base | {'object': 'chat.completion', 'usage': usage, 'choices': [{'index': 0, 'message': msg, 'finish_reason': fin}]}) return H def main(): p = argparse.ArgumentParser() p.add_argument('--model', required=True) p.add_argument('--name', default='tetu-code') p.add_argument('--host', default='127.0.0.1') p.add_argument('--port', type=int, default=8070) p.add_argument('--device', default='cuda') p.add_argument('--max_len', type=int, default=65536) p.add_argument('--max_new', type=int, default=4096) p.add_argument('--temp', type=float, default=0.2) p.add_argument('--top_p', type=float, default=0.95) p.add_argument('--respect_temp', action='store_true', help='use the temperature sent by the client') p.add_argument('--max_calls', type=int, default=40, help='max tool calls per conversation (0 = no limit)') p.add_argument('--compile', action='store_true', help='torch.compile the decoding step (+40 %% speed, 2-8 min at startup)') p.add_argument('--garde', type=int, default=1, help='1: check the final answer (cited file missing, "tested" without a command) and redraw if needed') p.add_argument('--retries', type=int, default=2, help='resamples when a call loops (0 = never)') p.add_argument('--force', choices=('smart', 'client', 'never'), default='smart', help='tool_choice "required": smart = only for requests that look like a task') p.add_argument('--verbose', action='store_true') p.add_argument('--memo', default='', help='optional add-on, not included in this release') a = p.parse_args() eng = Engine(a) ThreadingHTTPServer.daemon_threads = True srv = ThreadingHTTPServer((a.host, a.port), make_handler(eng, a)) print(f'Têtu Code ready: http://{a.host}:{a.port}/v1 (model "{a.name}")', flush=True) srv.serve_forever() if __name__ == '__main__': main()