tetu-AI's picture
Initial release
2309cb1 verified
Raw History Blame Contribute Delete
11.6 kB
# Têtu Code 2 — inference code (tetu-AI). License: Apache 2.0, see README.md.
import json
import re
SYSTEM = "Tu es Têtu Code, un agent de programmation qui travaille dans le terminal de l'utilisateur. Sers-toi des outils pour lire, chercher, modifier et tester le code, un appel à la fois. Lis un fichier avant de le modifier ; après une modification, vérifie (tests, exécution) quand c'est possible. Réponds dans la langue de l'utilisateur, brièvement et précisément."
ENV_KEYS = ('Working directory', 'Is directory a git repo', 'Platform', "Today's date")
TOOL_DESC = {'bash': 'run a shell command', 'read': 'read a file or list a directory', 'edit': 'replace an exact passage of a file (oldString must match exactly once, or set replaceAll)', 'write': 'write a whole file', 'glob': 'find files by name pattern', 'grep': 'search file contents with a regex', 'todowrite': 'update the task list', 'webfetch': 'fetch a web page'}
DROP = {'task', 'skill', 'question', 'todowrite', 'webfetch', 'subagent', 'websearch', 'execute'}
TOOL_ALIASES = {'shell': 'bash'}
ARG_ALIASES = {'read': {'path': 'filePath'}, 'write': {'path': 'filePath'}, 'edit': {'path': 'filePath'}}
OPENCODE_TOOLS = [('bash', [('command', 'string', True), ('timeout', 'integer', False), ('workdir', 'string', False)]), ('edit', [('filePath', 'string', True), ('oldString', 'string', True), ('newString', 'string', True), ('replaceAll', 'boolean', False)]), ('glob', [('pattern', 'string', True), ('path', 'string', False)]), ('grep', [('pattern', 'string', True), ('path', 'string', False), ('include', 'string', False)]), ('read', [('filePath', 'string', True), ('offset', 'integer', False), ('limit', 'integer', False)]), ('todowrite', [('todos', 'array', True)]), ('webfetch', [('url', 'string', True), ('format', 'string', False), ('timeout', 'number', False)]), ('write', [('content', 'string', True), ('filePath', 'string', True)])]
MAX_RESULT = 12000
def opencode_tools():
out = []
for name, params in OPENCODE_TOOLS:
props = {p: {'type': t} for p, t, _ in params}
out.append({'type': 'function', 'function': {'name': name, 'description': TOOL_DESC[name], 'parameters': {'type': 'object', 'properties': props, 'required': [p for p, _, r in params if r]}}})
return out
def keep_tools(tools):
return [t for t in tools or [] if t.get('function', {}).get('name') not in DROP]
def dialect(tools):
out, back = ([], {})
for t in tools or []:
f = t.get('function', {})
cname = f.get('name')
name = TOOL_ALIASES.get(cname, cname)
ps = f.get('parameters') or {}
props = ps.get('properties') or {}
ren = {p: m for p, m in ARG_ALIASES.get(name, {}).items() if p in props and m not in props}
if name != cname or ren:
ps = dict(ps, properties={ren.get(p, p): s for p, s in props.items()}, required=[ren.get(p, p) for p in ps.get('required', [])])
t = dict(t, function=dict(f, name=name, parameters=ps))
back[name] = (cname, {m: p for p, m in ren.items()})
out.append(t)
order = [n for n, _ in OPENCODE_TOOLS]
out.sort(key=lambda t: order.index(t['function']['name']) if t['function']['name'] in order else len(order))
return (out, back)
def to_model(messages, back):
fwd = {cname: (name, {c: m for m, c in args.items()}) for name, (cname, args) in back.items()}
out = []
for m in messages:
if m.get('role') == 'assistant' and m.get('tool_calls'):
calls = []
for c in m['tool_calls']:
f = c.get('function', {})
name, ren = fwd.get(f.get('name'), (f.get('name'), {}))
f = dict(f, name=name)
if ren:
try:
a = json.loads(f.get('arguments') or '{}')
except ValueError:
a = None
if isinstance(a, dict):
f['arguments'] = json.dumps({ren.get(k, k): v for k, v in a.items()}, ensure_ascii=False)
calls.append(dict(c, function=f))
m = dict(m, tool_calls=calls)
out.append(m)
return out
def to_client(call, back):
if call is None:
return None
name, args = call
cname, ren = back.get(name, (name, {}))
return (cname, {ren.get(k, k): v for k, v in args.items()})
def tools_text(tools):
lines = []
for t in keep_tools(tools):
f = t['function']
ps = f.get('parameters', {}) or {}
req = set(ps.get('required', []))
args = ', '.join((p + ('' if p in req else '?') for p in ps.get('properties') or {}))
desc = TOOL_DESC.get(f['name']) or re.split('(?<=[.!?])\\s|\\n', (f.get('description') or '').strip())[0][:120]
lines.append(f"{f['name']}({args}) — {desc}")
return '\n'.join(lines)
def env_from_system(text):
m = re.search('<env>(.*?)</env>', text or '', re.S)
env = {}
for line in (m.group(1) if m else '').splitlines():
k, _, v = line.strip().partition(':')
if k in ENV_KEYS:
env[k] = v.strip()
return env
def system_text(env=None):
env = env or {}
lines = [f'{k}: {env[k]}' for k in ENV_KEYS if env.get(k)]
return SYSTEM + ('\n<env>\n' + '\n'.join(lines) + '\n</env>' if lines else '')
def arg_text(v):
if isinstance(v, str):
return v
if isinstance(v, bool):
return 'true' if v else 'false'
if isinstance(v, (int, float)):
return str(int(v)) if float(v).is_integer() else str(v)
return json.dumps(v, ensure_ascii=False)
def call_text(name, args):
return f'<|call|>{name}\n' + ''.join((f'<|arg|>{k}\n{arg_text(v)}\n' for k, v in args.items())) + '<|/call|>'
def parse_call(text, tools=None):
name, _, rest = text.partition('<|arg|>')
name, args = (name.strip(), {})
for part in ('<|arg|>' + rest).split('<|arg|>')[1:]:
k, _, v = part.partition('\n')
args[k.strip()] = v[:-1] if v.endswith('\n') else v
types = {}
for t in tools or []:
if t['function']['name'] == name:
types = {p: s.get('type') for p, s in (t['function'].get('parameters', {}).get('properties') or {}).items()}
for k, v in list(args.items()):
ty = types.get(k)
try:
if ty in ('number', 'integer'):
args[k] = int(v) if re.fullmatch('\\s*-?\\d+\\s*', v) else float(v)
elif ty == 'boolean':
args[k] = v.strip().lower() == 'true'
elif ty in ('array', 'object'):
args[k] = json.loads(v)
except ValueError:
pass
return (name, args)
def clip(text, limit=MAX_RESULT):
if len(text) <= limit:
return text
head, tail = (text[:limit * 2 // 3], text[-limit // 3:])
head, tail = (head[:head.rfind('\n') + 1] or head, tail[tail.find('\n') + 1:] or tail)
cut = text.count('\n', len(head), len(text) - len(tail))
return f'{head}… ({cut} lines cut) …\n{tail}'
def compact_read(text):
m = re.match('\\s*<path>(.*?)</path>\\s*<type>(.*?)</type>\\s*<content>\\n?(.*?)\\n?</content>\\s*$', text, re.S)
if not m:
return compact_read2(text)
path, kind, body = m.groups()
note = ''
n = re.search('\\n*\\((End of file - total (\\d+) lines|[^()\\n]*(?:offset|lines)[^()\\n]*)\\)\\s*$', body)
if n:
body, note = (body[:n.start()], n.group(1))
lines = body.split('\n')
nums = [re.match('(\\d+): ?', l) for l in lines]
if kind.strip() == 'file' and lines and all(nums):
first, last = (int(nums[0].group(1)), int(nums[-1].group(1)))
body = '\n'.join((l[k.end():] for l, k in zip(lines, nums)))
total = n.group(2) if n and n.group(2) else (re.search('of (\\d+)', note) or [None, None])[1]
if n and n.group(2) and (first == 1):
head = f'{path} (end of file, {total} lines)'
else:
head = f'{path} (lines {first}-{last}' + (f' of {total}' if total else '') + ')'
return head + '\n' + clip(body)
return f'{path} ({kind.strip()})\n' + clip(body) + (f'\n({note})' if note else '')
def compact_read2(text):
m = re.match('Read (file|directory) (.+?), (?:lines|entries) (\\d+)-(\\d+)(?: of (\\d+))?\\r?\\n?(.*)\\Z', text, re.S)
if not m:
return text
kind, path, first, last, total, body = m.groups()
lines = body.replace('\r\n', '\n').rstrip('\r').split('\n')
if kind == 'directory':
entries = [l[:-1] + '/' if l.endswith('\\') else l for l in lines if l]
return f'{path} (directory)\n' + clip('\n'.join(entries)) + f'\n({len(entries)} entries)'
nums = [re.match('(\\d+): ?', l) for l in lines]
if not all(nums):
return text
body = '\n'.join((l[k.end():] for l, k in zip(lines, nums)))
first, last = (int(first), int(last))
if first == 1 and (total is None and last - first + 1 < 2000 or (total and last >= int(total))):
return f'{path} (end of file, {last} lines)\n' + clip(body)
return f'{path} (lines {first}-{last}' + (f' of {total}' if total else '') + ')\n' + clip(body)
def read_result(path, content, first=1, total=None):
n = content.count('\n') + 1 if content else 0
if first == 1 and (total is None or n >= total):
return f'{path} (end of file, {n} lines)\n' + clip(content)
return f'{path} (lines {first}-{first + n - 1}' + (f' of {total}' if total else '') + ')\n' + clip(content)
def compact_result(name, text):
text = text if isinstance(text, str) else json.dumps(text, ensure_ascii=False)
if text.startswith('{"error"'):
try:
text = 'Error: ' + json.loads(text)['error']['message']
except (ValueError, KeyError, TypeError):
pass
return compact_read(text) if name == 'read' else clip(text)
def content_text(c):
if isinstance(c, list):
return ''.join((x.get('text', '') for x in c if isinstance(x, dict)))
return c or ''
def think_text(m):
t = (m.get('reasoning') or m.get('reasoning_content') or '').strip()
return f'<|think|>{t}<|/think|>' if t else ''
def render(messages, tools=None, env=None, system=None):
out = [(f'<|system|>{(system if system is not None else system_text(env))}', False)]
if tools:
out.append((f'<|tools|>{tools_text(tools)}', False))
results = {m.get('tool_call_id'): m for m in messages if m['role'] == 'tool'}
for m in messages:
role = m['role']
if role == 'system':
continue
if role == 'user':
out.append((f"<|user|>{content_text(m.get('content'))}", False))
elif role == 'assistant':
text, calls = (content_text(m.get('content')).strip(), m.get('tool_calls') or [])
th = think_text(m)
learn = m.get('learn', True)
if not calls:
out += [('<|assistant|>', False), (f'{th}{text}<|end|>', learn)]
continue
for j, c in enumerate(calls):
f = c['function']
args = f.get('arguments') or '{}'
args = json.loads(args) if isinstance(args, str) else args
out += [('<|assistant|>', False), ((th + text + ('\n' if text else '') if j == 0 else '') + call_text(f['name'], args), learn)]
r = results.get(c.get('id'))
out.append((f"<|tool|>{compact_result(f['name'], content_text(r.get('content')) if r else '(no result)')}", False))
return out
def text_of(segments):
return ''.join((t for t, _ in segments))