Spaces:
Running
Running
File size: 3,489 Bytes
3865888 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | """Local speech conversation loop (V6 phase_12).
microphone/file -> local STT -> local memory retrieval -> local LLM
(typed reply schema) -> local TTS -> wav. Every stage is real and local;
any missing model yields structured UNAVAILABLE, never synthetic dialogue.
"""
import json
import os
import time
from typing import Any, Dict, Optional
def transcribe_file(stt, wav_path: str) -> Dict[str, Any]:
import soundfile as sf
import torch
data, sr = sf.read(wav_path)
if data.ndim > 1:
data = data.mean(axis=1)
if sr != 16000:
from scipy.signal import resample_poly
import math
g = math.gcd(sr, 16000)
data = resample_poly(data, 16000 // g, sr // g).astype(data.dtype)
sr = 16000
feats = stt["processor"](data, sampling_rate=sr, return_tensors="pt").input_features
with torch.no_grad():
ids = stt["model"].generate(feats, max_new_tokens=64)
text = stt["processor"].batch_decode(ids, skip_special_tokens=True)[0].strip()
return {"status": "TRANSCRIBED", "text": text}
def synthesize_reply(tts_pipe, voice: str, text: str, out_path: str) -> Dict[str, Any]:
import soundfile as sf
import torch
chunks = []
for _, _, audio in tts_pipe(text[:400], voice=voice, speed=1.0):
chunks.append(audio)
if not chunks:
raise RuntimeError("TTS produced no audio")
wav = torch.cat(chunks, dim=0).numpy()
os.makedirs(os.path.dirname(os.path.abspath(out_path)), exist_ok=True)
sf.write(out_path, wav, 24000)
return {"status": "SYNTHESIZED", "wav": out_path, "seconds": round(len(wav) / 24000, 2)}
def llm_reply(text_model, heard: str, memories: list) -> Dict[str, Any]:
from src.prompts.render import load_templates, render, extract_json, check_output
templates = load_templates()
last_err = ""
for attempt in range(2):
t = render("conversation_reply", templates, heard=heard[:200],
memories=json.dumps(memories)[:400])
out = text_model.create_chat_completion(
messages=[{"role": "system", "content": t["system"]},
{"role": "user", "content": t["user"]}],
max_tokens=t["max_tokens"], temperature=t["temperature"], seed=21 + attempt)
text = out["choices"][0]["message"]["content"]
try:
data = check_output(extract_json(text, "reply"), t["schema"])
data.setdefault("action", "SPEAK")
return data
except ValueError as e:
last_err = str(e)
raise ValueError(f"LLM produced no reply JSON ({last_err})")
def conversation_turn(agent, stt, text_model, tts_pipe, voice: str,
wav_in: str, wav_out: str) -> Dict[str, Any]:
heard = transcribe_file(stt, wav_in)
mems = agent.spatial.search_places(heard["text"][:60], limit=3)
reply = llm_reply(text_model, heard["text"], mems)
synth = synthesize_reply(tts_pipe, voice, reply["reply"], wav_out)
agent.org.episodes.append({"tick": agent.world.tick, "organism_id": agent.org.id,
"observation": [f"heard:{heard['text'][:80]}"],
"action": "SPEAK", "reward": 0.0,
"goal": "converse", "reply": reply["reply"][:200]})
return {"heard": heard["text"], "reply": reply["reply"][:200],
"action": reply["action"], "wav": synth["wav"],
"seconds": synth["seconds"], "provenance": "LOCAL_STT_LLM_TTS"}
|