Upload folder using huggingface_hub

app.py

@@ -272,6 +272,80 @@ async def generate_patch(request: Request):
     }
 
 
+@app.post("/audio/generate")
+async def audio_generate(request: Request):
+    """Generate code from an audio transcript without requiring an existing session.
+    Creates a temporary session, injects the transcript as audio evidence,
+    runs observer → builder → receipt."""
+    body = {}
+    try:
+        body = await request.json()
+    except Exception:
+        pass
+    transcript = body.get("transcript", "") or request.query_params.get("transcript", "")
+    audio_file = body.get("audio_file", "") or request.query_params.get("audio_file", "")
+    mode = body.get("mode", "continuous_code") or request.query_params.get("mode", "continuous_code")
+    if not transcript:
+        return JSONResponse({"error": "transcript required"}, 400)
+
+    # Create a temporary session
+    state = SessionState()
+    state.mode = mode
+    sessions[state.session_id] = state
+
+    # Inject transcript as an audio chunk
+    chunk = process_transcript(transcript, state)
+    state.add_audio_chunk(chunk)
+
+    # Compress and run the full pipeline
+    compact = compress_state(state)
+    ev_result = evidence_gate("", compact)
+    log_gate_decision("evidence_gate", ev_result, patch_hash="")
+    observation = observe(state, compact)
+    state.observer_state = observation
+    state.last_observer_ts = time.time()
+    ev_result2 = evidence_gate(observation.get("observer_output", ""), compact)
+    log_gate_decision("evidence_gate", ev_result2, patch_hash="")
+    result = build(state, observation, gate_result=ev_result2)
+    state.current_artifact = result["patch_output"]
+    state.last_patch_hash = result["patch_hash"]
+    state.last_builder_ts = time.time()
+    state.receipt_count += 1
+    save_receipt(result["receipt"])
+
+    patch_output = result["patch_output"]
+    extracted_code = extract_code_from_output(patch_output)
+    if not extracted_code:
+        extracted_code = patch_output
+    artifact_data = {
+        "patch_hash": result["patch_hash"],
+        "code": extracted_code,
+        "observer_output": patch_output,
+        "timestamp": time.time(),
+        "session_id": state.session_id,
+        "fallback_level": result.get("fallback_level", 0),
+        "artifact_type": result.get("artifact_type", ""),
+        "sensory_channels": ev_result2.sensory_channels,
+        "feature_attribution": ev_result2.feature_attribution,
+        "audio_file": audio_file,
+        "transcript": transcript,
+    }
+    artifact_store[result["patch_hash"]] = artifact_data
+    save_artifact_disk(result["patch_hash"], artifact_data)
+
+    return {
+        **result,
+        "session_id": state.session_id,
+        "gate": {
+            "fallback_level": ev_result2.fallback_level,
+            "artifact_type": ev_result2.artifact_type,
+            "sensory_channels": ev_result2.sensory_channels,
+            "feature_attribution": ev_result2.feature_attribution,
+            "reason": ev_result2.reason,
+        },
+    }
+
+
 # In-memory artifact store: patch_hash -> {code, output, timestamp}
 artifact_store: dict[str, dict] = {}
 

frontend/index.html

@@ -1738,6 +1738,24 @@
             <div class="etl-stage-status" id="etlReceiptStatus">idle</div>
           </div>
         </div>
+        <!-- LIVE TRANSCRIPT DURING RECORDING -->
+        <div id="liveTranscript"
+          style="margin-top:12px;padding:10px 14px;background:var(--bg2);border:1px solid var(--border);border-radius:8px;font-family:var(--mono);font-size:12px;color:var(--text2);min-height:40px;display:none">
+          <div style="font-size:9px;color:var(--text3);text-transform:uppercase;letter-spacing:1px;margin-bottom:4px">
+            Live Transcript</div>
+          <div id="liveTranscriptText" style="line-height:1.6">Waiting for speech...</div>
+        </div>
+
+        <!-- RECORDED AUDIO LIST -->
+        <div id="audioListPanel" style="margin-top:12px">
+          <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:6px">
+            <div style="font-size:10px;color:var(--text3);text-transform:uppercase;letter-spacing:1px;font-weight:700">
+              Recorded Audio Files</div>
+            <button class="toolbar-btn" onclick="loadAudioList()"
+              style="font-size:10px;padding:2px 8px">Refresh</button>
+          </div>
+          <div id="audioList" style="display:flex;flex-direction:column;gap:4px"></div>
+        </div>
       </div>
 
       <!-- CONTROLS -->
@@ -1930,6 +1948,7 @@
       if (view) view.classList.add('active');
       const link = document.querySelector(`.nav-link[data-view="${name}"]`);
       if (link) link.classList.add('active');
+      if (name === 'engine') loadAudioList();
     }
 
     // ── STATUS ──
@@ -2181,43 +2200,86 @@
       }
     }
 
+    let recRecognition = null;
+    let recTranscriptText = "";
+
     async function startAudioRecording() {
       try {
         audioStream = await navigator.mediaDevices.getUserMedia({ audio: true });
         audioChunks = [];
         currentAudioHash = null;
+        recTranscriptText = "";
         mediaRecorder = new MediaRecorder(audioStream);
         mediaRecorder.ondataavailable = (e) => { if (e.data.size > 0) audioChunks.push(e.data); };
         mediaRecorder.onstop = () => {
           if (audioStream) { audioStream.getTracks().forEach(t => t.stop()); audioStream = null; }
-          if (currentAudioHash) finishAudioUpload(currentAudioHash);
-          else {
-            // Standalone upload — no patch hash needed
-            if (audioChunks.length > 0) {
-              const blob = new Blob(audioChunks, { type: "audio/webm" });
-              const reader = new FileReader();
-              reader.onloadend = () => {
-                const b64 = reader.result.split(",")[1];
-                if (b64) {
-                  fetch("/audio/store", {
-                    method: "POST",
-                    headers: { "Content-Type": "application/json" },
-                    body: JSON.stringify({ audio_b64: b64, label: "standalone" })
-                  }).then(() => {
-                    audioChunks = [];
-                    document.getElementById("recStatus").textContent = "Stored";
-                    document.getElementById("audioStatus").textContent = "stored";
-                  }).catch(() => { });
-                }
-              };
-              reader.readAsDataURL(blob);
+          if (recRecognition) { try { recRecognition.stop(); } catch (e) { } recRecognition = null; }
+          // Show transcript
+          if (recTranscriptText.trim()) {
+            document.getElementById("liveTranscriptText").textContent = recTranscriptText;
+            setEtlStage('transcribe', 'done');
+            // Send transcript to WebSocket to trigger observer → builder
+            if (ws && ws.readyState === WebSocket.OPEN) {
+              ws.send(JSON.stringify({ type: "transcript", text: recTranscriptText.trim() }));
+              setEtlStage('observe', 'active');
+              setStatus("Transcript sent to observer LLM", "dot-thinking");
             }
           }
+          // Upload audio file
+          if (currentAudioHash) finishAudioUpload(currentAudioHash);
+          else if (audioChunks.length > 0) {
+            const blob = new Blob(audioChunks, { type: "audio/webm" });
+            const reader = new FileReader();
+            reader.onloadend = () => {
+              const b64 = reader.result.split(",")[1];
+              if (b64) {
+                fetch("/audio/store", {
+                  method: "POST",
+                  headers: { "Content-Type": "application/json" },
+                  body: JSON.stringify({ audio_b64: b64, label: "standalone", transcript: recTranscriptText })
+                }).then(r => r.json()).then(data => {
+                  audioChunks = [];
+                  document.getElementById("recStatus").textContent = "Stored";
+                  document.getElementById("audioStatus").textContent = "stored";
+                  loadAudioList();
+                  // If we have transcript but no WS, trigger generate directly
+                  if (recTranscriptText.trim() && (!ws || ws.readyState !== WebSocket.OPEN)) {
+                    triggerGenerateFromAudio(recTranscriptText.trim(), data.audio_file);
+                  }
+                }).catch(() => { });
+              }
+            };
+            reader.readAsDataURL(blob);
+          }
         };
         mediaRecorder.start();
+        // Start speech recognition in parallel for live transcription
+        const SR = window.SpeechRecognition || window.webkitSpeechRecognition;
+        if (SR) {
+          recRecognition = new SR();
+          recRecognition.lang = "en-US";
+          recRecognition.continuous = true;
+          recRecognition.interimResults = true;
+          recRecognition.onresult = (event) => {
+            let interim = "";
+            let final = "";
+            for (let i = event.resultIndex; i < event.results.length; i++) {
+              if (event.results[i].isFinal) final += event.results[i][0].transcript + " ";
+              else interim += event.results[i][0].transcript;
+            }
+            recTranscriptText += final;
+            const display = recTranscriptText + (interim ? '<span style="color:var(--text3)">' + interim + '</span>' : '');
+            document.getElementById("liveTranscriptText").innerHTML = display || "Listening...";
+            if (final.trim()) setEtlStage('transcribe', 'active');
+          };
+          recRecognition.onerror = (e) => { console.warn('Speech error during recording:', e.error); };
+          try { recRecognition.start(); } catch (e) { console.warn('SR already started'); }
+        }
         document.getElementById("recButton").classList.add("recording");
         document.getElementById("recStatus").textContent = "Recording...";
         document.getElementById("audioStatus").textContent = "recording";
+        document.getElementById("liveTranscript").style.display = "block";
+        document.getElementById("liveTranscriptText").textContent = "Listening...";
         setEtlStage('capture', 'active');
         recSeconds = 0;
         recTimer = setInterval(() => {
@@ -2234,12 +2296,81 @@
             bar.style.height = (Math.random() * 36 + 4) + "px";
           });
         }, 100);
-        setStatus("Audio recording started", "dot-active");
+        setStatus("Audio recording + transcription started", "dot-active");
       } catch (e) {
         setStatus("Audio recording error: " + e.message, "dot-error");
       }
     }
 
+    // Trigger code generation from audio transcript when WS not connected
+    async function triggerGenerateFromAudio(transcript, audioFile) {
+      setEtlStage('observe', 'active');
+      setStatus("Sending audio transcript to LLM...", "dot-thinking");
+      try {
+        const resp = await fetch('/audio/generate', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ transcript: transcript, audio_file: audioFile, mode: document.getElementById('mode')?.value || 'continuous_code' })
+        });
+        const data = await resp.json();
+        if (data.patch_output) {
+          setEtlStage('observe', 'done');
+          setEtlStage('build', 'done');
+          setEtlStage('receipt', 'done');
+          addPatchEntry(data.patch_output, data.patch_hash, data.receipt, undefined);
+          patchCount++;
+          document.getElementById("patchCount").textContent = patchCount + " patch" + (patchCount !== 1 ? "es" : "");
+          setStatus("Code patch generated from audio", "dot-active");
+          // Store artifact
+          const sections = parseSections(data.patch_output);
+          const codeText = (sections.CODE || "").replace(/^```python\s*/i, "").replace(/^```\s*/, "").replace(/```$/, "").trim();
+          if (codeText && data.patch_hash) {
+            fetch("/store-artifact?patch_hash=" + data.patch_hash + "&code=" + encodeURIComponent(codeText) + "&reasoning=" + encodeURIComponent(sections.REASONING || "") + "&evidence=" + encodeURIComponent(sections.EVIDENCE || "") + "&observer_output=" + encodeURIComponent(data.patch_output), { method: "POST" }).catch(() => { });
+            // Link audio to this artifact
+            if (audioChunks.length > 0) {
+              currentAudioHash = data.patch_hash;
+              finishAudioUpload(data.patch_hash);
+            }
+            if (document.getElementById("autoRun") && document.getElementById("autoRun").checked) {
+              setTimeout(() => runCode(data.patch_hash, null), 800);
+            }
+          }
+        } else if (data.error) {
+          setStatus("LLM error: " + data.error, "dot-error");
+        }
+      } catch (e) {
+        setStatus("Generate error: " + e.message, "dot-error");
+      }
+    }
+
+    // Load recorded audio files from server
+    async function loadAudioList() {
+      try {
+        const resp = await fetch('/audio/list');
+        const data = await resp.json();
+        const el = document.getElementById('audioList');
+        if (data.total === 0) {
+          el.innerHTML = '<div style="font-size:11px;color:var(--text3);padding:8px;font-family:var(--mono)">No recordings yet.</div>';
+          return;
+        }
+        el.innerHTML = data.files.map(f => {
+          const sizeKB = (f.size_bytes / 1024).toFixed(1);
+          const date = new Date(f.modified * 1000).toLocaleTimeString();
+          return '<div style="display:flex;align-items:center;gap:10px;padding:8px 12px;background:var(--panel2);border:1px solid var(--border);border-radius:8px">' +
+            '<span style="font-size:16px">🎵</span>' +
+            '<div style="flex:1;min-width:0">' +
+            '<div style="font-size:11px;font-family:var(--mono);color:var(--text);overflow:hidden;text-overflow:ellipsis;white-space:nowrap">' + f.filename + '</div>' +
+            '<div style="font-size:9px;color:var(--text3)">' + sizeKB + 'KB · ' + date + '</div>' +
+            '</div>' +
+            '<audio controls preload="none" src="/audio/' + f.filename + '" style="height:28px;width:200px" />' +
+            '<a href="/audio/' + f.filename + '" download style="font-size:11px;color:var(--accent);text-decoration:none">Download</a>' +
+            '</div>';
+        }).join('');
+      } catch (e) {
+        document.getElementById('audioList').innerHTML = '<div style="font-size:11px;color:var(--red);padding:8px">Error: ' + escapeHtml(e.message) + '</div>';
+      }
+    }
+
     function finishAudioUpload(patchHash) {
       if (audioChunks.length === 0) return;
       const blob = new Blob(audioChunks, { type: "audio/webm" });