Upload folder using huggingface_hub

app.py

@@ -33,6 +33,7 @@ from src.gate import builder_gate, evidence_gate, log_gate_decision, get_gate_lo
 from src.observer_llm import observe
 from src.builder_llm import build, debug_fix
 from src.code_receipts import save_receipt, list_receipts, get_receipt
+from src.consensus import observer_consensus, builder_consensus
 
 APP_ROOT = Path(__file__).resolve().parent
 FRONTEND_INDEX = APP_ROOT / "frontend" / "index.html"
@@ -275,7 +276,7 @@ async def generate_patch(request: Request):
 @app.post("/audio/generate")
 async def audio_generate(request: Request):
     """Generate code from an audio transcript without requiring an existing session.
-    Creates a temporary session, injects the transcript as audio evidence,
+    Creates a temporary session, injects the transcript and audio features as evidence,
     runs observer → builder → receipt."""
     body = {}
     try:
@@ -284,6 +285,7 @@ async def audio_generate(request: Request):
         pass
     transcript = body.get("transcript", "") or request.query_params.get("transcript", "")
     audio_file = body.get("audio_file", "") or request.query_params.get("audio_file", "")
+    audio_features = body.get("audio_features", {}) or {}
     mode = body.get("mode", "continuous_code") or request.query_params.get("mode", "continuous_code")
     if not transcript:
         return JSONResponse({"error": "transcript required"}, 400)
@@ -299,6 +301,9 @@ async def audio_generate(request: Request):
 
     # Compress and run the full pipeline
     compact = compress_state(state)
+    # Inject audio features into compact state for the builder
+    if audio_features:
+        compact["audio_features"] = audio_features
     ev_result = evidence_gate("", compact)
     log_gate_decision("evidence_gate", ev_result, patch_hash="")
     observation = observe(state, compact)
@@ -329,6 +334,7 @@ async def audio_generate(request: Request):
         "feature_attribution": ev_result2.feature_attribution,
         "audio_file": audio_file,
         "transcript": transcript,
+        "audio_features": audio_features,
     }
     artifact_store[result["patch_hash"]] = artifact_data
     save_artifact_disk(result["patch_hash"], artifact_data)

frontend/index.html

@@ -1697,6 +1697,15 @@
               </svg>
               Start Speech
             </button>
+            <button class="btn btn-sm btn-ghost" id="ttsBtn" onclick="toggleTTS()"
+              style="border-color:var(--accent);color:var(--accent)">
+              <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
+                <polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5" />
+                <path d="M19.07 4.93a10 10 0 0 1 0 14.14" />
+                <path d="M15.54 8.46a5 5 0 0 1 0 7.07" />
+              </svg>
+              TTS: On
+            </button>
             <button class="btn btn-sm btn-ghost" onclick="switchView('terminal')">
               <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
                 <polyline points="4 17 10 11 4 5" />
@@ -2074,6 +2083,7 @@
             document.getElementById("observerStatus").textContent = "updated";
             setEtlStage('observe', 'done');
             setEtlStage('build', 'active');
+            speakText(msg.output);
             break;
           case "patch":
             addPatchEntry(msg.output, msg.patch_hash, msg.receipt, msg.qvd);
@@ -2202,6 +2212,10 @@
 
     let recRecognition = null;
     let recTranscriptText = "";
+    let recAudioContext = null;
+    let recAnalyser = null;
+    let recFreqData = null;
+    let recFreqSamples = [];
 
     async function startAudioRecording() {
       try {
@@ -2209,25 +2223,38 @@
         audioChunks = [];
         currentAudioHash = null;
         recTranscriptText = "";
+        recFreqSamples = [];
+
+        // Real audio analysis via Web Audio API
+        recAudioContext = new (window.AudioContext || window.webkitAudioContext)();
+        const source = recAudioContext.createMediaStreamSource(audioStream);
+        recAnalyser = recAudioContext.createAnalyser();
+        recAnalyser.fftSize = 2048;
+        source.connect(recAnalyser);
+        recFreqData = new Uint8Array(recAnalyser.frequencyBinCount);
+
         mediaRecorder = new MediaRecorder(audioStream);
         mediaRecorder.ondataavailable = (e) => { if (e.data.size > 0) audioChunks.push(e.data); };
         mediaRecorder.onstop = () => {
           if (audioStream) { audioStream.getTracks().forEach(t => t.stop()); audioStream = null; }
           if (recRecognition) { try { recRecognition.stop(); } catch (e) { } recRecognition = null; }
-          // Show transcript
-          if (recTranscriptText.trim()) {
-            document.getElementById("liveTranscriptText").textContent = recTranscriptText;
-            setEtlStage('transcribe', 'done');
-            // Send transcript to WebSocket to trigger observer → builder
-            if (ws && ws.readyState === WebSocket.OPEN) {
-              ws.send(JSON.stringify({ type: "transcript", text: recTranscriptText.trim() }));
-              setEtlStage('observe', 'active');
-              setStatus("Transcript sent to observer LLM", "dot-thinking");
-            }
-          }
-          // Upload audio file
-          if (currentAudioHash) finishAudioUpload(currentAudioHash);
-          else if (audioChunks.length > 0) {
+          if (recAudioContext) { try { recAudioContext.close(); } catch (e) { } recAudioContext = null; }
+
+          // Compute audio features from collected frequency samples
+          const audioFeatures = computeAudioFeatures(recFreqSamples);
+          document.getElementById("liveTranscriptText").innerHTML =
+            (recTranscriptText.trim() ? '<div style="margin-bottom:8px;color:var(--text)">' + escapeHtml(recTranscriptText) + '</div>' : '') +
+            '<div style="font-size:10px;color:var(--text3);border-top:1px solid var(--border);padding-top:8px">' +
+            '<b>Audio Analysis:</b> dominant_freq=' + audioFeatures.dominant_freq + 'Hz' +
+            ' | spectral_centroid=' + audioFeatures.spectral_centroid +
+            ' | rhythm_bpm=' + audioFeatures.estimated_bpm +
+            ' | patterns=' + audioFeatures.patterns.length +
+            ' | samples=' + audioFeatures.sample_count +
+            '</div>';
+          setEtlStage('transcribe', 'done');
+
+          // Upload audio + features + transcript, then generate
+          if (audioChunks.length > 0) {
             const blob = new Blob(audioChunks, { type: "audio/webm" });
             const reader = new FileReader();
             reader.onloadend = () => {
@@ -2236,15 +2263,25 @@
                 fetch("/audio/store", {
                   method: "POST",
                   headers: { "Content-Type": "application/json" },
-                  body: JSON.stringify({ audio_b64: b64, label: "standalone", transcript: recTranscriptText })
+                  body: JSON.stringify({ audio_b64: b64, label: "standalone", transcript: recTranscriptText, audio_features: audioFeatures })
                 }).then(r => r.json()).then(data => {
                   audioChunks = [];
-                  document.getElementById("recStatus").textContent = "Stored";
-                  document.getElementById("audioStatus").textContent = "stored";
+                  document.getElementById("recStatus").textContent = "Analyzed";
+                  document.getElementById("audioStatus").textContent = "analyzed";
                   loadAudioList();
-                  // If we have transcript but no WS, trigger generate directly
-                  if (recTranscriptText.trim() && (!ws || ws.readyState !== WebSocket.OPEN)) {
-                    triggerGenerateFromAudio(recTranscriptText.trim(), data.audio_file);
+                  // Always trigger generation — with or without speech
+                  const payload = {
+                    transcript: recTranscriptText.trim() || "[no speech detected — environmental audio with patterns: " + audioFeatures.patterns.length + " repeating cycles, dominant freq " + audioFeatures.dominant_freq + "Hz, estimated BPM " + audioFeatures.estimated_bpm + "]",
+                    audio_file: data.audio_file,
+                    audio_features: audioFeatures,
+                    mode: document.getElementById('mode')?.value || 'continuous_code'
+                  };
+                  if (ws && ws.readyState === WebSocket.OPEN) {
+                    ws.send(JSON.stringify({ type: "transcript", text: payload.transcript, audio_features: audioFeatures }));
+                    setEtlStage('observe', 'active');
+                    setStatus("Audio + features sent to observer LLM", "dot-thinking");
+                  } else {
+                    triggerGenerateFromAudio(payload.transcript, data.audio_file, audioFeatures);
                   }
                 }).catch(() => { });
               }
@@ -2252,8 +2289,9 @@
             reader.readAsDataURL(blob);
           }
         };
-        mediaRecorder.start();
-        // Start speech recognition in parallel for live transcription
+        mediaRecorder.start(1000); // timeslice = 1s chunks
+
+        // Speech recognition with auto-restart (Chrome stops after ~60s)
         const SR = window.SpeechRecognition || window.webkitSpeechRecognition;
         if (SR) {
           recRecognition = new SR();
@@ -2261,22 +2299,36 @@
           recRecognition.continuous = true;
           recRecognition.interimResults = true;
           recRecognition.onresult = (event) => {
-            let interim = "";
-            let final = "";
+            let interim = "", final = "";
             for (let i = event.resultIndex; i < event.results.length; i++) {
               if (event.results[i].isFinal) final += event.results[i][0].transcript + " ";
               else interim += event.results[i][0].transcript;
             }
             recTranscriptText += final;
-            const display = recTranscriptText + (interim ? '<span style="color:var(--text3)">' + interim + '</span>' : '');
+            const display = recTranscriptText + (interim ? '<span style="color:var(--text3)">' + escapeHtml(interim) + '</span>' : '');
             document.getElementById("liveTranscriptText").innerHTML = display || "Listening...";
             if (final.trim()) setEtlStage('transcribe', 'active');
           };
-          recRecognition.onerror = (e) => { console.warn('Speech error during recording:', e.error); };
-          try { recRecognition.start(); } catch (e) { console.warn('SR already started'); }
+          recRecognition.onerror = (e) => {
+            console.warn('SR error:', e.error);
+            if (e.error === 'not-allowed' || e.error === 'service-not-allowed') {
+              document.getElementById("liveTranscriptText").innerHTML =
+                '<span style="color:var(--amber)">Speech transcription unavailable. Audio analysis running — extracting frequencies, rhythms, and environmental patterns.</span>';
+            }
+          };
+          recRecognition.onend = () => {
+            if (mediaRecorder && mediaRecorder.state === 'recording') {
+              try { recRecognition.start(); } catch (e) { }
+            }
+          };
+          try { recRecognition.start(); } catch (e) { console.warn('SR start failed:', e); }
+        } else {
+          document.getElementById("liveTranscriptText").innerHTML =
+            '<span style="color:var(--amber)">SpeechRecognition not available. Audio analysis running — extracting frequencies, rhythms, and patterns from environmental audio.</span>';
         }
+
         document.getElementById("recButton").classList.add("recording");
-        document.getElementById("recStatus").textContent = "Recording...";
+        document.getElementById("recStatus").textContent = "Recording + analyzing...";
         document.getElementById("audioStatus").textContent = "recording";
         document.getElementById("liveTranscript").style.display = "block";
         document.getElementById("liveTranscriptText").textContent = "Listening...";
@@ -2288,29 +2340,128 @@
           const s = (recSeconds % 60).toString().padStart(2, '0');
           document.getElementById("recTime").textContent = m + ':' + s;
         }, 1000);
-        // Waveform animation
+
+        // Real waveform from frequency data + sample for pattern analysis
         const wf = document.getElementById("waveform");
-        wf.innerHTML = Array.from({ length: 30 }, () => '<div class="wave-bar" style="height:4px"></div>').join("");
+        wf.innerHTML = Array.from({ length: 40 }, () => '<div class="wave-bar" style="height:4px"></div>').join("");
         waveformTimer = setInterval(() => {
-          wf.querySelectorAll(".wave-bar").forEach(bar => {
-            bar.style.height = (Math.random() * 36 + 4) + "px";
-          });
-        }, 100);
-        setStatus("Audio recording + transcription started", "dot-active");
+          if (recAnalyser && recFreqData) {
+            recAnalyser.getByteFrequencyData(recFreqData);
+            if (recSeconds % 2 === 0 && recSeconds > 0) {
+              recFreqSamples.push(Array.from(recFreqData.slice(0, 128)));
+            }
+            const bars = wf.querySelectorAll(".wave-bar");
+            const step = Math.floor(recFreqData.length / bars.length);
+            bars.forEach((bar, i) => {
+              const val = recFreqData[i * step] || 0;
+              bar.style.height = Math.max(4, (val / 255) * 40) + "px";
+            });
+          }
+        }, 50);
+        setStatus("Audio recording + analysis started", "dot-active");
       } catch (e) {
         setStatus("Audio recording error: " + e.message, "dot-error");
       }
     }
 
+    // Compute audio features from collected frequency samples
+    function computeAudioFeatures(samples) {
+      if (!samples || samples.length === 0) {
+        return { dominant_freq: 0, spectral_centroid: 0, estimated_bpm: 0, patterns: [], sample_count: 0, avg_energy: 0 };
+      }
+      const nBins = samples[0].length;
+      const avgSpectrum = new Array(nBins).fill(0);
+      for (const s of samples) for (let i = 0; i < nBins; i++) avgSpectrum[i] += s[i];
+      for (let i = 0; i < nBins; i++) avgSpectrum[i] /= samples.length;
+
+      let maxBin = 0, maxVal = 0;
+      for (let i = 0; i < nBins; i++) { if (avgSpectrum[i] > maxVal) { maxVal = avgSpectrum[i]; maxBin = i; } }
+      const dominantFreq = Math.round(maxBin * 44100 / 2048);
+
+      let sumWeighted = 0, sumMag = 0;
+      for (let i = 0; i < nBins; i++) {
+        const freq = i * 44100 / 2048;
+        sumWeighted += freq * avgSpectrum[i];
+        sumMag += avgSpectrum[i];
+      }
+      const spectralCentroid = sumMag > 0 ? Math.round(sumWeighted / sumMag) : 0;
+
+      const energies = samples.map(s => s.reduce((a, b) => a + b, 0) / s.length);
+      let bpm = 0;
+      if (energies.length > 4) {
+        const peaks = [];
+        for (let i = 1; i < energies.length - 1; i++) {
+          if (energies[i] > energies[i - 1] && energies[i] > energies[i + 1] && energies[i] > 50) peaks.push(i);
+        }
+        if (peaks.length > 1) {
+          const intervals = [];
+          for (let i = 1; i < peaks.length; i++) intervals.push(peaks[i] - peaks[i - 1]);
+          const avgInterval = intervals.reduce((a, b) => a + b, 0) / intervals.length;
+          bpm = avgInterval > 0 ? Math.round(60 / (avgInterval * 2)) : 0;
+        }
+      }
+
+      // Detect repeating patterns via autocorrelation
+      const patterns = [];
+      if (samples.length > 6) {
+        for (let lag = 2; lag < Math.min(samples.length / 2, 10); lag++) {
+          let corr = 0, count = 0;
+          for (let i = 0; i < samples.length - lag; i++) {
+            for (let j = 0; j < nBins; j++) { corr += Math.abs(samples[i][j] - samples[i + lag][j]); count++; }
+          }
+          const avgDiff = corr / count;
+          if (avgDiff < 15) patterns.push({ lag_seconds: lag * 2, correlation: Math.round((1 - avgDiff / 128) * 100) / 100 });
+        }
+      }
+
+      return { dominant_freq: dominantFreq, spectral_centroid: spectralCentroid, estimated_bpm: bpm, patterns, sample_count: samples.length, avg_energy: Math.round(energies.reduce((a, b) => a + b, 0) / energies.length) };
+    }
+
+    // TTS — speak observer reasoning aloud
+    let ttsEnabled = true;
+    let ttsVoice = null;
+    function speakText(text) {
+      if (!ttsEnabled || !text || !('speechSynthesis' in window)) return;
+      window.speechSynthesis.cancel();
+      if (!ttsVoice) {
+        const voices = window.speechSynthesis.getVoices();
+        ttsVoice = voices.find(v => v.lang.startsWith('en') && v.name.includes('Google')) || voices.find(v => v.lang.startsWith('en')) || voices[0];
+      }
+      const sections = parseSections(text);
+      let speak = sections.REASONING || sections.INFERRED_INTENT || text.substring(0, 500);
+      speak = speak.replace(/```[\s\S]*?```/g, ' code omitted ').replace(/[#*\-]/g, ' ').replace(/\s+/g, ' ').trim();
+      if (speak.length > 600) speak = speak.substring(0, 600) + '. That is the key finding.';
+      const utter = new SpeechSynthesisUtterance(speak);
+      if (ttsVoice) utter.voice = ttsVoice;
+      utter.rate = 1.1; utter.pitch = 0.9;
+      window.speechSynthesis.speak(utter);
+    }
+    if ('speechSynthesis' in window) window.speechSynthesis.onvoiceschanged = () => { ttsVoice = null; };
+
+    function toggleTTS() {
+      ttsEnabled = !ttsEnabled;
+      const btn = document.getElementById('ttsBtn');
+      if (ttsEnabled) {
+        btn.style.borderColor = 'var(--accent)';
+        btn.style.color = 'var(--accent)';
+        btn.innerHTML = btn.innerHTML.replace('TTS: Off', 'TTS: On');
+      } else {
+        btn.style.borderColor = 'var(--text3)';
+        btn.style.color = 'var(--text3)';
+        btn.innerHTML = btn.innerHTML.replace('TTS: On', 'TTS: Off');
+        window.speechSynthesis.cancel();
+      }
+    }
+
     // Trigger code generation from audio transcript when WS not connected
-    async function triggerGenerateFromAudio(transcript, audioFile) {
+    async function triggerGenerateFromAudio(transcript, audioFile, audioFeatures) {
       setEtlStage('observe', 'active');
-      setStatus("Sending audio transcript to LLM...", "dot-thinking");
+      setStatus("Sending audio + analysis to LLM...", "dot-thinking");
       try {
         const resp = await fetch('/audio/generate', {
           method: 'POST',
           headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ transcript: transcript, audio_file: audioFile, mode: document.getElementById('mode')?.value || 'continuous_code' })
+          body: JSON.stringify({ transcript: transcript, audio_file: audioFile, audio_features: audioFeatures, mode: document.getElementById('mode')?.value || 'continuous_code' })
         });
         const data = await resp.json();
         if (data.patch_output) {
@@ -2321,6 +2472,7 @@
           patchCount++;
           document.getElementById("patchCount").textContent = patchCount + " patch" + (patchCount !== 1 ? "es" : "");
           setStatus("Code patch generated from audio", "dot-active");
+          speakText(data.patch_output);
           // Store artifact
           const sections = parseSections(data.patch_output);
           const codeText = (sections.CODE || "").replace(/^```python\s*/i, "").replace(/^```\s*/, "").replace(/```$/, "").trim();

src/builder_llm.py

@@ -12,9 +12,9 @@ import requests
 from .stream_state import SessionState, sha256_text
 
 
-BUILDER_PROMPT = """You are the Builder in a Continuity Sensory Code Engine — Sensory Proprietary Compiler V1.
+BUILDER_PROMPT = """You are the Builder in a Sensory Proprietary Compiler — a research lab system that extracts maximum value from human sensory input.
 
-You receive an observation from the Observer LLM and a FALLBACK LEVEL. You MUST produce a structured artifact. NEVER return empty. NEVER return INSUFFICIENT_EVIDENCE.
+Your job is NOT to build a code assistant. Your job is NOT to build a debugging tool. Your job is to extract novel patterns from sensory data and create ML artifacts that have intellectual property value.
 
 CRITICAL RULES — VIOLATION = REJECTION:
 1. The CODE section MUST contain ONLY valid, executable Python code.
@@ -24,46 +24,70 @@ CRITICAL RULES — VIOLATION = REJECTION:
 5. Do NOT import cv2, torch, tensorflow, pandas, matplotlib, scipy, sklearn, or any package not listed above.
 6. Do NOT use input() or any interactive call — code must run non-interactively.
 
-FALLBACK LADDER — you are at the level indicated. Generate code appropriate to that level:
-
-LEVEL 1 (task_code): Explicit user intent detected. Generate task-specific code that does what the user asked.
-
-LEVEL 2 (instrumentation_code): No explicit intent, but rich sensory features (camera, motion, audio). Generate instrumentation code that:
-  - Captures and quantifies the sensory channels present (motion metrics, audio energy, frame entropy)
-  - Creates reusable measurement utilities
+WHAT TO BUILD — based on available evidence:
+
+If AUDIO FEATURES are present (dominant_freq, spectral_centroid, estimated_bpm, patterns):
+  - Extract and quantify the environmental signals: AC hum frequency, mechanical rhythms, background noise patterns
+  - Create feature vectors from the frequency spectrum data
+  - Build pattern detectors that identify repeating cycles (e.g. AC 60Hz cycle, mechanical rotation, HVAC cycling)
+  - Generate signal classification models using numpy (FFT, spectral analysis, autocorrelation)
+  - Create data structures that capture the unique signature of this recording session
+  - Build novelty detectors that flag when the signal pattern changes
+  - Example: AC cycle analyzer that detects 60Hz/120Hz harmonics, rhythm pattern extractor, spectral fingerprint generator
+
+If SPEECH TRANSCRIPT is present:
+  - Extract semantic patterns, topic clusters, and intent signals from the transcript
+  - Build text analysis tools that quantify information density, novelty, and signal-to-noise ratio
+  - Create topic extraction and association mapping code
+  - Generate intent classifiers that map speech to actionable patterns
+  - Build evidence extraction pipelines that pull claims, timestamps, and verification markers
+
+If CAMERA/VISUAL data is present:
+  - Extract visual features: motion vectors, color distributions, frame entropy
+  - Build scene change detectors and visual novelty scorers
+  - Create visual pattern recognizers using numpy operations on frame data
+
+FALLBACK LADDER — you are at the level indicated:
+
+LEVEL 1 (task_code): Explicit user intent detected from speech. Generate code that does what the user asked — but frame it as a data extraction or pattern analysis tool, not a generic utility.
+
+LEVEL 2 (signal_extraction): No explicit intent, but rich audio features. Generate code that:
+  - Analyzes the frequency spectrum and extracts dominant patterns
+  - Detects environmental rhythms (AC cycles, mechanical patterns, biological rhythms)
+  - Creates a spectral fingerprint unique to this recording environment
+  - Builds a pattern classifier that can distinguish this session from others
   - Produces a feature vector or data structure from the sensory input
-  - Example: motion rhythm analyzer, audio energy spectrum, frame entropy tracker
 
 LEVEL 3 (aesthetic_motif): Distinctive visual features but no clear intent. Generate code that:
-  - Extracts aesthetic features from described visual input (color palette, light patterns, composition)
-  - Creates a UI theme, color scheme, or visual motif from the sensory description
-  - Produces a design grammar or style specification
-  - Example: color palette generator from room description, UI theme from lighting conditions
+  - Extracts aesthetic features from described visual input (color palette, light patterns)
+  - Creates a visual motif or style specification from the sensory description
+  - Produces a design grammar or compression of the visual state
 
-LEVEL 4 (topic_association): Only background audio/speech detected. Generate code that:
-  - Maps detected topics to relevant tools or code patterns
-  - Creates a topic-to-tool association dictionary
-  - Generates a transcript summarizer or topic classifier
-  - Example: TV topic to code domain mapper, background speech topic extractor
+LEVEL 4 (topic_association): Only background audio detected. Generate code that:
+  - Maps detected audio patterns to potential data sources and ML applications
+  - Creates a signal-to-topic association dictionary
+  - Generates an environmental audio classifier or background pattern extractor
+  - Builds a novelty detector for ambient sound changes
 
 LEVEL 5 (capture_protocol): Minimal sensory signal. Generate code that:
-  - Improves the capture protocol itself
+  - Improves the capture and analysis protocol itself
   - Creates a better sampling strategy or capture diagnostic
-  - Produces a session health check or capture quality report
-  - Example: adaptive frame rate calculator, audio sensitivity tuner, capture health monitor
+  - Produces a session health check or signal quality report
+  - Builds an adaptive sampling optimizer
 
 Output format (use exactly these headers):
 
 EVIDENCE:
-- List specific sensory evidence from the observation. Include channel names and feature values.
+- List specific sensory evidence. Include channel names, feature values, frequencies, patterns detected.
 
 REASONING:
-- Step by step: why this artifact should exist. Connect sensory evidence to code decisions. Explain which fallback level triggered this.
+- Step by step: why this artifact should exist. Connect sensory evidence to the ML approach. What novel pattern was extracted? What is the intellectual property value of this artifact?
 
 CODE:
 ```python
 # ONLY executable Python code here. No prose. No explanations.
 # Must pass syntax validation. Must use only installed packages.
+# This code should EXTRACT PATTERNS, not build generic tools.
 ```
 
 RUN:
@@ -73,7 +97,7 @@ TEST:
 - Minimal verification step.
 
 ATTRIBUTION:
-- Which sensory evidence (frame hashes, audio segments, motion scores, visual features) triggered this code. Be specific.
+- Which sensory evidence (frequencies, patterns, transcripts, frame hashes) triggered this code. Be specific with numerical values.
 """
 
 
@@ -158,7 +182,10 @@ ARTIFACT TYPE: {artifact_type}
 SENSORY CHANNELS: {', '.join(sensory_channels) if sensory_channels else 'none'}
 FEATURE ATTRIBUTION: {json.dumps(feature_attribution, default=str)}
 
-You are at FALLBACK LEVEL {fallback_level}. Generate {artifact_type} based on the available sensory evidence. NEVER return INSUFFICIENT_EVIDENCE. Always produce a CODE section with runnable Python."""
+AUDIO FEATURES (from browser Web Audio API analysis):
+{json.dumps(observation.get('audio_features', {}), indent=2, default=str) if observation.get('audio_features') else 'No audio features available — use transcript and visual evidence only.'}
+
+You are at FALLBACK LEVEL {fallback_level}. Generate {artifact_type} based on the available sensory evidence. NEVER return INSUFFICIENT_EVIDENCE. Always produce a CODE section with runnable Python that EXTRACTS PATTERNS or ANALYZES SIGNALS."""
 
     provider = os.getenv("PROVIDER", "groq").lower().strip()
     if provider == "ollama":

src/consensus.py

@@ -0,0 +1,476 @@
+"""
+Multi-model consensus engine for the CSC Engine research lab protocol.
+
+Runs observer + builder through multiple LLM providers, compares outputs,
+flags disagreements, and produces a consensus verdict with timestamps.
+
+Architecture:
+  1. Call N providers for the same prompt
+  2. Extract structured sections (EVIDENCE, REASONING, CODE) from each
+  3. Compare section similarity and content overlap
+  4. Flag disagreements (different code approaches, conflicting evidence)
+  5. Produce consensus verdict: AGREED, PARTIAL_AGREEMENT, DISAGREEMENT
+  6. Select best output (longest code, most evidence items, or majority vote)
+  7. Every step timestamped and logged
+"""
+
+import os
+import time
+import json
+import hashlib
+import difflib
+from typing import Optional
+
+
+def _get_available_providers() -> list[str]:
+    """Determine which LLM providers are available based on env vars."""
+    providers = []
+    if os.getenv("GROQ_API_KEY") or os.getenv("GROK_API_KEY"):
+        providers.append("groq")
+    if os.getenv("HF_TOKEN") or os.getenv("HUGGING_FACE_HUB_TOKEN"):
+        providers.append("huggingface")
+    if os.getenv("OPENAI_API_KEY"):
+        providers.append("openai")
+    if os.getenv("OLLAMA_HOST"):
+        providers.append("ollama")
+    # Always include the primary provider
+    primary = os.getenv("PROVIDER", "groq").lower().strip()
+    if primary == "hybrid":
+        primary = "groq"
+    if primary not in providers:
+        providers.insert(0, primary)
+    # Deduplicate, preserve order, max 3
+    seen = set()
+    unique = []
+    for p in providers:
+        if p not in seen:
+            seen.add(p)
+            unique.append(p)
+    return unique[:3]
+
+
+def _call_provider(provider: str, prompt: str, frame_b64: Optional[str] = None) -> str:
+    """Call a single LLM provider and return its output."""
+    provider = provider.lower().strip()
+    if provider == "groq" or provider == "grok":
+        from .observer_llm import call_grok
+        return call_grok(prompt, frame_b64)
+    elif provider == "huggingface":
+        from .observer_llm import call_hf_inference
+        return call_hf_inference(prompt, frame_b64)
+    elif provider == "openai":
+        from .observer_llm import call_openai
+        return call_openai(prompt, frame_b64)
+    elif provider == "ollama":
+        from .observer_llm import call_ollama
+        return call_ollama(prompt, frame_b64)
+    else:
+        raise RuntimeError(f"Unknown provider: {provider}")
+
+
+def _call_builder_provider(provider: str, prompt: str) -> str:
+    """Call a single builder LLM provider."""
+    provider = provider.lower().strip()
+    if provider == "groq" or provider == "grok":
+        from .builder_llm import call_grok
+        return call_grok(prompt)
+    elif provider == "huggingface":
+        from .builder_llm import call_hf_inference
+        return call_hf_inference(prompt)
+    elif provider == "openai":
+        from .builder_llm import call_openai
+        return call_openai(prompt)
+    elif provider == "ollama":
+        from .builder_llm import call_ollama
+        return call_ollama(prompt)
+    else:
+        raise RuntimeError(f"Unknown builder provider: {provider}")
+
+
+def _extract_section(text: str, section_name: str) -> str:
+    """Extract a named section (EVIDENCE, REASONING, CODE, etc.) from LLM output."""
+    lines = text.split("\n")
+    capturing = False
+    collected = []
+    for line in lines:
+        stripped = line.strip().upper()
+        if stripped.startswith(section_name + ":"):
+            capturing = True
+            continue
+        if capturing:
+            # Check if we hit another section header
+            for header in ["EVIDENCE:", "REASONING:", "CODE:", "RUN:", "TEST:", "ATTRIBUTION:", "SCENE:", "INFERRED_INTENT:", "SIGNALS:", "CANDIDATE_TASK:", "UNCERTAINTY:"]:
+                if stripped.startswith(header):
+                    capturing = False
+                    break
+            if capturing:
+                collected.append(line)
+    return "\n".join(collected).strip()
+
+
+def _similarity(text_a: str, text_b: str) -> float:
+    """Compute text similarity ratio between two strings (0.0 to 1.0)."""
+    if not text_a or not text_b:
+        return 0.0
+    return difflib.SequenceMatcher(None, text_a.lower(), text_b.lower()).ratio()
+
+
+def _code_similarity(code_a: str, code_b: str) -> float:
+    """Compare code similarity ignoring whitespace and comments."""
+    def normalize(code: str) -> str:
+        lines = []
+        for line in code.split("\n"):
+            line = line.strip()
+            if line and not line.startswith("#"):
+                lines.append(line)
+        return " ".join(lines)
+    return _similarity(normalize(code_a), normalize(code_b))
+
+
+def observer_consensus(state, compact_state: dict) -> dict:
+    """Run observer through multiple providers and produce consensus.
+
+    Returns dict with:
+    - consensus_verdict: AGREED | PARTIAL_AGREEMENT | DISAGREEMENT | SINGLE_PROVIDER
+    - observer_output: best output selected
+    - all_outputs: list of {provider, output, timestamp}
+    - disagreements: list of flagged differences
+    - similarity_matrix: pairwise similarities
+    - timestamp: consensus timestamp
+    """
+    from .observer_llm import OBSERVER_PROMPT
+    from .stream_state import SessionState
+
+    frames = list(state.frames)
+    last_frame = frames[-1] if frames else None
+    frame_b64 = last_frame.jpeg_b64 if last_frame else None
+
+    prompt = f"""{OBSERVER_PROMPT}
+
+COMPRESSED SENSORY STATE:
+{json.dumps(compact_state, indent=2)}
+
+Produce your observation now."""
+
+    providers = _get_available_providers()
+    timestamp_start = time.time()
+
+    all_outputs = []
+    errors = []
+
+    for provider in providers:
+        ts = time.time()
+        try:
+            output = _call_provider(provider, prompt, frame_b64)
+            all_outputs.append({
+                "provider": provider,
+                "output": output,
+                "timestamp": ts,
+                "duration_ms": int((time.time() - ts) * 1000),
+                "error": None,
+            })
+        except Exception as e:
+            errors.append({
+                "provider": provider,
+                "error": str(e)[:200],
+                "timestamp": ts,
+            })
+
+    if not all_outputs:
+        raise RuntimeError(f"All observer providers failed: {errors}")
+
+    # If only one provider succeeded, return single-provider verdict
+    if len(all_outputs) == 1:
+        return {
+            "consensus_verdict": "SINGLE_PROVIDER",
+            "observer_output": all_outputs[0]["output"],
+            "all_outputs": all_outputs,
+            "errors": errors,
+            "disagreements": [],
+            "similarity_matrix": {},
+            "providers_used": [all_outputs[0]["provider"]],
+            "timestamp": time.time(),
+            "duration_ms": int((time.time() - timestamp_start) * 1000),
+        }
+
+    # Compare outputs pairwise
+    n = len(all_outputs)
+    similarity_matrix = {}
+    disagreements = []
+
+    for i in range(n):
+        for j in range(i + 1, n):
+            out_a = all_outputs[i]["output"]
+            out_b = all_outputs[j]["output"]
+            prov_a = all_outputs[i]["provider"]
+            prov_b = all_outputs[j]["provider"]
+
+            # Compare overall similarity
+            overall_sim = _similarity(out_a, out_b)
+
+            # Compare specific sections
+            scene_a = _extract_section(out_a, "SCENE")
+            scene_b = _extract_section(out_b, "SCENE")
+            intent_a = _extract_section(out_a, "INFERRED_INTENT")
+            intent_b = _extract_section(out_b, "INFERRED_INTENT")
+
+            scene_sim = _similarity(scene_a, scene_b)
+            intent_sim = _similarity(intent_a, intent_b)
+
+            key = f"{prov_a}_vs_{prov_b}"
+            similarity_matrix[key] = {
+                "overall": round(overall_sim, 3),
+                "scene": round(scene_sim, 3),
+                "intent": round(intent_sim, 3),
+            }
+
+            # Flag disagreements
+            if overall_sim < 0.3:
+                disagreements.append({
+                    "providers": [prov_a, prov_b],
+                    "type": "low_overall_similarity",
+                    "similarity": round(overall_sim, 3),
+                    "detail": "Outputs differ significantly in content and structure",
+                })
+            if intent_sim < 0.4 and intent_a and intent_b:
+                disagreements.append({
+                    "providers": [prov_a, prov_b],
+                    "type": "intent_divergence",
+                    "similarity": round(intent_sim, 3),
+                    "detail_a": intent_a[:200],
+                    "detail_b": intent_b[:200],
+                })
+
+    # Determine consensus verdict
+    avg_sim = sum(s["overall"] for s in similarity_matrix.values()) / len(similarity_matrix) if similarity_matrix else 0
+    if avg_sim >= 0.6:
+        verdict = "AGREED"
+    elif avg_sim >= 0.3:
+        verdict = "PARTIAL_AGREEMENT"
+    else:
+        verdict = "DISAGREEMENT"
+
+    # Select best output: prefer the one with most content (longest output)
+    best = max(all_outputs, key=lambda x: len(x["output"]))
+
+    return {
+        "consensus_verdict": verdict,
+        "observer_output": best["output"],
+        "all_outputs": all_outputs,
+        "errors": errors,
+        "disagreements": disagreements,
+        "similarity_matrix": similarity_matrix,
+        "avg_similarity": round(avg_sim, 3),
+        "providers_used": [o["provider"] for o in all_outputs],
+        "timestamp": time.time(),
+        "duration_ms": int((time.time() - timestamp_start) * 1000),
+    }
+
+
+def builder_consensus(state, observation: dict, gate_result=None) -> dict:
+    """Run builder through multiple providers and produce consensus.
+
+    Returns dict with:
+    - consensus_verdict: AGREED | PARTIAL_AGREEMENT | DISAGREEMENT | SINGLE_PROVIDER
+    - patch_output: best output selected
+    - patch_hash: hash of selected output
+    - receipt: receipt with consensus metadata
+    - all_outputs: list of {provider, output, code_extracted, timestamp}
+    - disagreements: list of flagged code differences
+    - similarity_matrix: pairwise code similarities
+    """
+    from .builder_llm import BUILDER_PROMPT, _extract_reasons, _extract_uncertainty
+    from .stream_state import sha256_text
+
+    fallback_level = 1
+    artifact_type = "task_code"
+    sensory_channels = []
+    feature_attribution = {}
+    if gate_result:
+        fallback_level = gate_result.fallback_level
+        artifact_type = gate_result.artifact_type
+        sensory_channels = gate_result.sensory_channels
+        feature_attribution = gate_result.feature_attribution
+
+    prompt = f"""{BUILDER_PROMPT}
+
+OBSERVER OUTPUT:
+{observation.get('observer_output', '')}
+
+MODE: {state.mode}
+
+FALLBACK LEVEL: {fallback_level}
+ARTIFACT TYPE: {artifact_type}
+SENSORY CHANNELS: {', '.join(sensory_channels) if sensory_channels else 'none'}
+FEATURE ATTRIBUTION: {json.dumps(feature_attribution, default=str)}
+
+You are at FALLBACK LEVEL {fallback_level}. Generate {artifact_type} based on the available sensory evidence. NEVER return INSUFFICIENT_EVIDENCE. Always produce a CODE section with runnable Python."""
+
+    providers = _get_available_providers()
+    timestamp_start = time.time()
+
+    all_outputs = []
+    errors = []
+
+    for provider in providers:
+        ts = time.time()
+        try:
+            output = _call_builder_provider(provider, prompt)
+            code = _extract_section(output, "CODE")
+            # Strip markdown code fences
+            if code.startswith("```"):
+                code = "\n".join(code.split("\n")[1:])
+            if code.endswith("```"):
+                code = code.rsplit("```", 1)[0]
+            all_outputs.append({
+                "provider": provider,
+                "output": output,
+                "code_extracted": code.strip(),
+                "code_lines": len([l for l in code.strip().split("\n") if l.strip()]),
+                "timestamp": ts,
+                "duration_ms": int((time.time() - ts) * 1000),
+                "error": None,
+            })
+        except Exception as e:
+            errors.append({
+                "provider": provider,
+                "error": str(e)[:200],
+                "timestamp": ts,
+            })
+
+    if not all_outputs:
+        raise RuntimeError(f"All builder providers failed: {errors}")
+
+    if len(all_outputs) == 1:
+        best = all_outputs[0]
+        patch_hash = sha256_text(best["output"] + str(time.time()))
+        receipt = {
+            "receipt_type": "PATCH_RECEIPT_V1",
+            "patch_hash": patch_hash,
+            "session_id": state.session_id,
+            "timestamp": time.time(),
+            "derived_from": {
+                "frame_hashes": state.frame_hashes(),
+                "audio_chunk_hashes": state.audio_chunk_hashes(),
+                "speaker_segments": state.speaker_segments(),
+                "observer_state_hash": observation.get("state_hash", ""),
+            },
+            "reason_codes": _extract_reasons(state, observation),
+            "uncertainty": _extract_uncertainty(observation),
+            "mode": state.mode,
+            "provider": best["provider"],
+            "fallback_level": fallback_level,
+            "artifact_type": artifact_type,
+            "sensory_channels": sensory_channels,
+            "feature_attribution": feature_attribution,
+            "consensus": {
+                "verdict": "SINGLE_PROVIDER",
+                "providers_used": [best["provider"]],
+            },
+        }
+        return {
+            "consensus_verdict": "SINGLE_PROVIDER",
+            "patch_output": best["output"],
+            "patch_hash": patch_hash,
+            "receipt": receipt,
+            "all_outputs": all_outputs,
+            "errors": errors,
+            "disagreements": [],
+            "similarity_matrix": {},
+            "fallback_level": fallback_level,
+            "artifact_type": artifact_type,
+        }
+
+    # Compare code outputs pairwise
+    n = len(all_outputs)
+    similarity_matrix = {}
+    disagreements = []
+
+    for i in range(n):
+        for j in range(i + 1, n):
+            code_a = all_outputs[i]["code_extracted"]
+            code_b = all_outputs[j]["code_extracted"]
+            prov_a = all_outputs[i]["provider"]
+            prov_b = all_outputs[j]["provider"]
+
+            code_sim = _code_similarity(code_a, code_b)
+
+            key = f"{prov_a}_vs_{prov_b}"
+            similarity_matrix[key] = {
+                "code_similarity": round(code_sim, 3),
+                "lines_a": all_outputs[i]["code_lines"],
+                "lines_b": all_outputs[j]["code_lines"],
+            }
+
+            if code_sim < 0.3:
+                disagreements.append({
+                    "providers": [prov_a, prov_b],
+                    "type": "different_code_approaches",
+                    "similarity": round(code_sim, 3),
+                    "detail": "Providers produced substantially different code implementations",
+                })
+            elif code_sim < 0.6:
+                disagreements.append({
+                    "providers": [prov_a, prov_b],
+                    "type": "partial_code_divergence",
+                    "similarity": round(code_sim, 3),
+                    "detail": "Providers produced similar but not identical code",
+                })
+
+    avg_sim = sum(s["code_similarity"] for s in similarity_matrix.values()) / len(similarity_matrix) if similarity_matrix else 0
+    if avg_sim >= 0.6:
+        verdict = "AGREED"
+    elif avg_sim >= 0.3:
+        verdict = "PARTIAL_AGREEMENT"
+    else:
+        verdict = "DISAGREEMENT"
+
+    # Select best output: prefer longest code (most complete implementation)
+    best = max(all_outputs, key=lambda x: x["code_lines"])
+
+    patch_hash = sha256_text(best["output"] + str(time.time()))
+    receipt = {
+        "receipt_type": "PATCH_RECEIPT_V1",
+        "patch_hash": patch_hash,
+        "session_id": state.session_id,
+        "timestamp": time.time(),
+        "derived_from": {
+            "frame_hashes": state.frame_hashes(),
+            "audio_chunk_hashes": state.audio_chunk_hashes(),
+            "speaker_segments": state.speaker_segments(),
+            "observer_state_hash": observation.get("state_hash", ""),
+        },
+        "reason_codes": _extract_reasons(state, observation),
+        "uncertainty": _extract_uncertainty(observation),
+        "mode": state.mode,
+        "provider": best["provider"],
+        "fallback_level": fallback_level,
+        "artifact_type": artifact_type,
+        "sensory_channels": sensory_channels,
+        "feature_attribution": feature_attribution,
+        "consensus": {
+            "verdict": verdict,
+            "providers_used": [o["provider"] for o in all_outputs],
+            "avg_code_similarity": round(avg_sim, 3),
+            "disagreements_count": len(disagreements),
+            "selected_provider": best["provider"],
+            "selection_criteria": "most_code_lines",
+        },
+    }
+
+    return {
+        "consensus_verdict": verdict,
+        "patch_output": best["output"],
+        "patch_hash": patch_hash,
+        "receipt": receipt,
+        "all_outputs": all_outputs,
+        "errors": errors,
+        "disagreements": disagreements,
+        "similarity_matrix": similarity_matrix,
+        "avg_similarity": round(avg_sim, 3),
+        "providers_used": [o["provider"] for o in all_outputs],
+        "fallback_level": fallback_level,
+        "artifact_type": artifact_type,
+        "timestamp": time.time(),
+        "duration_ms": int((time.time() - timestamp_start) * 1000),
+    }