Upload folder using huggingface_hub

Dockerfile

@@ -16,7 +16,7 @@ COPY frontend/ ./frontend/
 RUN mkdir -p /app/data/receipts
 
 ENV PORT=7860
-ENV PROVIDER=huggingface
+ENV PROVIDER=grok
 EXPOSE 7860
 
 CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -57,7 +57,7 @@ def health():
         "ok": True,
         "app": "csc-engine",
         "version": "1.0.0",
-        "provider": os.getenv("PROVIDER", "huggingface"),
+        "provider": os.getenv("PROVIDER", "grok"),
         "sessions_active": len(sessions),
         "time": datetime.now(timezone.utc).isoformat(),
     }

frontend/index.html

@@ -84,7 +84,7 @@
   <header>
     <div>
       <h1>CSC <span>Engine</span></h1>
-      <div class="sub">Continuity Sensory Code Engine — camera + audio + speech → observer → builder → code patches</div>
+      <div class="sub">Continuity Sensory Code Engine — camera + audio → LLM infers intent → generates code from movement & sound</div>
     </div>
     <div id="status"><span class="status-dot dot-idle"></span>Idle</div>
   </header>
@@ -102,14 +102,8 @@
         <option value="research_prism">research_prism</option>
       </select>
     </label>
-    <label>Instruction
-      <textarea id="instruction" rows="1" placeholder="What do you want to build?"></textarea>
-    </label>
     <label>Frame ms
-      <input id="frameMs" value="1200" style="width:60px" />
-    </label>
-    <label>Synth sec
-      <input id="synthSec" value="8" style="width:40px" />
+      <input id="frameMs" value="1500" style="width:60px" />
     </label>
   </div>
 
@@ -147,7 +141,7 @@
     <div class="pane" id="pane-patches">
       <h2>Generated Code Patches</h2>
       <div id="patchStream">
-        <div style="color:#6e7681;font-size:13px;padding:20px;text-align:center">No patches yet. Camera + LLM will generate code when QVD crosses threshold.</div>
+        <div style="color:#6e7681;font-size:13px;padding:20px;text-align:center">No patches yet. Camera movement or sound will trigger code generation.</div>
       </div>
     </div>
   </div>
@@ -252,18 +246,10 @@ function sendFrame(force = false) {
   ws.send(JSON.stringify({
     type: "frame",
     frame: dataUrl,
-    instruction: document.getElementById("instruction").value,
     mode: document.getElementById("mode").value,
     auto: true,
     force,
-    synthesis_interval: Number(document.getElementById("synthSec").value || 8)
   }));
-  if (document.getElementById("instruction").value.trim()) {
-    ws.send(JSON.stringify({
-      type: "transcript",
-      text: document.getElementById("instruction").value
-    }));
-  }
 }
 
 async function startLive() {
@@ -296,8 +282,7 @@ function startSpeech() {
       text += event.results[i][0].transcript + " ";
     }
     text = text.trim();
-    document.getElementById("instruction").value = text;
-    if (ws && ws.readyState === WebSocket.OPEN) {
+    if (text && ws && ws.readyState === WebSocket.OPEN) {
       ws.send(JSON.stringify({ type: "transcript", text }));
     }
   };
@@ -337,14 +322,26 @@ function addPatchEntry(output, hash, receipt, qvd) {
   const div = document.createElement("div");
   div.className = "patch-entry";
   const ts = new Date().toLocaleTimeString();
-  div.innerHTML = `
-    <div class="meta">
-      <span>${ts}</span>
-      <span>hash: ${hash.slice(0, 12)}</span>
-      ${qvd !== undefined ? `<span>QVD: ${qvd.toFixed(3)}</span>` : ''}
-    </div>
-    <pre>${escapeHtml(output)}</pre>
-  `;
+  const reasoningMatch = output.match(/REASONING:?\s*([\s\S]*?)(?=\n\s*(?:CODE|RUN|TEST|ATTRIBUTION|INSUFFICIENT)|$)/i);
+  const codeMatch = output.match(/CODE:?\s*([\s\S]*?)(?=\n\s*(?:RUN|TEST|ATTRIBUTION)|$)/i);
+  const reasoning = reasoningMatch ? reasoningMatch[1].trim() : '';
+  const code = codeMatch ? codeMatch[1].trim() : '';
+
+  let html = `<div class="meta"><span>${ts}</span><span>hash: ${hash.slice(0, 12)}</span>`;
+  if (qvd !== undefined) html += `<span>QVD: ${qvd.toFixed(3)}</span>`;
+  html += `</div>`;
+
+  if (reasoning) {
+    html += `<div style="color:#00d4ff;font-size:11px;text-transform:uppercase;margin:6px 0 3px">Reasoning</div>`;
+    html += `<pre style="color:#8b949e;font-style:italic">${escapeHtml(reasoning)}</pre>`;
+  }
+  if (code) {
+    html += `<div style="color:#34d399;font-size:11px;text-transform:uppercase;margin:8px 0 3px">Code</div>`;
+    html += `<pre style="color:#e8ecf4">${escapeHtml(code)}</pre>`;
+  } else {
+    html += `<pre>${escapeHtml(output)}</pre>`;
+  }
+  div.innerHTML = html;
   patchStream.prepend(div);
 }
 

src/builder_llm.py

@@ -15,16 +15,19 @@ from .stream_state import SessionState, sha256_text
 BUILDER_PROMPT = """You are the Builder in a Continuity Sensory Code Engine.
 
 You receive an observation from the Observer LLM and generate:
-1. CODE: complete runnable code or patch
-2. RUN: exact run command
-3. TEST: a minimal test or verification step
-4. ATTRIBUTION: which sensory evidence triggered this code
+1. REASONING: Think step by step about what code to write based on the observer's inferred intent and sensory evidence.
+2. CODE: complete runnable Python code or patch
+3. RUN: exact run command
+4. TEST: a minimal test or verification step
+5. ATTRIBUTION: which sensory evidence (camera movement, sound, visual scene) triggered this code
 
 Rules:
 - Do not execute code automatically.
 - Do not generate malware, surveillance, or credential theft.
-- Cite the evidence (frame hashes, audio segments) that informed your output.
+- The code must be directly correlated with what the camera saw or what audio detected.
+- If the observer reported movement or sound, your code should relate to that evidence.
 - If the observation is insufficient, return INSUFFICIENT_EVIDENCE.
+- Always show reasoning before code.
 - Be concise and practical.
 """
 
@@ -51,6 +54,19 @@ def call_openai(prompt: str) -> str:
     return response.choices[0].message.content
 
 
+def call_grok(prompt: str) -> str:
+    """Call x.ai Grok API for code generation."""
+    from openai import OpenAI
+    model = os.getenv("GROK_TEXT_MODEL", "grok-2-1212")
+    client = OpenAI(api_key=os.getenv("GROK_API_KEY"), base_url="https://api.x.ai/v1")
+    response = client.chat.completions.create(
+        model=model,
+        messages=[{"role": "user", "content": prompt}],
+        max_tokens=2000,
+    )
+    return response.choices[0].message.content
+
+
 def call_hf_inference(prompt: str) -> str:
     """Call HF Inference API for code generation."""
     from huggingface_hub import InferenceClient
@@ -76,20 +92,21 @@ def build(state: SessionState, observation: dict) -> dict:
 OBSERVER OUTPUT:
 {observation.get('observer_output', '')}
 
-USER INSTRUCTION: {state.user_instruction or '[none]'}
 MODE: {state.mode}
 
-Generate a code patch now."""
+The observer has inferred the intent from camera + audio. Generate code based on that inferred intent now."""
 
-    provider = os.getenv("PROVIDER", "huggingface").lower().strip()
+    provider = os.getenv("PROVIDER", "grok").lower().strip()
     if provider == "ollama":
         output = call_ollama(prompt)
     elif provider == "openai":
         output = call_openai(prompt)
+    elif provider == "grok":
+        output = call_grok(prompt)
     elif provider == "huggingface":
         output = call_hf_inference(prompt)
     else:
-        raise RuntimeError(f"PROVIDER must be 'ollama', 'openai', or 'huggingface', got: {provider!r}.")
+        raise RuntimeError(f"PROVIDER must be 'grok', 'ollama', 'openai', or 'huggingface', got: {provider!r}.")
 
     # Build patch receipt
     patch_hash = sha256_text(output + str(time.time()))

src/observer_llm.py

@@ -11,22 +11,29 @@ import requests
 from .stream_state import SessionState
 
 
+GROK_API_KEY = os.getenv("GROK_API_KEY", "")
+GROK_BASE_URL = "https://api.x.ai/v1"
+
+
 OBSERVER_PROMPT = """You are the Observer in a Continuity Sensory Code Engine.
 
 You receive a compressed sensory state from live camera + audio streams.
-Your job is to produce a compact observation:
+You infer intent from what the camera sees and what audio detects — the user does NOT type instructions.
+Your job is to produce a visible reasoning trace, then a compact observation:
 
-1. SCENE: what is visible (objects, screens, code, people, environment)
-2. INTENT: what the user seems to want
-3. SIGNALS: what audio/speech signals are present, separated by speaker
-4. CANDIDATE_TASK: what code/artifact should be generated
-5. UNCERTAINTY: what you cannot determine from the evidence
+1. REASONING: Think step by step about what you see and hear. What is the user doing? What are they looking at? What code problem is visible? What is the environment?
+2. SCENE: what is visible (objects, screens, code, people, environment)
+3. INFERRED_INTENT: what the user seems to want — infer this from the visual + audio evidence, not from typed instructions
+4. SIGNALS: what audio/speech signals are present, separated by speaker
+5. CANDIDATE_TASK: what code/artifact should be generated based on the inferred intent
+6. UNCERTAINTY: what you cannot determine from the evidence
 
 Rules:
 - Do not hallucinate. If evidence is insufficient, say so.
 - Separate user speech from background audio.
 - Do not generate code. That is the Builder's job.
-- Be concise. This is a state summary, not a chat response.
+- Always show your reasoning first, then the structured observation.
+- If there is camera movement or sound, that IS the signal — infer what the user wants from it.
 """
 
 
@@ -60,6 +67,25 @@ def call_openai(prompt: str, frame_b64: str | None = None) -> str:
     return response.choices[0].message.content
 
 
+def call_grok(prompt: str, frame_b64: str | None = None) -> str:
+    """Call x.ai Grok API (OpenAI-compatible) with vision support."""
+    from openai import OpenAI
+    model = os.getenv("GROK_MODEL", "grok-2-vision-1212")
+    client = OpenAI(api_key=os.getenv("GROK_API_KEY"), base_url="https://api.x.ai/v1")
+    content = [{"type": "text", "text": prompt}]
+    if frame_b64:
+        content.append({
+            "type": "image_url",
+            "image_url": {"url": f"data:image/jpeg;base64,{frame_b64}"},
+        })
+    response = client.chat.completions.create(
+        model=model,
+        messages=[{"role": "user", "content": content}],
+        max_tokens=1000,
+    )
+    return response.choices[0].message.content
+
+
 def call_hf_inference(prompt: str, frame_b64: str | None = None) -> str:
     """Call HF Inference API using huggingface_hub InferenceClient.
 
@@ -96,15 +122,17 @@ COMPRESSED SENSORY STATE:
 
 Produce your observation now."""
 
-    provider = os.getenv("PROVIDER", "huggingface").lower().strip()
+    provider = os.getenv("PROVIDER", "grok").lower().strip()
     if provider == "ollama":
         output = call_ollama(prompt, frame_b64)
     elif provider == "openai":
         output = call_openai(prompt, frame_b64)
+    elif provider == "grok":
+        output = call_grok(prompt, frame_b64)
     elif provider == "huggingface":
         output = call_hf_inference(prompt, frame_b64)
     else:
-        raise RuntimeError(f"PROVIDER must be 'ollama', 'openai', or 'huggingface', got: {provider!r}.")
+        raise RuntimeError(f"PROVIDER must be 'grok', 'ollama', 'openai', or 'huggingface', got: {provider!r}.")
 
     return {
         "observer_output": output,