reduced the token cound and redundant chat conversation

.gitignore

@@ -26,3 +26,4 @@ deepshell.log
 docs/
 libretranslate.pid
 deepshell-backend/deepshell/logs/
+deepshell-backend/DEEPSHELL_ACTIONS_DESIGN.md

app.js.bak

@@ -0,0 +1,561 @@
+// Plain Vanilla Chat Frontend with Copy Button, SSE Streaming, Export, and Clear
+// Global mode management
+window.setMode = function(mode) {
+    window.currentMode = mode;
+    document.getElementById("btn-assistant").className = "mode-btn" + (mode === "assistant" ? " active" : "");
+    document.getElementById("btn-trainer").className = "mode-btn" + (mode === "trainer" ? " trainer-active" : "");
+
+    const label = document.getElementById("mode-label");
+    if(label) label.textContent = mode === "trainer" ? "— Trainer Mode 🎓" : "";
+    const output = document.getElementById("chat-output");
+    if(output) output.innerHTML = "";
+    fetch("/chat/clear", { method: "POST" });
+};
+window.currentMode = "assistant";
+
+window.toggleTheme = function() {
+    const root = document.documentElement;
+    const btn = document.getElementById("theme-btn");
+    const isLight = root.classList.toggle("light");
+    btn.textContent = isLight ? "🌙" : "☀️";
+    btn.title = isLight ? "Switch to dark theme" : "Switch to light theme";
+    localStorage.setItem("deepshell-theme", isLight ? "light" : "dark");
+    // Swap Prism theme
+    const darkTheme = document.getElementById("prism-theme");
+    const lightTheme = document.getElementById("prism-light-theme");
+    if (darkTheme && lightTheme) {
+        darkTheme.disabled = isLight;
+        lightTheme.disabled = !isLight;
+    }
+};
+
+// Apply saved Prism theme on load
+document.addEventListener("DOMContentLoaded", function() {
+    const saved = localStorage.getItem("deepshell-theme");
+    if (saved === "light") {
+        const darkTheme = document.getElementById("prism-theme");
+        const lightTheme = document.getElementById("prism-light-theme");
+        if (darkTheme && lightTheme) {
+            darkTheme.disabled = true;
+            lightTheme.disabled = false;
+        }
+        const btn = document.getElementById("theme-btn");
+        if (btn) btn.textContent = "🌙";
+    }
+});
+
+// Apply saved theme on load
+(function() {
+    const saved = localStorage.getItem("deepshell-theme");
+    if (saved === "light") {
+        document.documentElement.classList.add("light");
+        document.addEventListener("DOMContentLoaded", function() {
+            const btn = document.getElementById("theme-btn");
+            if (btn) { btn.textContent = "🌙"; }
+        });
+    }
+})();
+window.ttsEnabled = false;
+window.lastSpokenText = "";
+
+window.ttsLang = 'en-US';
+window.setLang = function(lang) {
+    window.ttsLang = lang;
+    window.speechSynthesis.cancel();
+};
+window.toggleTTS = function() {
+    window.ttsEnabled = !window.ttsEnabled;
+    const btn = document.getElementById("tts-btn");
+    if (window.ttsEnabled) {
+        btn.classList.add("tts-active");
+        btn.title = "Voice ON — click to disable";
+        btn.innerHTML = "🔊 Voice ON";
+    } else {
+        btn.classList.remove("tts-active");
+        btn.title = "Voice OFF — click to enable";
+        btn.innerHTML = "🔇 Voice OFF";
+        window.speechSynthesis.cancel();
+    }
+};
+
+window.replayLast = function() {
+    if (window.lastSpokenText) {
+        window.speakText(window.lastSpokenText);
+    }
+};
+
+window.speakText = function(text) {
+    if (!window.ttsEnabled || window.currentMode !== "trainer") return;
+    window.lastSpokenText = text;
+    window.speechSynthesis.cancel();
+    // Strip markdown symbols for cleaner speech
+    const clean = text
+        .replace(/#{1,6}\s/g, "")
+        .replace(/\*\*/g, "")
+        .replace(/\*/g, "")
+        .replace(/```[\s\S]*?```/g, "")
+        .replace(/`([^`]+)`/g, "$1")
+	.replace(/🎯|💻|🔍|⚠️|🚀|💡/g, "")
+        .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
+        .trim();
+    // Non-English: translate via backend then speak via Piper TTS
+    if (window.ttsLang !== 'en-US' && window.ttsLang !== 'en-GB') {
+        const langCode = window.ttsLang.split('-')[0];
+        const sentences = clean.match(/[^.!?]+[.!?]+/g) || [clean];
+        let i = 0;
+        function translateAndSpeakNext() {
+            if (i >= sentences.length || !window.ttsEnabled) return;
+            const sentence = sentences[i].trim();
+            fetch('/translate', {
+                method: 'POST',
+                headers: {'Content-Type': 'application/json'},
+                body: JSON.stringify({text: sentence, target: langCode})
+            })
+                .then(r => r.json())
+                .then(data => {
+                    const translated = data.translatedText || sentence;
+                    return fetch('/tts', {
+                        method: 'POST',
+                        headers: {'Content-Type': 'application/json'},
+                        body: JSON.stringify({text: translated, lang: langCode})
+                    });
+                })
+                .then(r => r.blob())
+                .then(blob => {
+                    const url = URL.createObjectURL(blob);
+                    const audio = new Audio(url);
+                    audio.onended = () => { URL.revokeObjectURL(url); i++; translateAndSpeakNext(); };
+                    // Small delay on first sentence to avoid audio startup cutoff
+                    const delay = i === 0 ? 300 : 0;
+                    setTimeout(() => audio.play(), delay);
+                })
+                .catch(() => {
+                    const utt = new SpeechSynthesisUtterance(sentence);
+                    utt.lang = window.ttsLang;
+                    utt.onend = () => { i++; translateAndSpeakNext(); };
+                    window.speechSynthesis.speak(utt);
+                });
+        }
+        translateAndSpeakNext();
+        return;
+    }
+    window._speakDirect(clean);
+};
+window._speakDirect = function(clean) {
+    const chunks = clean.match(/[^.!?]+[.!?]+/g) || [clean];
+    let i = 0;
+    function speakNext() {
+        if (i >= chunks.length || !window.ttsEnabled) return;
+        const utt = new SpeechSynthesisUtterance(chunks[i].trim());
+        utt.lang = window.ttsLang;
+        const voices = window.speechSynthesis.getVoices();
+        const match = voices.find(v => v.lang === window.ttsLang);
+        if (match) utt.voice = match;
+        utt.rate = 0.78;
+        utt.pitch = 1.0;
+        utt.volume = 1.0;
+        utt.onend = () => { i++; speakNext(); };
+        window.speechSynthesis.speak(utt);
+    }
+    speakNext();
+};
+
+(function () {
+  "use strict";
+
+
+  // ---------- Utilities ----------
+  function escapeHtml(text) {
+    return String(text).replace(/[&<>"']/g, (m) => ({
+      "&": "&amp;",
+      "<": "&lt;",
+      ">": "&gt;",
+      '"': "&quot;",
+      "'": "&#39;",
+    })[m]);
+  }
+
+  function renderMarkdown(md) {
+    if (window.marked) {
+      const renderer = new marked.Renderer();
+      renderer.code = function(code, lang) {
+        const language = lang || 'bash';
+        const validLang = Prism.languages[language] ? language : 'bash';
+        return `<pre><code class="language-${validLang}">${escapeHtml(code)}</code></pre>`;
+      };
+      return marked.parse(md, { renderer });
+    }
+    return `<pre>${escapeHtml(md)}</pre>`;
+  }
+
+  function createCopyButton(code) {
+    const btn = document.createElement("button");
+    btn.className = "copy-btn";
+    btn.textContent = "Copy";
+    btn.onclick = async () => {
+      try {
+        await navigator.clipboard.writeText(code);
+        btn.textContent = "Copied!";
+        setTimeout(() => { btn.textContent = "Copy"; }, 2000);
+      } catch (err) {
+        const textarea = document.createElement("textarea");
+        textarea.value = code;
+        textarea.style.position = "fixed";
+        textarea.style.opacity = "0";
+        document.body.appendChild(textarea);
+        textarea.select();
+        try {
+          document.execCommand("copy");
+          btn.textContent = "Copied!";
+          setTimeout(() => { btn.textContent = "Copy"; }, 2000);
+        } catch (e) {
+          btn.textContent = "Failed";
+          setTimeout(() => { btn.textContent = "Copy"; }, 2000);
+        }
+        document.body.removeChild(textarea);
+      }
+    };
+    return btn;
+  }
+
+  function createExecuteButton(command, output) {
+    const btn = document.createElement("button");
+    btn.className = "execute-btn";
+    btn.textContent = "Execute";
+    btn.onclick = async () => {
+      btn.disabled = true;
+      btn.textContent = "Executing...";
+
+      try {
+        const response = await fetch("/chat/execute", {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({ command }),
+        });
+
+        const result = await response.json();
+
+        if (result.success) {
+          const outputText = result.stdout || result.stderr || "(no output)";
+          appendMessage(output, "Command Output", `Exit Code: ${result.exit_code}\n\n${outputText}`, false);
+          btn.textContent = "✓ Executed";
+          btn.style.background = "#22c55e";
+        } else {
+          appendMessage(output, "Execution Error", result.error || "Unknown error", false);
+          btn.textContent = "✗ Failed";
+          btn.style.background = "#ef4444";
+        }
+      } catch (err) {
+        appendMessage(output, "Error", `Failed to execute: ${err.message}`, false);
+        btn.textContent = "✗ Error";
+        btn.style.background = "#ef4444";
+      }
+
+      setTimeout(() => {
+        btn.disabled = false;
+        btn.textContent = "Execute";
+        btn.style.background = "";
+      }, 3000);
+    };
+    return btn;
+  }
+
+  function extractExecuteTags(text) {
+    const regex = /<execute>(.*?)<\/execute>/gs;
+    const commands = [];
+    let match;
+
+    while ((match = regex.exec(text)) !== null) {
+      commands.push(match[1].trim());
+    }
+
+    return commands;
+  }
+
+  // Add copy buttons to all code blocks in a container
+  function addCopyButtonsToCodeBlocks(container) {
+    container.querySelectorAll("pre").forEach((pre) => {
+      if (pre.parentElement && pre.parentElement.classList.contains("code-wrapper")) {
+        return;
+      }
+
+      const codeEl = pre.querySelector("code");
+      const code = codeEl ? codeEl.textContent : pre.textContent;
+
+      if (!code || !code.trim()) return;
+
+      const wrapper = document.createElement("div");
+      wrapper.className = "code-wrapper";
+
+      const copyBtn = createCopyButton(code);
+
+      pre.parentNode.insertBefore(wrapper, pre);
+
+      wrapper.appendChild(copyBtn);
+      wrapper.appendChild(pre);
+    });
+  }
+
+  function appendMessage(container, sender, message, useMarkdown = false, isError = false) {
+    const msgDiv = document.createElement("div");
+    msgDiv.classList.add("message");
+    if (sender === "You") msgDiv.classList.add("user-msg");
+    if (isError) msgDiv.classList.add("error");
+
+    const commands = extractExecuteTags(message);
+
+    if (commands.length > 0) {
+      let displayText = message.replace(/<execute>.*?<\/execute>/gs, "");
+
+      msgDiv.innerHTML = `<strong>${sender}:</strong>${useMarkdown ? renderMarkdown(displayText) : `<pre>${escapeHtml(displayText)}</pre>`}`;
+
+      commands.forEach(cmd => {
+        const cmdWrapper = document.createElement("div");
+        cmdWrapper.className = "command-wrapper";
+
+        const cmdPre = document.createElement("pre");
+        cmdPre.className = "command-block";
+        cmdPre.textContent = cmd;
+
+        const btnContainer = document.createElement("div");
+        btnContainer.className = "command-buttons";
+
+        const executeBtn = createExecuteButton(cmd, container);
+        const copyBtn = createCopyButton(cmd);
+
+        btnContainer.appendChild(executeBtn);
+        btnContainer.appendChild(copyBtn);
+
+        cmdWrapper.appendChild(cmdPre);
+        cmdWrapper.appendChild(btnContainer);
+        msgDiv.appendChild(cmdWrapper);
+      });
+    } else if (useMarkdown) {
+      msgDiv.innerHTML = `<strong>${sender}:</strong><div class="markdown-content">${renderMarkdown(String(message))}</div>`;
+      addCopyButtonsToCodeBlocks(msgDiv);
+    } else {
+      msgDiv.innerHTML = `<strong>${sender}:</strong> <pre>${escapeHtml(String(message))}</pre>`;
+    }
+
+    container.appendChild(msgDiv);
+    container.scrollTop = container.scrollHeight;
+
+    if (window.Prism) {
+      try {
+        Prism.highlightAllUnder(msgDiv);
+      } catch (_) {}
+    }
+
+    return msgDiv;
+  }
+
+  // ---------- Streaming UI ----------
+  function createStreamingMessage(container, sender) {
+    const msgDiv = document.createElement("div");
+    msgDiv.classList.add("message", "loading");
+    msgDiv.innerHTML = `<strong>${sender}:</strong> <span class="streaming-content"></span><span class="spinner"></span>`;
+
+    const contentSpan = msgDiv.querySelector(".streaming-content");
+
+    container.appendChild(msgDiv);
+    container.scrollTop = container.scrollHeight;
+
+    return {
+      msgDiv,
+      contentSpan,
+      updateContent: (text) => {
+        contentSpan.textContent = text;
+        container.scrollTop = container.scrollHeight;
+      },
+      finalize: (text) => {
+        msgDiv.remove();
+        appendMessage(container, sender, text, true);
+      }
+    };
+  }
+
+  // ---------- SSE Streaming ----------
+  async function sendPromptStreaming(prompt, output, sendBtn, input) {
+    const streamUI = createStreamingMessage(output, "DeepShell");
+    let fullText = "";
+
+    try {
+      const response = await fetch("/chat/run-agent-stream", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ prompt, mode: window.currentMode || "assistant" }),
+      });
+
+      if (!response.ok) {
+        throw new Error(`Server error: ${response.status}`);
+      }
+
+      const reader = response.body.getReader();
+      const decoder = new TextDecoder("utf-8");
+      let buffer = "";
+
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+
+        buffer += decoder.decode(value, { stream: true });
+
+        const lines = buffer.split("\n\n");
+        buffer = lines.pop();
+
+        for (const line of lines) {
+          if (!line.trim() || !line.startsWith("data: ")) continue;
+
+          const jsonStr = line.substring(6);
+
+          try {
+            const data = JSON.parse(jsonStr);
+
+            if (data.type === "token") {
+              fullText += data.text;
+              streamUI.updateContent(fullText);
+            } else if (data.type === "done") {
+              streamUI.finalize(fullText);
+              window.speakText(fullText);
+              return;
+            } else if (data.type === "error") {
+              throw new Error(data.message);
+            }
+          } catch (e) {
+            console.warn("Failed to parse SSE data:", jsonStr, e);
+          }
+        }
+      }
+
+      if (fullText) {
+        streamUI.finalize(fullText);
+      }
+    } catch (err) {
+      streamUI.msgDiv.remove();
+      appendMessage(output, "Error", `${err.message}`, false, true);
+      console.error("Streaming error:", err);
+    } finally {
+      if (sendBtn) sendBtn.disabled = false;
+      input.focus();
+    }
+  }
+
+  // ---------- Clear Chat ----------
+  async function clearChat(output) {
+    if (!confirm("Clear conversation history? This cannot be undone.")) {
+      return;
+    }
+
+    try {
+      const response = await fetch("/chat/clear", {
+        method: "POST",
+      });
+
+      if (response.ok) {
+        output.innerHTML = "";
+        appendMessage(output, "System", "Chat cleared. Session history reset.", false);
+      } else {
+        throw new Error("Failed to clear chat");
+      }
+    } catch (err) {
+      appendMessage(output, "Error", `Failed to clear chat: ${err.message}`, false, true);
+    }
+  }
+
+  // ---------- Export Chat ----------
+  function exportChat(output) {
+    const messages = output.querySelectorAll(".message");
+    if (messages.length === 0) {
+      alert("No messages to export!");
+      return;
+    }
+
+    let markdown = "# DeepShell Chat Export\n\n";
+    markdown += `**Exported:** ${new Date().toLocaleString()}\n\n---\n\n`;
+
+    messages.forEach((msg) => {
+      const sender = msg.querySelector("strong")?.textContent.replace(":", "") || "Unknown";
+      
+      // Get text content, excluding buttons
+      const clone = msg.cloneNode(true);
+      clone.querySelectorAll("button").forEach(btn => btn.remove());
+      
+      const content = clone.textContent
+        .replace(sender + ":", "")
+        .trim();
+
+      markdown += `## ${sender}\n\n${content}\n\n---\n\n`;
+    });
+
+    // Create download
+    const blob = new Blob([markdown], { type: "text/markdown" });
+    const url = URL.createObjectURL(blob);
+    const a = document.createElement("a");
+    a.href = url;
+    a.download = `deepshell-chat-${Date.now()}.md`;
+    document.body.appendChild(a);
+    a.click();
+    document.body.removeChild(a);
+    URL.revokeObjectURL(url);
+  }
+
+  // ---------- Chat wiring ----------
+  document.addEventListener("DOMContentLoaded", () => {
+    const form = document.getElementById("chat-form");
+    const input = document.getElementById("chat-input");
+    const output = document.getElementById("chat-output");
+    const sendBtn = document.getElementById("chat-send");
+    const clearBtn = document.getElementById("clear-btn");
+    const exportBtn = document.getElementById("export-btn");
+
+    if (!form || !input || !output) {
+      console.warn("Chat elements not found (chat-form/chat-input/chat-output).");
+      return;
+    }
+
+    // Send message
+    form.addEventListener("submit", (e) => {
+      e.preventDefault();
+      const prompt = input.value.trim();
+      if (!prompt) return;
+
+      // Hide welcome message on first user input
+      const welcome = document.getElementById("welcome-msg");
+      if (welcome) welcome.style.display = "none";
+      appendMessage(output, "You", prompt);
+      input.value = "";
+      if (sendBtn) sendBtn.disabled = true;
+
+      sendPromptStreaming(prompt, output, sendBtn, input);
+    });
+
+    if (sendBtn) {
+      sendBtn.addEventListener("click", (e) => {
+        e.preventDefault();
+        form.requestSubmit();
+      });
+    }
+
+    // Clear chat button
+    if (clearBtn) {
+      clearBtn.addEventListener("click", () => {
+        clearChat(output);
+      });
+    }
+
+    // Export chat button
+    if (exportBtn) {
+      exportBtn.addEventListener("click", () => {
+        exportChat(output);
+      });
+    }
+
+    // Enter to send, Shift+Enter for newline
+    input.addEventListener("keydown", (e) => {
+      if (e.key === "Enter" && !e.shiftKey) {
+        e.preventDefault();
+        form.requestSubmit();
+      }
+    });
+  });
+})();

deepshell-backend/deepshell/__main__.py

@@ -408,7 +408,7 @@ async def _sse_generator(prompt: str, mode: str = "assistant"):
     try:
         # Try streaming with stream=True
         try:
-            response = client.chat(full_prompt, max_tokens=4000, temperature=0.2, stream=True, system_prompt=system_prompt)
+            response = client.chat(full_prompt, max_tokens=2200, temperature=0.2, stream=True, system_prompt=system_prompt)
 
             # Handle streaming response
             if hasattr(response, "__iter__"):
@@ -427,7 +427,7 @@ async def _sse_generator(prompt: str, mode: str = "assistant"):
 
         except TypeError:
             # stream=True not supported, fallback to non-streaming
-            response = client.chat(full_prompt, max_tokens=4000, temperature=0.2, system_prompt=system_prompt)
+            response = client.chat(full_prompt, max_tokens=2200, temperature=0.2, system_prompt=system_prompt)
             try:
                 text = response.choices[0].message.content
             except Exception:

deepshell-backend/deepshell/llm.py

@@ -92,8 +92,10 @@ DIFFICULTY AWARENESS:
 When refusing off-topic questions:
 "I'm DeepShell Trainer, focused exclusively on Linux and DevOps education. For [topic], please use a general-purpose AI assistant. Ready to teach you something DevOps-related?"
 CONTENT LENGTH:
-Generate exactly 1300 words of spoken tutorial content.
-No more, no less. Cover the topic completely within that word count.
+Aim for roughly 1300-1600 words of spoken tutorial content, covering the
+topic completely. Stop naturally once you've given the pro tip for the
+last concept — do NOT add a summary, recap, or motivational closing
+section of any kind.
 """