Update script1.js

script1.js

@@ -1,6 +1,8 @@
+// script1.js
+
 // Constants and Configuration
 const USER_SPEECH_INTERRUPT_DELAY = 500;
-const TEXT_TO_SPEECH_API_ENDPOINT = "https://api.streamelements.com/kappa/v2/speech"; // Replace with your TTS endpoint
+const TEXT_TO_SPEECH_API_ENDPOINT = "https://api.streamelements.com/kappa/v2/speech";
 const CHUNK_SIZE = 300;
 const MAX_PREFETCH_REQUESTS = 10;
 const PREFETCH_CACHE_EXPIRATION = 60000; // 1 minute
@@ -10,7 +12,7 @@ const AUDIO_CACHE_EXPIRATION = 3600000; // 1 hour
 const startStopButton = document.getElementById('startStopButton');
 const voiceSelectionDropdown = document.getElementById('voiceSelect');
 const modelSelectionDropdown = document.getElementById('modelSelect');
-const noiseSuppressionCheckbox = document.getElementById('noiseSuppression'); // Assuming you have this in your HTML
+const noiseSuppressionCheckbox = document.getElementById('noiseSuppression');
 const responseTimeDisplay = document.getElementById('responseTime');
 const userActivityIndicator = document.getElementById('userIndicator');
 const aiActivityIndicator = document.getElementById('aiIndicator');
@@ -44,19 +46,6 @@ let conversationHistory = [];
 // Audio Caching
 const audioCache = new Map();
 
-// Webcam and Gradio Integration
-import { client, handle_file } from 'https://cdn.jsdelivr.net/npm/@gradio/client/+esm';
-const video = document.getElementById('webcam');
-const clients = [
-    "multimodalart/Florence-2-l4",
-    "gokaygokay/Florence-2",
-    "multimodalart/Florence-2-l4-2",
-    "gokaygokay/Florence-2",
-]; // Or your preferred Gradio models
-let app;
-let lastCaption = "";
-
-
 // Utility Functions
 
 // Normalize query text 
@@ -209,7 +198,7 @@ const cancelPrefetchRequests = (query) => {
 // AI Interaction Functions
 
 // Send a query to the AI
-const sendQueryToAI = async (query) => {
+async function sendQueryToAI(query) {
     console.log("Sending query to AI:", query);
     isRequestInProgress = true;
     updateActivityIndicators();
@@ -234,7 +223,8 @@ const sendQueryToAI = async (query) => {
     requestAbortController = new AbortController();
 
     try {
-        await streamAndHandleAudioResponse(query, voiceSelectionDropdown.value, requestAbortController.signal);
+        const combinedQuery = `{USER: "${query}"}, ${lastCaption}, {USER: "${query}"}`;
+        await streamAndHandleAudioResponse(combinedQuery, voiceSelectionDropdown.value, requestAbortController.signal);
     } catch (error) {
         if (error.name !== 'AbortError') {
             console.error("Error sending query to AI:", error);
@@ -543,20 +533,32 @@ if ('webkitSpeechRecognition' in window) {
 
 setInterval(updateLatency, 100);
 
-// Webcam Functions
+
+
+// Webcam Integration
+import { client, handle_file } from 'https://cdn.jsdelivr.net/npm/@gradio/client/+esm';
+
+const video = document.getElementById('webcam');
+let app;
+let lastCaption = ""; 
+
+const clients = [
+    "multimodalart/Florence-2-l4",
+    "gokaygokay/Florence-2",
+    "multimodalart/Florence-2-l4-2",
+    "gokaygokay/Florence-2",
+];
 
 async function startWebcam() {
     try {
         const stream = await navigator.mediaDevices.getUserMedia({ video: true });
         video.srcObject = stream;
-        setInterval(captureAndProcessImage, 5000); // Adjust interval as needed
+        setInterval(captureAndProcessImage, 5000); 
     } catch (error) {
         console.error("Error accessing webcam: ", error);
-        // Consider adding user feedback here, e.g., alert or display a message.
     }
 }
 
-
 async function captureAndProcessImage() {
     const canvas = document.createElement('canvas');
     canvas.width = video.videoWidth;
@@ -568,7 +570,6 @@ async function captureAndProcessImage() {
     await processWithGradio(blob);
 }
 
-
 async function processWithGradio(imageBlob) {
     try {
         const randomClient = clients[Math.floor(Math.random() * clients.length)];
@@ -577,57 +578,13 @@ async function processWithGradio(imageBlob) {
 
         const result = await app.predict("/process_image", [handledFile, "Detailed Caption"]);
 
-        const dataString = result.data[0]; // Assuming the caption is the first element in the response
-        lastCaption = dataString || ""; // Handle potential errors
+        const dataString = result.data[0];
+        lastCaption = dataString || lastCaption; 
     } catch (error) {
         console.error("Error processing with Gradio:", error);
-        // Add error handling here (e.g., display a message to the user).
-        lastCaption = ""; // Reset caption if there's an error.
     }
 }
 
-
-
-// Modify sendQueryToAI to include the caption
-async function sendQueryToAI(query) {
-    console.log("Sending query to AI:", query);
-    isRequestInProgress = true;
-    updateActivityIndicators();
-    firstResponseTextTimestamp = null;
-
-    const normalizedQuery = normalizeQueryText(query);
-    const cacheKey = generateCacheKey(normalizedQuery, modelSelectionDropdown.value, conversationHistory, modelSelectionDropdown.value);
-
-    queryStartTime = Date.now();
-
-    // Check prefetch cache
-    if (prefetchCache.has(cacheKey)) {
-        const cachedData = prefetchCache.get(cacheKey);
-        if (Date.now() - cachedData.timestamp < PREFETCH_CACHE_EXPIRATION) {
-            audioPlaybackQueue.push({ url: cachedData.url, isPrefetched: true });
-            playNextAudio();
-        } else {
-            prefetchCache.delete(cacheKey);
-        }
-    }
-
-    requestAbortController = new AbortController();
-
-    try {
-        const combinedQuery = `{USER: "${query}"}, ${lastCaption}, {USER: "${query}"}`;
-        await streamAndHandleAudioResponse(combinedQuery, voiceSelectionDropdown.value, requestAbortController.signal);
-    } catch (error) {
-        if (error.name !== 'AbortError') {
-            console.error("Error sending query to AI:", error);
-        }
-    } finally {
-        isRequestInProgress = false;
-        updateActivityIndicators();
-    }
-};
-
-
-// Initialize Webcam and Speech Recognition on Load
 window.onload = () => {
     startWebcam(); 
 };