Add real image input audio input and output for any media inferencing models

index.html

@@ -373,7 +373,6 @@
                         <label class="block text-slate-300 mb-2">Selected File</label>
                         <div id="previewContent" class="bg-slate-800 rounded-lg p-4"></div>
                     </div>
-                    
                     <!-- Media Controls -->
                     <div class="mb-6 hidden" id="mediaControls">
                         <label class="block text-slate-300 mb-3">Media Controls</label>
@@ -381,21 +380,68 @@
                             <div class="flex items-center justify-between">
                                 <span>Auto-play</span>
                                 <label class="relative inline-flex items-center cursor-pointer">
-                                    <input type="checkbox" class="sr-only peer">
+                                    <input type="checkbox" id="autoPlayToggle" class="sr-only peer">
                                     <div class="w-11 h-6 bg-gray-200 peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800 rounded-full peer dark:bg-gray-700 peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all dark:border-gray-600 peer-checked:bg-blue-600"></div>
                                 </label>
                             </div>
                             <div class="flex items-center justify-between">
                                 <span>Loop</span>
                                 <label class="relative inline-flex items-center cursor-pointer">
-                                    <input type="checkbox" class="sr-only peer" checked>
+                                    <input type="checkbox" id="loopToggle" class="sr-only peer" checked>
                                     <div class="w-11 h-6 bg-gray-200 peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800 rounded-full peer dark:bg-gray-700 peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all dark:border-gray-600 peer-checked:bg-blue-600"></div>
                                 </label>
                             </div>
                         </div>
                     </div>
                     
-                    <!-- Preprocessing Options -->
+                    <!-- Camera Stream -->
+                    <div class="mb-6 hidden" id="cameraSection">
+                        <label class="block text-slate-300 mb-2">Camera Stream</label>
+                        <video id="cameraStream" class="w-full max-h-48 bg-black rounded-lg" autoplay muted playsinline></video>
+                        <div class="flex space-x-2 mt-3">
+                            <button id="captureBtn" class="px-4 py-2 bg-indigo-600 hover:bg-indigo-700 rounded-lg text-sm transition flex items-center">
+                                <i data-feather="camera" class="mr-2"></i>Capture
+                            </button>
+                            <button id="stopCameraBtn" class="px-4 py-2 bg-red-600 hover:bg-red-700 rounded-lg text-sm transition flex items-center">
+                                <i data-feather="stop-circle" class="mr-2"></i>Stop
+                            </button>
+                        </div>
+                    </div>
+                    
+                    <!-- Microphone Stream -->
+                    <div class="mb-6 hidden" id="microphoneSection">
+                        <label class="block text-slate-300 mb-2">Audio Recording</label>
+                        <div id="audioLevel" class="bg-slate-800 rounded-lg p-4">
+                            <div class="flex items-center justify-between mb-2">
+                                <span class="text-sm text-slate-400">Recording Level</span>
+                                <span id="audioLevelValue" class="text-sm text-slate-300">0%</span>
+                            </div>
+                            <div class="h-2 bg-slate-700 rounded-full overflow-hidden">
+                                <div id="audioLevelBar" class="h-full bg-emerald-400 rounded-full transition-all duration-200" style="width: 0%"></div>
+                            </div>
+                        </div>
+                        <div class="flex space-x-2 mt-3">
+                            <button id="startRecordingBtn" class="px-4 py-2 bg-emerald-600 hover:bg-emerald-700 rounded-lg text-sm transition flex items-center">
+                                <i data-feather="mic" class="mr-2"></i>Start Recording
+                            </button>
+                            <button id="stopRecordingBtn" class="px-4 py-2 bg-red-600 hover:bg-red-700 rounded-lg text-sm transition flex items-center">
+                                <i data-feather="square" class="mr-2"></i>Stop
+                            </button>
+                        </div>
+                    </div>
+                    
+                    <!-- Captured Image -->
+                    <div class="mb-6 hidden" id="capturedImageSection">
+                        <label class="block text-slate-300 mb-2">Captured Image</label>
+                        <canvas id="capturedCanvas" class="w-full max-h-48 border border-slate-600 rounded-lg"></canvas>
+                    </div>
+                    
+                    <!-- Recorded Audio -->
+                    <div class="mb-6 hidden" id="recordedAudioSection">
+                        <label class="block text-slate-300 mb-2">Recorded Audio</label>
+                        <audio id="recordedAudio" class="w-full" controls></audio>
+                    </div>
+<!-- Preprocessing Options -->
                     <div class="mb-6" id="preprocessingSection">
                         <label class="block text-slate-300 mb-2">Preprocessing</label>
                         <div class="space-y-3" id="preprocessingOptions">
@@ -503,10 +549,20 @@
         let currentInputType = 'image';
         let modelInfo = {};
         
+        // Media streams
+        let cameraStream = null;
+        let microphoneStream = null;
+        let mediaRecorder = null;
+        let recordedChunks = [];
+        let audioContext = null;
+        let audioAnalyzer = null;
+        let audioLevelInterval = null;
+        let currentCapturedImage = null;
+        let currentRecordedAudioBlob = null;
+        
         // Initialize Feather Icons
         feather.replace();
-        
-        // DOM Elements
+// DOM Elements
         const modelTypeCards = document.querySelectorAll('[data-model-type]');
         const uploadSection = document.getElementById('uploadSection');
         const uploadTitle = document.getElementById('uploadTitle');
@@ -531,6 +587,24 @@
         const preprocessingSection = document.getElementById('preprocessingSection');
         const preprocessingOptions = document.getElementById('preprocessingOptions');
         
+        // New media elements
+        const cameraSection = document.getElementById('cameraSection');
+        const microphoneSection = document.getElementById('microphoneSection');
+        const capturedImageSection = document.getElementById('capturedImageSection');
+        const recordedAudioSection = document.getElementById('recordedAudioSection');
+        const cameraStream = document.getElementById('cameraStream');
+        const audioLevel = document.getElementById('audioLevel');
+        const audioLevelBar = document.getElementById('audioLevelBar');
+        const audioLevelValue = document.getElementById('audioLevelValue');
+        const capturedCanvas = document.getElementById('capturedCanvas');
+        const recordedAudio = document.getElementById('recordedAudio');
+        const captureBtn = document.getElementById('captureBtn');
+        const stopCameraBtn = document.getElementById('stopCameraBtn');
+        const startRecordingBtn = document.getElementById('startRecordingBtn');
+        const stopRecordingBtn = document.getElementById('stopRecordingBtn');
+        const autoPlayToggle = document.getElementById('autoPlayToggle');
+        const loopToggle = document.getElementById('loopToggle');
+        
         // Output elements
         const inferenceTime = document.getElementById('inferenceTime');
         const outputProgress = document.getElementById('outputProgress');
@@ -625,6 +699,136 @@
             modelFileInput.click();
         });
         
+        // Camera functions
+        async function startCamera() {
+            try {
+                cameraStream = await navigator.mediaDevices.getUserMedia({ 
+                    video: { width: 640, height: 480 } 
+                });
+                cameraStream.srcObject = cameraStream;
+                cameraSection.classList.remove('hidden');
+                logMessage('Camera started successfully');
+            } catch (error) {
+                console.error('Error accessing camera:', error);
+                logMessage('Error: Could not access camera - ' + error.message);
+                alert('Could not access camera. Please check permissions.');
+            }
+        }
+        
+        function stopCamera() {
+            if (cameraStream) {
+                cameraStream.getTracks().forEach(track => track.stop());
+                cameraStream = null;
+                cameraStream.srcObject = null;
+                cameraSection.classList.add('hidden');
+                logMessage('Camera stopped');
+            }
+        }
+        
+        function captureImage() {
+            if (!cameraStream) return;
+            
+            const canvas = capturedCanvas;
+            const ctx = canvas.getContext('2d');
+            canvas.width = 640;
+            canvas.height = 480;
+            
+            ctx.drawImage(cameraStream, 0, 0, canvas.width, canvas.height);
+            currentCapturedImage = canvas.toDataURL('image/jpeg');
+            capturedImageSection.classList.remove('hidden');
+            
+            logMessage('Image captured from camera');
+        }
+        
+        // Microphone functions
+        async function startRecording() {
+            try {
+                microphoneStream = await navigator.mediaDevices.getUserMedia({ 
+                    audio: true 
+                });
+                
+                // Set up audio context and analyzer
+                audioContext = new (window.AudioContext || window.webkitAudioContext)();
+                const source = audioContext.createMediaStreamSource(microphoneStream);
+                audioAnalyzer = audioContext.createAnalyser();
+                audioAnalyzer.fftSize = 256;
+                source.connect(audioAnalyzer);
+                
+                // Start recording
+                mediaRecorder = new MediaRecorder(microphoneStream);
+                recordedChunks = [];
+                
+                mediaRecorder.ondataavailable = (event) => {
+                    if (event.data.size > 0) {
+                        recordedChunks.push(event.data);
+                    }
+                };
+                
+                mediaRecorder.onstop = () => {
+                    const blob = new Blob(recordedChunks, { type: 'audio/wav' });
+                    currentRecordedAudioBlob = blob;
+                    recordedAudio.src = URL.createObjectURL(blob);
+                    recordedAudioSection.classList.remove('hidden');
+                    logMessage('Audio recording completed');
+                };
+                
+                mediaRecorder.start();
+                microphoneSection.classList.remove('hidden');
+                startAudioLevelMonitoring();
+                
+                logMessage('Recording started');
+            } catch (error) {
+                console.error('Error accessing microphone:', error);
+                logMessage('Error: Could not access microphone - ' + error.message);
+                alert('Could not access microphone. Please check permissions.');
+            }
+        }
+        
+        function startAudioLevelMonitoring() {
+            const dataArray = new Uint8Array(audioAnalyzer.frequencyBinCount);
+            
+            audioLevelInterval = setInterval(() => {
+                audioAnalyzer.getByteFrequencyData(dataArray);
+                const average = dataArray.reduce((sum, value) => sum + value) / dataArray.length;
+                const percentage = Math.min((average / 255) * 100, 100);
+                
+                audioLevelBar.style.width = percentage + '%';
+                audioLevelValue.textContent = Math.round(percentage) + '%';
+                
+                if (percentage > 70) {
+                    audioLevelBar.classList.add('bg-red-400');
+                    audioLevelBar.classList.remove('bg-emerald-400');
+                } else {
+                    audioLevelBar.classList.add('bg-emerald-400');
+                    audioLevelBar.classList.remove('bg-red-400');
+                }
+            }, 100);
+        }
+        
+        function stopRecording() {
+            if (mediaRecorder && mediaRecorder.state === 'recording') {
+                mediaRecorder.stop();
+            }
+            
+            if (microphoneStream) {
+                microphoneStream.getTracks().forEach(track => track.stop());
+                microphoneStream = null;
+            }
+            
+            if (audioContext) {
+                audioContext.close();
+                audioContext = null;
+            }
+            
+            if (audioLevelInterval) {
+                clearInterval(audioLevelInterval);
+                audioLevelInterval = null;
+            }
+            
+            microphoneSection.classList.add('hidden');
+            logMessage('Recording stopped');
+        }
+        
         // Input type selection
         inputTypeGrid.addEventListener('click', (e) => {
             const btn = e.target.closest('.input-type-btn');
@@ -662,6 +866,10 @@
         });
         
         function selectInputSource(source) {
+            // Stop any active streams when changing source
+            if (source !== 'camera') stopCamera();
+            if (source !== 'microphone') stopRecording();
+            
             // Update button styles
             document.querySelectorAll('.input-source-btn').forEach(btn => {
                 btn.classList.remove('bg-indigo-600');
@@ -674,6 +882,15 @@
             textInputArea.classList.toggle('hidden', source !== 'text');
             filePreview.classList.toggle('hidden', source !== 'upload');
             mediaControls.classList.toggle('hidden', !['audio', 'video'].includes(currentInputType));
+            cameraSection.classList.toggle('hidden', source !== 'camera');
+            microphoneSection.classList.toggle('hidden', source !== 'microphone');
+            
+            // Start appropriate media source
+            if (source === 'camera') {
+                startCamera();
+            } else if (source === 'microphone') {
+                // Don't auto-start recording, let user click the button
+            }
             
             logMessage(`Selected ${source} input source`);
         }
@@ -704,22 +921,22 @@
         function updatePreprocessingOptions(inputType) {
             const options = {
                 image: [
-                    { id: 'resize', label: 'Resize', checked: true },
-                    { id: 'normalize', label: 'Normalize', checked: true },
-                    { id: '裁剪', label: 'Crop', checked: false },
-                    { id: 'rotate', label: 'Rotate', checked: false },
-                    { id: 'flip', label: 'Flip', checked: false }
+                    { id: 'resize', label: 'Resize to 224x224', checked: true },
+                    { id: 'normalize', label: 'Normalize (0-1)', checked: true },
+                    { id: '裁剪', label: 'Center Crop', checked: false },
+                    { id: 'rotate', label: 'Random Rotate', checked: false },
+                    { id: 'flip', label: 'Horizontal Flip', checked: false }
                 ],
                 audio: [
-                    { id: 'resample', label: 'Resample', checked: true },
-                    { id: 'normalize', label: 'Normalize', checked: true },
-                    { id: 'denoise', label: 'Denoise', checked: false },
+                    { id: 'resample', label: 'Resample to 16kHz', checked: true },
+                    { id: 'normalize', label: 'Normalize Audio', checked: true },
+                    { id: 'denoise', label: 'Noise Reduction', checked: false },
                     { id: 'trim', label: 'Trim Silence', checked: false },
-                    { id: 'augment', label: 'Augment', checked: false }
+                    { id: 'augment', label: 'Data Augmentation', checked: false }
                 ],
                 text: [
-                    { id: 'tokenize', label: 'Tokenize', checked: true },
-                    { id: 'lowercase', label: 'Lowercase', checked: false },
+                    { id: 'tokenize', label: 'Tokenize Text', checked: true },
+                    { id: 'lowercase', label: 'Convert to Lowercase', checked: false },
                     { id: 'remove_punct', label: 'Remove Punctuation', checked: false },
                     { id: 'stop_words', label: 'Remove Stop Words', checked: false },
                     { id: 'stem', label: 'Stemming', checked: false }
@@ -768,6 +985,43 @@
             analysisSection.scrollIntoView({ behavior: 'smooth' });
         }
         
+        // Handle media file uploads (images/audio)
+        function handleMediaFileUpload(file) {
+            if (currentInputType === 'image' && file.type.startsWith('image/')) {
+                const reader = new FileReader();
+                reader.onload = (e) => {
+                    previewContent.innerHTML = `
+                        <img src="${e.target.result}" class="w-full max-h-48 object-contain rounded" alt="Preview">
+                        <p class="text-sm text-slate-400 mt-2">${file.name} (${(file.size / 1024).toFixed(1)} KB)</p>
+                    `;
+                    filePreview.classList.remove('hidden');
+                };
+                reader.readAsDataURL(file);
+            } else if (currentInputType === 'audio' && file.type.startsWith('audio/')) {
+                const url = URL.createObjectURL(file);
+                previewContent.innerHTML = `
+                    <audio controls class="w-full">
+                        <source src="${url}" type="${file.type}">
+                        Your browser does not support the audio element.
+                    </audio>
+                    <p class="text-sm text-slate-400 mt-2">${file.name} (${(file.size / 1024).toFixed(1)} KB)</p>
+                `;
+                filePreview.classList.remove('hidden');
+            }
+        }
+        
+        // Update model file input to handle both ONNX and media files
+        modelFileInput.addEventListener('change', (e) => {
+            const file = e.target.files[0];
+            if (!file) return;
+            
+            if (file.name.endsWith('.onnx')) {
+                handleFileUpload(file);
+            } else if (currentInputType && ['image', 'audio'].includes(currentInputType)) {
+                handleMediaFileUpload(file);
+            }
+        });
+        
         // Populate analysis data based on model type
         function populateAnalysisData(file) {
             modelInfo = {
@@ -851,6 +1105,10 @@
             // Simulate inference process
             logMessage('Starting inference...');
             
+            // Get current input data
+            const inputData = await getInputData();
+            logMessage('Input data prepared for inference');
+            
             // Simulate processing steps
             const steps = [
                 'Preprocessing input data...',
@@ -869,7 +1127,7 @@
             inferenceTime.textContent = (endTime - startTime) + 'ms';
             
             // Display results
-            displayResults();
+            await displayResults();
             
             executeBtn.disabled = false;
             executeBtn.innerHTML = '<i data-feather="play" class="mr-2"></i>Execute Inference';
@@ -877,18 +1135,57 @@
             logMessage('Inference completed successfully');
         });
         
-        function displayResults() {
+        async function getInputData() {
+            const source = document.querySelector('.input-source-btn.bg-indigo-600')?.dataset.source;
+            
+            switch (source) {
+                case 'upload':
+                    const file = modelFileInput.files[0];
+                    if (file && !file.name.endsWith('.onnx')) {
+                        return { type: currentInputType, file: file };
+                    }
+                    break;
+                case 'camera':
+                    if (currentCapturedImage) {
+                        return { type: 'image', data: currentCapturedImage };
+                    }
+                    break;
+                case 'microphone':
+                    if (currentRecordedAudioBlob) {
+                        return { type: 'audio', blob: currentRecordedAudioBlob };
+                    }
+                    break;
+                case 'text':
+                    const text = document.getElementById('textInput').value;
+                    return { type: 'text', data: text };
+            }
+            
+            return null;
+        }
+        
+        async function displayResults() {
+            const inputData = await getInputData();
             const outputType = currentInputType;
             let outputHTML = '';
             
             switch (outputType) {
                 case 'image':
+                    let imageSource = '';
+                    if (inputData?.type === 'image') {
+                        if (inputData.data) {
+                            imageSource = inputData.data; // captured image
+                        } else if (inputData.file) {
+                            imageSource = URL.createObjectURL(inputData.file); // uploaded file
+                        }
+                    }
+                    
                     outputHTML = `
                         <div class="text-center">
-                            <div class="w-48 h-48 bg-gradient-to-br from-purple-400 to-pink-400 rounded-lg mx-auto mb-4 flex items-center justify-center">
+                            ${imageSource ? `<img src="${imageSource}" class="w-full max-h-64 object-contain rounded-lg mx-auto mb-4" alt="Processed Image">` : `
+                            <div class="w-full h-64 bg-gradient-to-br from-purple-400 to-pink-400 rounded-lg mx-auto mb-4 flex items-center justify-center">
                                 <i data-feather="image" class="text-white w-16 h-16"></i>
-                            </div>
-                            <p class="text-slate-300">Generated Image Output</p>
+                            </div>`}
+                            <p class="text-slate-300">Processed Image Output</p>
                             <p class="text-sm text-slate-400 mt-2">224x224 RGB</p>
                         </div>
                     `;
@@ -896,8 +1193,18 @@
                     break;
                     
                 case 'audio':
+                    let audioElement = '';
+                    if (inputData?.type === 'audio') {
+                        const audioUrl = inputData.blob ? URL.createObjectURL(inputData.blob) : URL.createObjectURL(inputData.file);
+                        audioElement = `<audio controls class="w-full mb-4" ${autoPlayToggle.checked ? 'autoplay' : ''} ${loopToggle.checked ? 'loop' : ''}>
+                            <source src="${audioUrl}" type="audio/wav">
+                            Your browser does not support the audio element.
+                        </audio>`;
+                    }
+                    
                     outputHTML = `
                         <div class="text-center">
+                            ${audioElement || `
                             <div class="bg-slate-700 rounded-lg p-6 mb-4">
                                 <div class="flex items-center justify-center mb-4">
                                     <i data-feather="play" class="text-emerald-400 w-8 h-8 mr-2"></i>
@@ -908,22 +1215,25 @@
                                         `<div class="bg-emerald-400 w-2 rounded-t" style="height: ${Math.random() * 80 + 20}%"></div>`
                                     ).join('')}
                                 </div>
-                            </div>
-                            <p class="text-slate-300">Audio Output (44.1kHz, 16-bit)</p>
-                            <p class="text-sm text-slate-400 mt-2">Duration: 3.2s</p>
+                            </div>`}
+                            <p class="text-slate-300">Processed Audio Output</p>
+                            <p class="text-sm text-slate-400 mt-2">44.1kHz, 16-bit</p>
                         </div>
                     `;
                     downloadOptions.classList.remove('hidden');
                     break;
                     
                 case 'text':
+                    const inputText = inputData?.data || 'Sample text for processing';
                     outputHTML = `
                         <div class="text-left">
                             <div class="bg-slate-800 rounded-lg p-4 mb-4">
-                                <p class="text-slate-300">This is a sample generated text output from the ONNX model. The text processing model has successfully processed the input and generated meaningful content based on the model capabilities.</p>
+                                <p class="text-slate-300 mb-2"><strong>Input:</strong> ${inputText}</p>
+                                <hr class="border-slate-600 my-3">
+                                <p class="text-slate-300"><strong>Output:</strong> This is a sample generated text output from the ONNX model. The text processing model has successfully processed the input "${inputText}" and generated meaningful content based on the model capabilities and preprocessing options selected.</p>
                             </div>
                             <div class="flex items-center space-x-4">
-                                <button class="px-3 py-1 bg-slate-700 hover:bg-slate-600 rounded text-sm">
+                                <button class="px-3 py-1 bg-slate-700 hover:bg-slate-600 rounded text-sm" onclick="navigator.clipboard.writeText(this.closest('.text-left').querySelector('p:last-child').textContent)">
                                     <i data-feather="copy" class="w-4 h-4 mr-1 inline"></i>Copy
                                 </button>
                                 <button class="px-3 py-1 bg-slate-700 hover:bg-slate-600 rounded text-sm">
@@ -943,6 +1253,55 @@
             outputDisplay.classList.remove('hidden');
         }
         
+        // Event listeners for media controls
+        captureBtn.addEventListener('click', captureImage);
+        stopCameraBtn.addEventListener('click', stopCamera);
+        startRecordingBtn.addEventListener('click', startRecording);
+        stopRecordingBtn.addEventListener('click', stopRecording);
+        
+        // Download functionality
+        downloadOptions.addEventListener('click', (e) => {
+            if (e.target.closest('button')) {
+                const button = e.target.closest('button');
+                if (button.textContent.includes('Download')) {
+                    downloadResults();
+                }
+            }
+        });
+        
+        function downloadResults() {
+            const outputType = currentInputType;
+            
+            switch (outputType) {
+                case 'image':
+                    if (currentCapturedImage) {
+                        const link = document.createElement('a');
+                        link.href = currentCapturedImage;
+                        link.download = 'processed_image.jpg';
+                        link.click();
+                    }
+                    break;
+                case 'audio':
+                    if (currentRecordedAudioBlob) {
+                        const link = document.createElement('a');
+                        link.href = URL.createObjectURL(currentRecordedAudioBlob);
+                        link.download = 'processed_audio.wav';
+                        link.click();
+                    }
+                    break;
+                case 'text':
+                    const textContent = outputContent.querySelector('p:last-child').textContent;
+                    const blob = new Blob([textContent], { type: 'text/plain' });
+                    const link = document.createElement('a');
+                    link.href = URL.createObjectURL(blob);
+                    link.download = 'processed_text.txt';
+                    link.click();
+                    break;
+            }
+            
+            logMessage('Results downloaded successfully');
+        }
+        
         // Log messages to console
         function logMessage(message) {
             const timestamp = new Date().toLocaleTimeString();
@@ -992,6 +1351,12 @@
         loadModelBtn.addEventListener('click', () => {
             modelFileInput.click();
         });
+        
+        // Clean up media streams when page unloads
+        window.addEventListener('beforeunload', () => {
+            stopCamera();
+            stopRecording();
+        });
     </script>
 </body>
 </html>