Upload index.js with huggingface_hub

index.js

@@ -1,76 +1,325 @@
-import { pipeline } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.4.1';
+import {
+    AutoProcessor,
+    AutoModelForImageTextToText,
+    TextStreamer,
+} from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.2';
 
-// Reference the elements that we will need
-const status = document.getElementById('status');
-const fileUpload = document.getElementById('upload');
-const imageContainer = document.getElementById('container');
-const example = document.getElementById('example');
+let processor = null;
+let model = null;
+let videoFile = null;
+let frames = [];
+let captions = [];
 
-const EXAMPLE_URL = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/city-streets.jpg';
+// DOM Elements
+const uploadArea = document.getElementById('uploadArea');
+const videoInput = document.getElementById('videoInput');
+const videoSection = document.getElementById('videoSection');
+const videoPlayer = document.getElementById('videoPlayer');
+const frameCanvas = document.getElementById('frameCanvas');
+const processBtn = document.getElementById('processBtn');
+const progressSection = document.getElementById('progressSection');
+const progressFill = document.getElementById('progressFill');
+const progressText = document.getElementById('progressText');
+const resultsSection = document.getElementById('resultsSection');
+const framesList = document.getElementById('framesList');
+const deviceSelect = document.getElementById('deviceSelect');
 
-// Create a new object detection pipeline
-status.textContent = 'Loading model...';
-const detector = await pipeline('object-detection', 'Xenova/detr-resnet-50');
-status.textContent = 'Ready';
+// Check WebGPU support
+async function checkWebGPU() {
+    if (!navigator.gpu) {
+        deviceSelect.querySelector('option[value="webgpu"]').disabled = true;
+        deviceSelect.value = 'wasm';
+    }
+}
+
+// Initialize model
+async function initializeModel() {
+    try {
+        progressText.textContent = 'Loading processor...';
+        progressFill.style.width = '30%';
+        
+        const model_id = 'onnx-community/FastVLM-0.5B-ONNX';
+        processor = await AutoProcessor.from_pretrained(model_id);
+        
+        progressText.textContent = 'Loading model (this may take a moment)...';
+        progressFill.style.width = '60%';
+        
+        const device = deviceSelect.value === 'webgpu' ? 'webgpu' : 'wasm';
+        model = await AutoModelForImageTextToText.from_pretrained(model_id, {
+            device: device,
+            dtype: {
+                embed_tokens: 'fp16',
+                vision_encoder: 'q4',
+                decoder_model_merged: 'q4',
+            },
+        });
+        
+        progressFill.style.width = '100%';
+        progressText.textContent = 'Model loaded successfully!';
+        
+        return true;
+    } catch (error) {
+        console.error('Error initializing model:', error);
+        progressText.textContent = 'Error loading model. Please refresh and try again.';
+        return false;
+    }
+}
+
+// Upload handling
+uploadArea.addEventListener('click', () => videoInput.click());
+
+uploadArea.addEventListener('dragover', (e) => {
+    e.preventDefault();
+    uploadArea.classList.add('dragover');
+});
 
-example.addEventListener('click', (e) => {
+uploadArea.addEventListener('dragleave', () => {
+    uploadArea.classList.remove('dragover');
+});
+
+uploadArea.addEventListener('drop', (e) => {
     e.preventDefault();
-    detect(EXAMPLE_URL);
+    uploadArea.classList.remove('dragover');
+    const files = e.dataTransfer.files;
+    if (files.length > 0 && files[0].type.startsWith('video/')) {
+        handleVideoFile(files[0]);
+    }
 });
 
-fileUpload.addEventListener('change', function (e) {
-    const file = e.target.files[0];
-    if (!file) {
+videoInput.addEventListener('change', (e) => {
+    if (e.target.files.length > 0) {
+        handleVideoFile(e.target.files[0]);
+    }
+});
+
+function handleVideoFile(file) {
+    if (file.size > 100 * 1024 * 1024) {
+        alert('File size exceeds 100MB limit');
         return;
     }
+    
+    videoFile = file;
+    const url = URL.createObjectURL(file);
+    videoPlayer.src = url;
+    videoSection.classList.remove('hidden');
+    resultsSection.classList.add('hidden');
+    frames = [];
+    captions = [];
+}
 
-    const reader = new FileReader();
+// Extract frames from video
+async function extractFrames() {
+    const interval = parseInt(document.getElementById('frameInterval').value);
+    const ctx = frameCanvas.getContext('2d');
+    const duration = videoPlayer.duration;
+    frames = [];
+    
+    for (let time = 0; time < duration; time += interval) {
+        videoPlayer.currentTime = time;
+        await new Promise(resolve => {
+            videoPlayer.onseeked = resolve;
+        });
+        
+        frameCanvas.width = videoPlayer.videoWidth;
+        frameCanvas.height = videoPlayer.videoHeight;
+        ctx.drawImage(videoPlayer, 0, 0);
+        
+        const blob = await new Promise(resolve => {
+            frameCanvas.toBlob(resolve, 'image/jpeg', 0.9);
+        });
+        
+        frames.push({
+            time: time,
+            blob: blob,
+            dataUrl: await blobToDataUrl(blob)
+        });
+    }
+    
+    return frames;
+}
 
-    // Set up a callback when the file is loaded
-    reader.onload = e2 => detect(e2.target.result);
+function blobToDataUrl(blob) {
+    return new Promise((resolve) => {
+        const reader = new FileReader();
+        reader.onloadend = () => resolve(reader.result);
+        reader.readAsDataURL(blob);
+    });
+}
+
+// Generate caption for a frame
+async function generateCaption(imageDataUrl, frameIndex, totalFrames) {
+    try {
+        progressText.textContent = `Processing frame ${frameIndex + 1} of ${totalFrames}...`;
+        progressFill.style.width = `${((frameIndex + 1) / totalFrames) * 100}%`;
+        
+        const messages = [
+            {
+                role: 'user',
+                content: '<image>Describe this video frame in detail. What is happening in this scene?',
+            },
+        ];
+        
+        const prompt = processor.apply_chat_template(messages, {
+            add_generation_prompt: true,
+        });
+        
+        // Create image element from data URL
+        const img = new Image();
+        img.src = imageDataUrl;
+        await new Promise(resolve => img.onload = resolve);
+        
+        const inputs = await processor(img, prompt, {
+            add_special_tokens: false,
+        });
+        
+        let streamedText = '';
+        const outputs = await model.generate({
+            ...inputs,
+            max_new_tokens: 256,
+            do_sample: false,
+            streamer: new TextStreamer(processor.tokenizer, {
+                skip_prompt: true,
+                skip_special_tokens: false,
+                callback_function: (text) => {
+                    streamedText += text;
+                },
+            }),
+        });
+        
+        const decoded = processor.batch_decode(
+            outputs.slice(null, [inputs.input_ids.dims.at(-1), null]),
+            { skip_special_tokens: true }
+        );
+        
+        return decoded[0];
+    } catch (error) {
+        console.error('Error generating caption:', error);
+        return 'Error generating caption for this frame';
+    }
+}
 
-    reader.readAsDataURL(file);
+// Process video
+processBtn.addEventListener('click', async () => {
+    if (!videoFile) return;
+    
+    processBtn.disabled = true;
+    progressSection.classList.remove('hidden');
+    resultsSection.classList.add('hidden');
+    
+    try {
+        // Initialize model if not already loaded
+        if (!model || !processor) {
+            const success = await initializeModel();
+            if (!success) {
+                processBtn.disabled = false;
+                return;
+            }
+        }
+        
+        // Extract frames
+        progressText.textContent = 'Extracting frames...';
+        progressFill.style.width = '20%';
+        frames = await extractFrames();
+        
+        // Generate captions
+        captions = [];
+        for (let i = 0; i < frames.length; i++) {
+            const caption = await generateCaption(frames[i].dataUrl, i, frames.length);
+            captions.push({
+                time: frames[i].time,
+                caption: caption,
+                thumbnail: frames[i].dataUrl
+            });
+            
+            // Update results in real-time
+            displayResults();
+            resultsSection.classList.remove('hidden');
+        }
+        
+        progressText.textContent = 'Processing complete!';
+        setTimeout(() => {
+            progressSection.classList.add('hidden');
+        }, 2000);
+        
+    } catch (error) {
+        console.error('Processing error:', error);
+        progressText.textContent = 'Error processing video. Please try again.';
+    }
+    
+    processBtn.disabled = false;
 });
 
+// Display results
+function displayResults() {
+    framesList.innerHTML = '';
+    
+    captions.forEach((item, index) => {
+        const frameCard = document.createElement('div');
+        frameCard.className = 'frame-card';
+        
+        const time = formatTime(item.time);
+        
+        frameCard.innerHTML = `
+            <div class="frame-thumbnail">
+                <img src="${item.thumbnail}" alt="Frame at ${time}">
+                <span class="frame-time">${time}</span>
+            </div>
+            <div class="frame-caption">
+                <p>${item.caption}</p>
+            </div>
+        `;
+        
+        framesList.appendChild(frameCard);
+    });
+}
 
-// Detect objects in the image
-async function detect(img) {
-    imageContainer.innerHTML = '';
-    imageContainer.style.backgroundImage = `url(${img})`;
+function formatTime(seconds) {
+    const mins = Math.floor(seconds / 60);
+    const secs = Math.floor(seconds % 60);
+    return `${mins.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
+}
 
-    status.textContent = 'Analysing...';
-    const output = await detector(img, {
-        threshold: 0.5,
-        percentage: true,
+// Export functions
+document.getElementById('exportJson').addEventListener('click', () => {
+    const data = JSON.stringify(captions, null, 2);
+    downloadFile(data, 'captions.json', 'application/json');
+});
+
+document.getElementById('exportSrt').addEventListener('click', () => {
+    let srt = '';
+    captions.forEach((item, index) => {
+        const startTime = formatSrtTime(item.time);
+        const endTime = formatSrtTime(item.time + 5);
+        srt += `${index + 1}\n${startTime} --> ${endTime}\n${item.caption}\n\n`;
+    });
+    downloadFile(srt, 'captions.srt', 'text/plain');
+});
+
+document.getElementById('exportTxt').addEventListener('click', () => {
+    let txt = '';
+    captions.forEach(item => {
+        txt += `[${formatTime(item.time)}] ${item.caption}\n\n`;
     });
-    status.textContent = '';
-    output.forEach(renderBox);
+    downloadFile(txt, 'captions.txt', 'text/plain');
+});
+
+function formatSrtTime(seconds) {
+    const hours = Math.floor(seconds / 3600);
+    const mins = Math.floor((seconds % 3600) / 60);
+    const secs = Math.floor(seconds % 60);
+    const ms = Math.floor((seconds % 1) * 1000);
+    return `${hours.toString().padStart(2, '0')}:${mins.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')},${ms.toString().padStart(3, '0')}`;
 }
 
-// Render a bounding box and label on the image
-function renderBox({ box, label }) {
-    const { xmax, xmin, ymax, ymin } = box;
-
-    // Generate a random color for the box
-    const color = '#' + Math.floor(Math.random() * 0xFFFFFF).toString(16).padStart(6, 0);
-
-    // Draw the box
-    const boxElement = document.createElement('div');
-    boxElement.className = 'bounding-box';
-    Object.assign(boxElement.style, {
-        borderColor: color,
-        left: 100 * xmin + '%',
-        top: 100 * ymin + '%',
-        width: 100 * (xmax - xmin) + '%',
-        height: 100 * (ymax - ymin) + '%',
-    })
-
-    // Draw label
-    const labelElement = document.createElement('span');
-    labelElement.textContent = label;
-    labelElement.className = 'bounding-box-label';
-    labelElement.style.backgroundColor = color;
-
-    boxElement.appendChild(labelElement);
-    imageContainer.appendChild(boxElement);
+function downloadFile(content, filename, type) {
+    const blob = new Blob([content], { type });
+    const url = URL.createObjectURL(blob);
+    const a = document.createElement('a');
+    a.href = url;
+    a.download = filename;
+    a.click();
+    URL.revokeObjectURL(url);
 }
+
+// Initialize
+checkWebGPU();