Upload index.js with huggingface_hub

index.js

@@ -1,76 +1,182 @@
-import { pipeline } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.4.1';
-
-// Reference the elements that we will need
-const status = document.getElementById('status');
-const fileUpload = document.getElementById('upload');
-const imageContainer = document.getElementById('container');
-const example = document.getElementById('example');
-
-const EXAMPLE_URL = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/city-streets.jpg';
-
-// Create a new object detection pipeline
-status.textContent = 'Loading model...';
-const detector = await pipeline('object-detection', 'Xenova/detr-resnet-50');
-status.textContent = 'Ready';
-
-example.addEventListener('click', (e) => {
-    e.preventDefault();
-    detect(EXAMPLE_URL);
-});
-
-fileUpload.addEventListener('change', function (e) {
-    const file = e.target.files[0];
-    if (!file) {
-        return;
+import { AutoProcessor, AutoTokenizer, Moondream1ForConditionalGeneration, RawImage } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@2.17.2';
+
+class MoondreamAssistant {
+    constructor() {
+        this.model = null;
+        this.processor = null;
+        this.tokenizer = null;
+        this.currentImage = null;
+        this.isModelLoaded = false;
+        
+        this.initializeElements();
+        this.bindEvents();
+        this.loadModel();
     }
 
-    const reader = new FileReader();
+    initializeElements() {
+        this.uploadArea = document.getElementById('uploadArea');
+        this.fileInput = document.getElementById('fileInput');
+        this.imagePreview = document.getElementById('imagePreview');
+        this.previewImage = document.getElementById('previewImage');
+        this.removeBtn = document.getElementById('removeBtn');
+        this.querySection = document.getElementById('querySection');
+        this.questionInput = document.getElementById('questionInput');
+        this.askBtn = document.getElementById('askBtn');
+        this.responseSection = document.getElementById('responseSection');
+        this.loading = document.getElementById('loading');
+        this.responseText = document.getElementById('responseText');
+    }
 
-    // Set up a callback when the file is loaded
-    reader.onload = e2 => detect(e2.target.result);
+    bindEvents() {
+        this.uploadArea.addEventListener('click', () => this.fileInput.click());
+        this.uploadArea.addEventListener('dragover', this.handleDragOver.bind(this));
+        this.uploadArea.addEventListener('drop', this.handleDrop.bind(this));
+        this.fileInput.addEventListener('change', this.handleFileSelect.bind(this));
+        this.removeBtn.addEventListener('click', this.removeImage.bind(this));
+        this.questionInput.addEventListener('input', this.handleQuestionInput.bind(this));
+        this.askBtn.addEventListener('click', this.askQuestion.bind(this));
+        this.questionInput.addEventListener('keypress', (e) => {
+            if (e.key === 'Enter' && !this.askBtn.disabled) {
+                this.askQuestion();
+            }
+        });
+    }
 
-    reader.readAsDataURL(file);
-});
+    async loadModel() {
+        try {
+            const model_id = 'Xenova/moondream2';
+            
+            this.updateUploadArea('Loading AI model...', true);
+            
+            [this.processor, this.tokenizer, this.model] = await Promise.all([
+                AutoProcessor.from_pretrained(model_id),
+                AutoTokenizer.from_pretrained(model_id),
+                Moondream1ForConditionalGeneration.from_pretrained(model_id, {
+                    dtype: {
+                        embed_tokens: 'fp16',
+                        vision_encoder: 'fp16',
+                        decoder_model_merged: 'q4',
+                    },
+                    device: 'webgpu',
+                })
+            ]);
+
+            this.isModelLoaded = true;
+            this.updateUploadArea('Drop an image here or browse');
+            console.log('Model loaded successfully');
+        } catch (error) {
+            console.error('Error loading model:', error);
+            this.updateUploadArea('Error loading AI model. Please refresh.', true);
+        }
+    }
 
+    handleDragOver(e) {
+        e.preventDefault();
+        this.uploadArea.classList.add('dragover');
+    }
 
-// Detect objects in the image
-async function detect(img) {
-    imageContainer.innerHTML = '';
-    imageContainer.style.backgroundImage = `url(${img})`;
+    handleDrop(e) {
+        e.preventDefault();
+        this.uploadArea.classList.remove('dragover');
+        const files = e.dataTransfer.files;
+        if (files.length > 0) {
+            this.handleImageFile(files[0]);
+        }
+    }
 
-    status.textContent = 'Analysing...';
-    const output = await detector(img, {
-        threshold: 0.5,
-        percentage: true,
-    });
-    status.textContent = '';
-    output.forEach(renderBox);
-}
+    handleFileSelect(e) {
+        const file = e.target.files[0];
+        if (file) {
+            this.handleImageFile(file);
+        }
+    }
+
+    async handleImageFile(file) {
+        if (!file.type.startsWith('image/')) {
+            alert('Please select a valid image file');
+            return;
+        }
+
+        try {
+            this.currentImage = await RawImage.fromBlob(file);
+            const url = URL.createObjectURL(file);
+            
+            this.previewImage.src = url;
+            this.imagePreview.style.display = 'block';
+            this.uploadArea.style.display = 'none';
+            this.querySection.style.display = 'block';
+            this.questionInput.focus();
+        } catch (error) {
+            console.error('Error loading image:', error);
+            alert('Error loading image. Please try again.');
+        }
+    }
 
-// Render a bounding box and label on the image
-function renderBox({ box, label }) {
-    const { xmax, xmin, ymax, ymin } = box;
-
-    // Generate a random color for the box
-    const color = '#' + Math.floor(Math.random() * 0xFFFFFF).toString(16).padStart(6, 0);
-
-    // Draw the box
-    const boxElement = document.createElement('div');
-    boxElement.className = 'bounding-box';
-    Object.assign(boxElement.style, {
-        borderColor: color,
-        left: 100 * xmin + '%',
-        top: 100 * ymin + '%',
-        width: 100 * (xmax - xmin) + '%',
-        height: 100 * (ymax - ymin) + '%',
-    })
-
-    // Draw label
-    const labelElement = document.createElement('span');
-    labelElement.textContent = label;
-    labelElement.className = 'bounding-box-label';
-    labelElement.style.backgroundColor = color;
-
-    boxElement.appendChild(labelElement);
-    imageContainer.appendChild(boxElement);
+    removeImage() {
+        this.currentImage = null;
+        this.imagePreview.style.display = 'none';
+        this.uploadArea.style.display = 'flex';
+        this.querySection.style.display = 'none';
+        this.responseSection.style.display = 'none';
+        this.questionInput.value = '';
+        this.askBtn.disabled = true;
+        
+        if (this.previewImage.src) {
+            URL.revokeObjectURL(this.previewImage.src);
+        }
+    }
+
+    handleQuestionInput() {
+        this.askBtn.disabled = this.questionInput.value.trim().length === 0;
+    }
+
+    async askQuestion() {
+        if (!this.isModelLoaded || !this.currentImage) return;
+
+        const question = this.questionInput.value.trim();
+        if (!question) return;
+
+        this.askBtn.disabled = true;
+        this.loading.style.display = 'flex';
+        this.responseSection.style.display = 'block';
+        this.responseText.textContent = '';
+
+        try {
+            const text = `<image>\n\nQuestion: ${question}\n\nAnswer:`;
+            const text_inputs = this.tokenizer(text);
+            const vision_inputs = await this.processor(this.currentImage);
+
+            const output = await this.model.generate({
+                ...text_inputs,
+                ...vision_inputs,
+                do_sample: false,
+                max_new_tokens: 128,
+            });
+
+            const decoded = this.tokenizer.batch_decode(output, { skip_special_tokens: true });
+            const response = decoded[0]
+                .split('Answer:')[1]
+                .trim()
+                .replace(/<\|.*?\|>/g, '');
+
+            this.responseText.textContent = response;
+        } catch (error) {
+            console.error('Error generating response:', error);
+            this.responseText.textContent = 'Sorry, there was an error processing your question. Please try again.';
+        } finally {
+            this.loading.style.display = 'none';
+            this.askBtn.disabled = false;
+        }
+    }
+
+    updateUploadArea(text, isError = false) {
+        const p = this.uploadArea.querySelector('p');
+        p.textContent = text;
+        this.uploadArea.style.pointerEvents = isError ? 'none' : 'auto';
+    }
 }
+
+// Initialize the application
+document.addEventListener('DOMContentLoaded', () => {
+    new MoondreamAssistant();
+});