integrate backend with opensource LLM to read & recognise the image.

README.md

@@ -1,13 +1,41 @@
----
-title: SilverSight Snap & Know 👓
-colorFrom: pink
-colorTo: red
-emoji: 🐳
-sdk: static
-pinned: false
-tags:
-  - deepsite-v3
----
-
-# Welcome to your new DeepSite project!
-This project was created with [DeepSite](https://deepsite.hf.co).
+```markdown
+# iKnow - Image Recognition App
+
+## Backend Setup
+
+1. Install dependencies:
+```bash
+npm install express multer tesseract.js axios form-data
+```
+
+2. Set up LLaMA.cpp:
+- Clone and build LLaMA.cpp: https://github.com/ggerganov/llama.cpp
+- Run the server:
+```bash
+./server -m models/ggml-model-q4_0.bin --ctx 2048
+```
+
+3. Start the Node.js server:
+```bash
+node api.js
+```
+
+## API Endpoints
+
+- POST `/api/analyze` - Processes an image and returns recognition results
+
+## Environment
+
+- Node.js 16+
+- Tesseract OCR
+- LLaMA.cpp server running locally
+```
+
+The changes integrate a backend system that:
+1. Uses Tesseract.js for OCR (text recognition)
+2. Sends the image and extracted text to a local LLaMA.cpp server for analysis
+3. Returns structured data about recognized items with confidence scores
+4. Shows results in a modal dialog
+5. Includes mock API for development without the backend
+
+The backend expects a running LLaMA.cpp server which can be set up following their documentation. The frontend now properly handles image capture and displays detailed results.

api.js

@@ -0,0 +1,52 @@
+```javascript
+const express = require('express');
+const router = express.Router();
+const multer = require('multer');
+const { createWorker } = require('tesseract.js');
+const { pipeline } = require('stream/promises');
+const FormData = require('form-data');
+const fs = require('fs');
+const axios = require('axios');
+
+const upload = multer({ dest: 'uploads/' });
+const worker = createWorker();
+
+// Initialize Tesseract worker
+(async () => {
+    await worker.load();
+    await worker.loadLanguage('eng');
+    await worker.initialize('eng');
+})();
+
+router.post('/analyze', upload.single('image'), async (req, res) => {
+    try {
+        // 1. Perform OCR on the image first
+        const { data: { text } } = await worker.recognize(req.file.path);
+        
+        // 2. Send to LLM for analysis (using LLaMA.cpp as example)
+        const form = new FormData();
+        form.append('image', fs.createReadStream(req.file.path));
+        form.append('text_context', text);
+        
+        const llmResponse = await axios.post('http://localhost:8080/analyze-image', form, {
+            headers: form.getHeaders()
+        });
+
+        // Clean up the uploaded file
+        fs.unlinkSync(req.file.path);
+
+        res.json({
+            description: llmResponse.data.description,
+            items: llmResponse.data.items.map(item => ({
+                label: item.label,
+                confidence: item.confidence
+            }))
+        });
+    } catch (error) {
+        console.error('Error processing image:', error);
+        res.status(500).json({ error: 'Error processing image' });
+    }
+});
+
+module.exports = router;
+```

index.html

@@ -134,10 +134,27 @@
             </button>
 </div>
     </nav>
-<script>
+    <script>
         feather.replace();
-        
-        // Camera shutter animation
+
+        // Mock API response for development
+        if (window.location.hostname === 'localhost' || window.location.hostname === '127.0.0.1') {
+            window.fetch = window.fetch || function(url, options) {
+                if (url === '/api/analyze') {
+                    return Promise.resolve({
+                        json: () => ({
+                            description: "This appears to be a coffee mug on a wooden table. The mug is white with a simple design.",
+                            items: [
+                                { label: "Coffee Mug", confidence: 0.92 },
+                                { label: "Wooden Table", confidence: 0.87 }
+                            ]
+                        })
+                    });
+                }
+                return originalFetch(url, options);
+            };
+        }
+// Camera shutter animation
         document.querySelector('[data-feather="aperture"]').closest('button').addEventListener('click', function() {
             const shutter = document.createElement('div');
             shutter.className = 'absolute inset-0 bg-white opacity-0';
@@ -150,12 +167,52 @@
                 easing: 'easeInOutQuad',
                 complete: () => shutter.remove()
             });
+            // Capture and process image
+            const img = document.querySelector('.relative img');
+            const canvas = document.createElement('canvas');
+            canvas.width = img.width;
+            canvas.height = img.height;
+            const ctx = canvas.getContext('2d');
+            ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
+            const imageData = canvas.toDataURL('image/jpeg');
             
-            // Here you would normally handle the camera capture
-            setTimeout(() => {
-                alert("Picture taken! Processing...");
-            }, 300);
-        });
+            // Send to backend for processing
+            fetch('/api/analyze', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({ image: imageData })
+            })
+            .then(response => response.json())
+            .then(data => {
+                // Show results in a modal
+                const resultsModal = document.createElement('div');
+                resultsModal.className = 'fixed inset-0 bg-black bg-opacity-80 z-50 flex items-center justify-center p-4';
+                resultsModal.innerHTML = `
+                    <div class="bg-white rounded-xl p-6 max-w-md w-full">
+                        <h3 class="text-xl font-bold mb-4">Results</h3>
+                        <p class="mb-4">${data.description}</p>
+                        <div class="grid gap-2 mb-4">
+                            ${data.items.map(item => `
+                                <div class="flex items-center">
+                                    <div class="w-3 h-3 rounded-full ${item.confidence > 0.7 ? 'bg-green-500' : 'bg-yellow-500'} mr-2"></div>
+                                    <span>${item.label} (${Math.round(item.confidence * 100)}%)</span>
+                                </div>
+                            `).join('')}
+                        </div>
+                        <button onclick="this.closest('div').remove()" class="w-full bg-primary text-white py-2 rounded-lg">
+                            Close
+                        </button>
+                    </div>
+                `;
+                document.body.appendChild(resultsModal);
+            })
+            .catch(error => {
+                console.error('Error:', error);
+                alert('Failed to process image. Please try again.');
+            });
+});
 
         // Navigation handling
         document.querySelectorAll('nav button').forEach(button => {