Spaces:
Running
Running
| import { pipeline, SamModel, AutoModel, AutoProcessor, RawImage } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.8.1'; | |
| // Reference the elements that we will need | |
| const status = document.getElementById('status'); | |
| const fileUpload = document.getElementById('upload'); | |
| const imageContainer = document.getElementById('container'); | |
| const example = document.getElementById('example'); | |
| const modelSelect = document.getElementById('model-select'); | |
| const EXAMPLE_URL = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/city-streets.jpg'; | |
| // State and Model Caches | |
| let currentRawImage = null; | |
| let currentImageDataUrl = null; | |
| const models = { | |
| detr: { detector: null, samModel: null, samProcessor: null }, | |
| gelan: { model: null, processor: null, samModel: null, samProcessor: null }, | |
| segformer: { segmenter: null }, | |
| depth: { estimator: null } | |
| }; | |
| // Listen for model changes and re-run if we have an image | |
| modelSelect.addEventListener('change', () => { | |
| if (currentImageDataUrl) detect(currentImageDataUrl); | |
| }); | |
| example.addEventListener('click', (e) => { | |
| e.preventDefault(); | |
| detect(EXAMPLE_URL); | |
| }); | |
| const randomBtn = document.getElementById('random-img'); | |
| randomBtn.addEventListener('click', () => { | |
| // Append timestamp to bust cache and get a new random image each time | |
| detect(`https://picsum.photos/640/480?t=${Date.now()}`); | |
| }); | |
| fileUpload.addEventListener('change', function (e) { | |
| const file = e.target.files[0]; | |
| if (!file) { | |
| return; | |
| } | |
| const reader = new FileReader(); | |
| reader.onload = e2 => detect(e2.target.result); | |
| reader.readAsDataURL(file); | |
| }); | |
| async function loadModelsForSelection() { | |
| const selection = modelSelect.value; | |
| if (selection === 'detr') { | |
| if (!models.detr.detector) { | |
| status.textContent = 'Loading DETR & SAM models...'; | |
| models.detr.detector = await pipeline('object-detection', 'Xenova/detr-resnet-50'); | |
| models.detr.samModel = await SamModel.from_pretrained('Xenova/slimsam-77-uniform'); | |
| models.detr.samProcessor = await AutoProcessor.from_pretrained('Xenova/slimsam-77-uniform'); | |
| } | |
| } else if (selection === 'gelan') { | |
| if (!models.gelan.model) { | |
| status.textContent = 'Loading GELAN & SAM models...'; | |
| // Use AutoModel because YOLOv9 is not yet supported in pipeline() | |
| models.gelan.model = await AutoModel.from_pretrained('Xenova/gelan-c_all', { dtype: 'fp32' }); | |
| models.gelan.processor = await AutoProcessor.from_pretrained('Xenova/gelan-c_all'); | |
| models.gelan.samModel = await SamModel.from_pretrained('Xenova/slimsam-77-uniform'); | |
| models.gelan.samProcessor = await AutoProcessor.from_pretrained('Xenova/slimsam-77-uniform'); | |
| } | |
| } else if (selection === 'segformer') { | |
| if (!models.segformer.segmenter) { | |
| status.textContent = 'Loading Segformer model...'; | |
| models.segformer.segmenter = await pipeline('image-segmentation', 'Xenova/segformer-b0-finetuned-ade-512-512'); | |
| } | |
| } else if (selection === 'depth') { | |
| if (!models.depth.estimator) { | |
| status.textContent = 'Loading Depth Estimation model...'; | |
| models.depth.estimator = await pipeline('depth-estimation', 'onnx-community/depth-anything-v2-small'); | |
| } | |
| } | |
| } | |
| // Detect objects in the image | |
| async function detect(img) { | |
| currentImageDataUrl = img; | |
| imageContainer.innerHTML = ''; | |
| // Use an actual img tag instead of background tricks | |
| const imageElement = document.createElement('img'); | |
| imageElement.crossOrigin = 'anonymous'; | |
| imageElement.src = img; | |
| imageContainer.appendChild(imageElement); | |
| // Wait for image to load, then snapshot to data URL so all panels use the same image | |
| await new Promise((resolve, reject) => { | |
| if (imageElement.complete) return resolve(); | |
| imageElement.onload = resolve; | |
| imageElement.onerror = reject; | |
| }); | |
| const snapCanvas = document.createElement('canvas'); | |
| snapCanvas.width = imageElement.naturalWidth; | |
| snapCanvas.height = imageElement.naturalHeight; | |
| snapCanvas.getContext('2d').drawImage(imageElement, 0, 0); | |
| const stableImgUrl = snapCanvas.toDataURL('image/png'); | |
| imageElement.src = stableImgUrl; | |
| img = stableImgUrl; | |
| await loadModelsForSelection(); | |
| status.textContent = 'Loading image...'; | |
| try { | |
| currentRawImage = await RawImage.read(img); | |
| } catch (err) { | |
| console.error(err); | |
| status.textContent = 'Failed to read image'; | |
| return; | |
| } | |
| const selection = modelSelect.value; | |
| status.textContent = 'Analysing image...'; | |
| const renderWrapper = document.createElement('div'); | |
| renderWrapper.style.position = 'absolute'; | |
| renderWrapper.style.top = '0'; | |
| renderWrapper.style.left = '0'; | |
| renderWrapper.style.width = '100%'; | |
| renderWrapper.style.height = '100%'; | |
| renderWrapper.style.pointerEvents = 'none'; | |
| imageContainer.appendChild(renderWrapper); | |
| try { | |
| if (selection === 'detr') { | |
| const output = await models.detr.detector(img, { threshold: 0.5, percentage: true }); | |
| status.textContent = 'Ready'; | |
| output.forEach(boxInfo => renderBox(boxInfo, renderWrapper, models.detr)); | |
| } else if (selection === 'gelan') { | |
| // Manual AutoModel processing path for YOLOv9 | |
| const inputs = await models.gelan.processor(currentRawImage); | |
| const { outputs } = await models.gelan.model(inputs); | |
| const predictions = outputs.tolist(); | |
| status.textContent = 'Ready'; | |
| predictions.forEach(pred => { | |
| const [xmin, ymin, xmax, ymax, score, id] = pred; | |
| if (score > 0.5) { | |
| // Convert raw coordinates to percentage | |
| const boxPercentage = { | |
| xmin: xmin / currentRawImage.width, | |
| ymin: ymin / currentRawImage.height, | |
| xmax: xmax / currentRawImage.width, | |
| ymax: ymax / currentRawImage.height | |
| }; | |
| const label = models.gelan.model.config.id2label[id]; | |
| renderBox({ box: boxPercentage, label }, renderWrapper, models.gelan); | |
| } | |
| }); | |
| } else if (selection === 'segformer') { | |
| const output = await models.segformer.segmenter(img); | |
| status.textContent = 'Ready'; | |
| // output is an array of objects containing { label, mask } where mask is a RawImage | |
| output.forEach(segmentData => renderSemanticMask(segmentData, renderWrapper)); | |
| } else if (selection === 'depth') { | |
| const { depth } = await models.depth.estimator(img); | |
| status.textContent = 'Ready'; | |
| // Remove the default renderWrapper | |
| imageContainer.removeChild(renderWrapper); | |
| // Override container styles for 3-panel horizontal layout | |
| imageContainer.style.overflow = 'visible'; | |
| imageContainer.style.display = 'flex'; | |
| imageContainer.style.flexDirection = 'row'; | |
| imageContainer.style.gap = '8px'; | |
| imageContainer.style.minWidth = '0'; | |
| imageContainer.style.minHeight = '0'; | |
| imageContainer.style.border = 'none'; | |
| imageContainer.style.maxWidth = '100%'; | |
| // Helper: create a labeled panel with an image + overlay area | |
| function createPanel(label) { | |
| const panel = document.createElement('div'); | |
| panel.style.cssText = 'flex:1; min-width:0; max-width:33.33%;'; | |
| const panelLabel = document.createElement('div'); | |
| panelLabel.textContent = label; | |
| panelLabel.style.cssText = 'font-size:12px; font-weight:bold; margin-bottom:4px; color:#666;'; | |
| const wrap = document.createElement('div'); | |
| wrap.style.cssText = 'position:relative; width:100%;'; | |
| const panelImg = document.createElement('img'); | |
| panelImg.crossOrigin = 'anonymous'; | |
| panelImg.src = img; | |
| panelImg.style.cssText = 'display:block; width:100%; height:auto;'; | |
| wrap.appendChild(panelImg); | |
| panel.appendChild(panelLabel); | |
| panel.appendChild(wrap); | |
| imageContainer.appendChild(panel); | |
| return { panel, wrap, panelImg }; | |
| } | |
| // Panel 1: Original image (reuse the already-appended img) | |
| const origPanel = document.createElement('div'); | |
| origPanel.style.cssText = 'flex:1; min-width:0; max-width:33.33%;'; | |
| const origLabel = document.createElement('div'); | |
| origLabel.textContent = 'Original'; | |
| origLabel.style.cssText = 'font-size:12px; font-weight:bold; margin-bottom:4px; color:#666;'; | |
| const origWrap = document.createElement('div'); | |
| origWrap.style.cssText = 'position:relative; width:100%;'; | |
| const existingImg = imageContainer.querySelector('img'); | |
| existingImg.style.cssText = 'display:block; width:100%; height:auto;'; | |
| origWrap.appendChild(existingImg); | |
| origPanel.appendChild(origLabel); | |
| origPanel.appendChild(origWrap); | |
| imageContainer.appendChild(origPanel); | |
| // Panel 2: Greyscale depth overlay | |
| const { wrap: gsWrap, panelImg: gsImg } = createPanel('Depth Map'); | |
| gsImg.style.visibility = 'hidden'; | |
| renderGreyscaleDepth(depth, gsWrap); | |
| // Panel 3: 3D layers | |
| const { wrap: wrap3d, panelImg: img3d } = createPanel('3D Layers'); | |
| // Wait for image to load so container has dimensions for WebGL canvas | |
| if (img3d.complete) { | |
| renderDepthMap(depth, wrap3d); | |
| } else { | |
| img3d.onload = () => renderDepthMap(depth, wrap3d); | |
| } | |
| } | |
| } catch (e) { | |
| console.error(e); | |
| status.textContent = 'Error during analysis'; | |
| } | |
| } | |
| // Draw the output from Segformer | |
| function renderSemanticMask({ label, mask }, container) { | |
| // Generate a random color for the mask | |
| const color = '#' + Math.floor(Math.random() * 0xFFFFFF).toString(16).padStart(6, 0); | |
| const r = parseInt(color.slice(1, 3), 16); | |
| const g = parseInt(color.slice(3, 5), 16); | |
| const b = parseInt(color.slice(5, 7), 16); | |
| const canvas = document.createElement('canvas'); | |
| canvas.width = mask.width; | |
| canvas.height = mask.height; | |
| const ctx = canvas.getContext('2d'); | |
| const imgData = ctx.createImageData(mask.width, mask.height); | |
| const maskSize = mask.width * mask.height; | |
| for (let i = 0; i < maskSize; ++i) { | |
| const val = mask.data[i]; | |
| if (val > 128) { | |
| imgData.data[i * 4 + 0] = r; | |
| imgData.data[i * 4 + 1] = g; | |
| imgData.data[i * 4 + 2] = b; | |
| imgData.data[i * 4 + 3] = 150; | |
| } else { | |
| imgData.data[i * 4 + 0] = 0; | |
| imgData.data[i * 4 + 1] = 0; | |
| imgData.data[i * 4 + 2] = 0; | |
| imgData.data[i * 4 + 3] = 0; | |
| } | |
| } | |
| ctx.putImageData(imgData, 0, 0); | |
| canvas.style.position = 'absolute'; | |
| canvas.style.top = '0'; | |
| canvas.style.left = '0'; | |
| canvas.style.width = '100%'; | |
| canvas.style.height = '100%'; | |
| let firstIdx = mask.data.findIndex(v => v > 128); | |
| if (firstIdx !== -1) { | |
| const labelElement = document.createElement('span'); | |
| labelElement.textContent = label; | |
| labelElement.className = 'bounding-box-label'; | |
| labelElement.style.backgroundColor = color; | |
| labelElement.style.display = 'block'; | |
| const labelY = Math.floor(firstIdx / mask.width); | |
| const labelX = firstIdx % mask.width; | |
| labelElement.style.left = (labelX / mask.width * 100) + '%'; | |
| labelElement.style.top = (labelY / mask.height * 100) + '%'; | |
| container.appendChild(labelElement); | |
| } | |
| container.appendChild(canvas); | |
| } | |
| // Render a bounding box and label on the image | |
| function renderBox({ box, label }, container, activeModelCache) { | |
| const { xmax, xmin, ymax, ymin } = box; | |
| const color = '#' + Math.floor(Math.random() * 0xFFFFFF).toString(16).padStart(6, 0); | |
| const boxElement = document.createElement('div'); | |
| boxElement.className = 'bounding-box'; | |
| Object.assign(boxElement.style, { | |
| borderColor: color, | |
| left: 100 * xmin + '%', | |
| top: 100 * ymin + '%', | |
| width: 100 * (xmax - xmin) + '%', | |
| height: 100 * (ymax - ymin) + '%', | |
| cursor: 'crosshair', | |
| zIndex: 10, | |
| pointerEvents: 'auto', | |
| }); | |
| const labelElement = document.createElement('span'); | |
| labelElement.textContent = label; | |
| labelElement.className = 'bounding-box-label'; | |
| labelElement.style.backgroundColor = color; | |
| labelElement.style.display = 'none'; | |
| boxElement.appendChild(labelElement); | |
| container.appendChild(boxElement); | |
| boxElement.addEventListener('click', async (e) => { | |
| e.stopPropagation(); | |
| if (!currentRawImage) return; | |
| status.textContent = `Generating mask for ${label}...`; | |
| const originalXmin = xmin * currentRawImage.width; | |
| const originalYmin = ymin * currentRawImage.height; | |
| const originalXmax = xmax * currentRawImage.width; | |
| const originalYmax = ymax * currentRawImage.height; | |
| const centerX = (originalXmin + originalXmax) / 2; | |
| const centerY = (originalYmin + originalYmax) / 2; | |
| const input_boxes = [[[originalXmin, originalYmin, originalXmax, originalYmax]]]; | |
| const input_points = [[[centerX, centerY]]]; | |
| try { | |
| const inputs = await activeModelCache.samProcessor(currentRawImage, { input_boxes, input_points }); | |
| const modelInputs = { ...inputs }; | |
| delete modelInputs.input_boxes; | |
| const outputs = await activeModelCache.samModel(modelInputs); | |
| const masks = await activeModelCache.samProcessor.post_process_masks(outputs.pred_masks, inputs.original_sizes, inputs.reshaped_input_sizes); | |
| const scores = outputs.iou_scores.data; | |
| let bestScoreIdx = 0; | |
| let maxScore = -Infinity; | |
| for (let i = 0; i < scores.length; i++) { | |
| if (scores[i] > maxScore) { | |
| maxScore = scores[i]; | |
| bestScoreIdx = i; | |
| } | |
| } | |
| const H = masks[0].dims[2]; | |
| const W = masks[0].dims[3]; | |
| const rawMaskData = masks[0].data; | |
| const maskSize = H * W; | |
| const canvas = document.createElement('canvas'); | |
| canvas.width = W; | |
| canvas.height = H; | |
| const ctx = canvas.getContext('2d'); | |
| const imgData = ctx.createImageData(W, H); | |
| const r = parseInt(color.slice(1, 3), 16); | |
| const g = parseInt(color.slice(3, 5), 16); | |
| const b = parseInt(color.slice(5, 7), 16); | |
| const channelOffset = bestScoreIdx * maskSize; | |
| for (let i = 0; i < maskSize; ++i) { | |
| const val = rawMaskData[channelOffset + i]; | |
| if (val) { | |
| imgData.data[i * 4 + 0] = r; | |
| imgData.data[i * 4 + 1] = g; | |
| imgData.data[i * 4 + 2] = b; | |
| imgData.data[i * 4 + 3] = 150; | |
| } else { | |
| imgData.data[i * 4 + 0] = 0; | |
| imgData.data[i * 4 + 1] = 0; | |
| imgData.data[i * 4 + 2] = 0; | |
| imgData.data[i * 4 + 3] = 0; | |
| } | |
| } | |
| ctx.putImageData(imgData, 0, 0); | |
| canvas.style.position = 'absolute'; | |
| canvas.style.top = '0'; | |
| canvas.style.left = '0'; | |
| canvas.style.width = '100%'; | |
| canvas.style.height = '100%'; | |
| canvas.style.pointerEvents = 'none'; | |
| canvas.style.zIndex = '5'; | |
| container.appendChild(canvas); | |
| labelElement.style.display = 'block'; | |
| status.textContent = 'Ready'; | |
| } catch (err) { | |
| console.error(err); | |
| status.textContent = 'Error during segmentation'; | |
| } | |
| }); | |
| } | |
| // ── Greyscale Depth Overlay (Xenova-style) ────────────────────────── | |
| function renderGreyscaleDepth(depthImage, container) { | |
| const W = depthImage.width; | |
| const H = depthImage.height; | |
| const pixelCount = W * H; | |
| const depthData = depthImage.data; | |
| let mn = Infinity, mx = -Infinity; | |
| for (let i = 0; i < pixelCount; i++) { | |
| if (depthData[i] < mn) mn = depthData[i]; | |
| if (depthData[i] > mx) mx = depthData[i]; | |
| } | |
| const rng = mx - mn || 1; | |
| const rgba = new Uint8ClampedArray(4 * pixelCount); | |
| for (let i = 0; i < pixelCount; i++) { | |
| const idx = 4 * i; | |
| rgba[idx] = 255; // Red channel | |
| rgba[idx + 3] = Math.round(255 * (1 - (depthData[i] - mn) / rng)); | |
| } | |
| const canvas = document.createElement('canvas'); | |
| canvas.width = W; | |
| canvas.height = H; | |
| canvas.getContext('2d').putImageData(new ImageData(rgba, W, H), 0, 0); | |
| Object.assign(canvas.style, { | |
| position: 'absolute', top: '0', left: '0', | |
| width: '100%', height: '100%', pointerEvents: 'none', zIndex: '10' | |
| }); | |
| container.appendChild(canvas); | |
| } | |
| // ── Per-Pixel Depth Layer Renderer ────────────────────────────────── | |
| function renderDepthMap(depthImage, container) { | |
| const Z_SPREAD = 1.6; | |
| const NUM_LAYERS = 4; | |
| const W = depthImage.width; | |
| const H = depthImage.height; | |
| const pixelCount = W * H; | |
| const depthData = depthImage.data; | |
| // ── 1. Normalize depth to 0..1 ── | |
| let dMin = Infinity, dMax = -Infinity; | |
| for (let i = 0; i < pixelCount; i++) { | |
| if (depthData[i] < dMin) dMin = depthData[i]; | |
| if (depthData[i] > dMax) dMax = depthData[i]; | |
| } | |
| const dRange = dMax - dMin || 1; | |
| const norm = new Float32Array(pixelCount); | |
| for (let i = 0; i < pixelCount; i++) { | |
| norm[i] = (depthData[i] - dMin) / dRange; | |
| } | |
| // ── 2. Build histogram of per-pixel depths, find natural boundaries ── | |
| const HIST_BINS = 256; | |
| const histogram = new Float32Array(HIST_BINS); | |
| for (let i = 0; i < pixelCount; i++) { | |
| const bin = Math.min(HIST_BINS - 1, Math.floor(norm[i] * HIST_BINS)); | |
| histogram[bin]++; | |
| } | |
| // Smooth histogram with a wide Gaussian kernel | |
| const smoothed = new Float32Array(HIST_BINS); | |
| const KERNEL_RADIUS = 8; | |
| for (let i = 0; i < HIST_BINS; i++) { | |
| let sum = 0, wt = 0; | |
| for (let k = -KERNEL_RADIUS; k <= KERNEL_RADIUS; k++) { | |
| const idx = Math.max(0, Math.min(HIST_BINS - 1, i + k)); | |
| const w = Math.exp(-0.5 * (k / (KERNEL_RADIUS / 2)) ** 2); | |
| sum += histogram[idx] * w; | |
| wt += w; | |
| } | |
| smoothed[i] = sum / wt; | |
| } | |
| // Find valleys (local minima) in smoothed histogram | |
| const valleys = []; | |
| for (let i = 2; i < HIST_BINS - 2; i++) { | |
| if (smoothed[i] <= smoothed[i - 1] && smoothed[i] <= smoothed[i + 1] && | |
| smoothed[i] < smoothed[i - 2] && smoothed[i] < smoothed[i + 2]) { | |
| valleys.push({ bin: i, value: smoothed[i] }); | |
| } | |
| } | |
| // Sort by depth of valley (lowest count = best separator) | |
| valleys.sort((a, b) => a.value - b.value); | |
| // Pick the best (NUM_LAYERS-1) valleys as cut points | |
| const numCuts = NUM_LAYERS - 1; | |
| let cutBins; | |
| if (valleys.length >= numCuts) { | |
| cutBins = valleys.slice(0, numCuts).map(v => v.bin).sort((a, b) => a - b); | |
| } else { | |
| // Fallback: uniform cuts | |
| cutBins = []; | |
| for (let i = 1; i < NUM_LAYERS; i++) { | |
| cutBins.push(Math.round((i / NUM_LAYERS) * HIST_BINS)); | |
| } | |
| } | |
| // Convert bin indices to depth boundaries | |
| const boundaries = [0, ...cutBins.map(b => b / HIST_BINS), 1.01]; | |
| console.log(`Depth: ${NUM_LAYERS} layers, boundaries: [${boundaries.map(b => b.toFixed(2)).join(', ')}]`); | |
| // ── 3. Assign each pixel to a layer ── | |
| const pixelLayer = new Uint8Array(pixelCount); | |
| for (let i = 0; i < pixelCount; i++) { | |
| const d = norm[i]; | |
| for (let l = 0; l < NUM_LAYERS; l++) { | |
| if (d >= boundaries[l] && d < boundaries[l + 1]) { | |
| pixelLayer[i] = l; | |
| break; | |
| } | |
| } | |
| } | |
| // Merge small layers into the one above (nearer) | |
| const MIN_PIXEL_RATIO = 0.05; // layers with < 5% of pixels get merged | |
| const layerCounts = new Uint32Array(NUM_LAYERS); | |
| for (let i = 0; i < pixelCount; i++) layerCounts[pixelLayer[i]]++; | |
| const mergeMap = new Uint8Array(NUM_LAYERS); | |
| for (let l = 0; l < NUM_LAYERS; l++) mergeMap[l] = l; | |
| for (let l = 0; l < NUM_LAYERS; l++) { | |
| if (layerCounts[l] < pixelCount * MIN_PIXEL_RATIO) { | |
| // Merge into the layer above (l+1), or below if it's the last | |
| const target = l < NUM_LAYERS - 1 ? l + 1 : l - 1; | |
| if (target >= 0) mergeMap[l] = mergeMap[target]; | |
| } | |
| } | |
| // Apply merge and remap to contiguous indices | |
| const usedLayers = [...new Set(Array.from(mergeMap))].sort((a, b) => a - b); | |
| const remapTable = new Uint8Array(NUM_LAYERS); | |
| usedLayers.forEach((old, idx) => remapTable[old] = idx); | |
| for (let i = 0; i < pixelCount; i++) { | |
| pixelLayer[i] = remapTable[mergeMap[pixelLayer[i]]]; | |
| } | |
| const FINAL_LAYERS = usedLayers.length; | |
| console.log(`After merge: ${FINAL_LAYERS} layers (from ${NUM_LAYERS})`); | |
| // ── 4. Get original image pixels ── | |
| const tmpCanvas = document.createElement('canvas'); | |
| tmpCanvas.width = W; tmpCanvas.height = H; | |
| const tmpCtx = tmpCanvas.getContext('2d'); | |
| const imgEl = container.querySelector('img'); | |
| tmpCtx.drawImage(imgEl, 0, 0, W, H); | |
| const origPixels = tmpCtx.getImageData(0, 0, W, H).data; | |
| imgEl.style.visibility = 'hidden'; | |
| // ── 5. Create masked textures per layer + build wall geometry ── | |
| const levelTextures = []; | |
| const aspect = W / H; | |
| // Per-layer: collect wall quad vertices (position xyz + color rgb, 6 floats/vertex) | |
| const wallVertArrays = []; // wallVertArrays[layer] = Float32Array of wall verts | |
| for (let layer = 0; layer < FINAL_LAYERS; layer++) { | |
| const rgba = new Uint8Array(W * H * 4); | |
| const wallVerts = []; // temp array to collect wall vertex data | |
| for (let i = 0; i < pixelCount; i++) { | |
| if (pixelLayer[i] !== layer) continue; | |
| rgba[i * 4] = origPixels[i * 4]; | |
| rgba[i * 4 + 1] = origPixels[i * 4 + 1]; | |
| rgba[i * 4 + 2] = origPixels[i * 4 + 2]; | |
| rgba[i * 4 + 3] = 255; | |
| // Only build walls for layers that have a layer behind them | |
| if (layer === 0) continue; | |
| const px = i % W, py = (i / W) | 0; | |
| const r = origPixels[i * 4] / 255; | |
| const g = origPixels[i * 4 + 1] / 255; | |
| const b = origPixels[i * 4 + 2] / 255; | |
| // Map pixel bounds to world coords | |
| const xL = aspect * (2 * px / W - 1); | |
| const xR = aspect * (2 * (px + 1) / W - 1); | |
| const yT = 1 - 2 * py / H; | |
| const yB = 1 - 2 * (py + 1) / H; | |
| // Check each face - if neighbor is a different layer, create a wall quad | |
| // Wall quad: z=0 is front (this layer), z=1 is back (layer behind) | |
| // Each quad = 2 triangles = 6 vertices, each vertex = (x,y,z, r,g,b) | |
| // Right wall | |
| if (px < W - 1 && pixelLayer[i + 1] !== layer) { | |
| wallVerts.push(xR, yT, 0, r, g, b, xR, yT, 1, r, g, b, xR, yB, 0, r, g, b); | |
| wallVerts.push(xR, yB, 0, r, g, b, xR, yT, 1, r, g, b, xR, yB, 1, r, g, b); | |
| } | |
| // Left wall | |
| if (px > 0 && pixelLayer[i - 1] !== layer) { | |
| wallVerts.push(xL, yT, 0, r, g, b, xL, yB, 0, r, g, b, xL, yT, 1, r, g, b); | |
| wallVerts.push(xL, yB, 0, r, g, b, xL, yB, 1, r, g, b, xL, yT, 1, r, g, b); | |
| } | |
| // Bottom wall | |
| if (py < H - 1 && pixelLayer[i + W] !== layer) { | |
| wallVerts.push(xL, yB, 0, r, g, b, xR, yB, 1, r, g, b, xR, yB, 0, r, g, b); | |
| wallVerts.push(xL, yB, 0, r, g, b, xL, yB, 1, r, g, b, xR, yB, 1, r, g, b); | |
| } | |
| // Top wall | |
| if (py > 0 && pixelLayer[i - W] !== layer) { | |
| wallVerts.push(xL, yT, 0, r, g, b, xR, yT, 0, r, g, b, xR, yT, 1, r, g, b); | |
| wallVerts.push(xL, yT, 0, r, g, b, xR, yT, 1, r, g, b, xL, yT, 1, r, g, b); | |
| } | |
| } | |
| levelTextures.push(rgba); | |
| wallVertArrays.push(new Float32Array(wallVerts)); | |
| } | |
| console.log(`Wall geometry: ${wallVertArrays.map((a, i) => `L${i}:${a.length / 6} verts`).join(', ')}`); | |
| // ── 6. WebGL 3D Render ── | |
| const glCanvas = document.createElement('canvas'); | |
| const rect = container.getBoundingClientRect(); | |
| glCanvas.width = rect.width * devicePixelRatio; | |
| glCanvas.height = rect.height * devicePixelRatio; | |
| Object.assign(glCanvas.style, { | |
| position: 'absolute', top: '0', left: '0', | |
| width: '100%', height: '100%', zIndex: '20', pointerEvents: 'auto', cursor: 'grab' | |
| }); | |
| container.appendChild(glCanvas); | |
| const gl = glCanvas.getContext('webgl', { alpha: true, premultipliedAlpha: false }); | |
| if (!gl) { console.error('WebGL not supported'); return; } | |
| gl.enable(gl.BLEND); | |
| gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA); | |
| gl.enable(gl.DEPTH_TEST); | |
| gl.depthFunc(gl.LEQUAL); | |
| function compileShader(src, type) { | |
| const s = gl.createShader(type); gl.shaderSource(s, src); gl.compileShader(s); | |
| if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) { console.error(gl.getShaderInfoLog(s)); return null; } | |
| return s; | |
| } | |
| // ── Face shader (textured quads) ── | |
| const faceProg = gl.createProgram(); | |
| gl.attachShader(faceProg, compileShader(` | |
| attribute vec2 aPos; attribute vec2 aUV; uniform mat4 uMVP; varying vec2 vUV; | |
| void main() { vUV = aUV; gl_Position = uMVP * vec4(aPos, 0.0, 1.0); }`, gl.VERTEX_SHADER)); | |
| gl.attachShader(faceProg, compileShader(` | |
| precision mediump float; varying vec2 vUV; uniform sampler2D uTex; | |
| void main() { vec4 c = texture2D(uTex, vUV); if (c.a < 0.01) discard; gl_FragColor = c; }`, gl.FRAGMENT_SHADER)); | |
| gl.linkProgram(faceProg); | |
| const faceAPos = gl.getAttribLocation(faceProg, 'aPos'); | |
| const faceAUV = gl.getAttribLocation(faceProg, 'aUV'); | |
| const faceUMVP = gl.getUniformLocation(faceProg, 'uMVP'); | |
| const faceUTex = gl.getUniformLocation(faceProg, 'uTex'); | |
| const faceVerts = new Float32Array([-aspect, 1, 0, 0, aspect, 1, 1, 0, -aspect, -1, 0, 1, aspect, -1, 1, 1]); | |
| const faceVBO = gl.createBuffer(); | |
| gl.bindBuffer(gl.ARRAY_BUFFER, faceVBO); | |
| gl.bufferData(gl.ARRAY_BUFFER, faceVerts, gl.STATIC_DRAW); | |
| // Upload face textures | |
| const glTextures = []; | |
| for (let lvl = 0; lvl < FINAL_LAYERS; lvl++) { | |
| const tex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, tex); | |
| gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, W, H, 0, gl.RGBA, gl.UNSIGNED_BYTE, levelTextures[lvl]); | |
| gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); | |
| gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); | |
| gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); | |
| gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); | |
| glTextures.push(tex); | |
| } | |
| // ── Wall shader (solid colored geometry) ── | |
| const wallProg = gl.createProgram(); | |
| gl.attachShader(wallProg, compileShader(` | |
| attribute vec3 aWallPos; attribute vec3 aWallColor; | |
| uniform mat4 uMVP; uniform float uZFront; uniform float uZBack; | |
| varying vec3 vColor; | |
| void main() { | |
| float z = mix(uZFront, uZBack, aWallPos.z); | |
| gl_Position = uMVP * vec4(aWallPos.xy, z, 1.0); | |
| vColor = aWallColor; | |
| }`, gl.VERTEX_SHADER)); | |
| gl.attachShader(wallProg, compileShader(` | |
| precision mediump float; varying vec3 vColor; | |
| void main() { gl_FragColor = vec4(vColor, 1.0); }`, gl.FRAGMENT_SHADER)); | |
| gl.linkProgram(wallProg); | |
| const wallAPos = gl.getAttribLocation(wallProg, 'aWallPos'); | |
| const wallAColor = gl.getAttribLocation(wallProg, 'aWallColor'); | |
| const wallUMVP = gl.getUniformLocation(wallProg, 'uMVP'); | |
| const wallUZFront = gl.getUniformLocation(wallProg, 'uZFront'); | |
| const wallUZBack = gl.getUniformLocation(wallProg, 'uZBack'); | |
| // Upload wall geometry buffers | |
| const wallVBOs = []; | |
| const wallVertCounts = []; | |
| for (let lvl = 0; lvl < FINAL_LAYERS; lvl++) { | |
| const vbo = gl.createBuffer(); | |
| gl.bindBuffer(gl.ARRAY_BUFFER, vbo); | |
| gl.bufferData(gl.ARRAY_BUFFER, wallVertArrays[lvl], gl.STATIC_DRAW); | |
| wallVBOs.push(vbo); | |
| wallVertCounts.push(wallVertArrays[lvl].length / 6); // 6 floats per vertex | |
| } | |
| // Matrix helpers | |
| const m4 = { | |
| perspective(fovY, asp, near, far) { | |
| const f = 1 / Math.tan(fovY / 2), nf = 1 / (near - far); | |
| return new Float32Array([f / asp, 0, 0, 0, 0, f, 0, 0, 0, 0, (far + near) * nf, -1, 0, 0, 2 * far * near * nf, 0]); | |
| }, | |
| rotY(a) { const c = Math.cos(a), s = Math.sin(a); return new Float32Array([c, 0, s, 0, 0, 1, 0, 0, -s, 0, c, 0, 0, 0, 0, 1]); }, | |
| rotX(a) { const c = Math.cos(a), s = Math.sin(a); return new Float32Array([1, 0, 0, 0, 0, c, -s, 0, 0, s, c, 0, 0, 0, 0, 1]); }, | |
| translate(x, y, z) { return new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, x, y, z, 1]); }, | |
| mul(a, b) { | |
| const o = new Float32Array(16); | |
| for (let i = 0; i < 4; i++) for (let j = 0; j < 4; j++) { | |
| o[j * 4 + i] = 0; for (let k = 0; k < 4; k++) o[j * 4 + i] += a[k * 4 + i] * b[j * 4 + k]; | |
| } | |
| return o; | |
| } | |
| }; | |
| const canvasAspect = glCanvas.width / glCanvas.height; | |
| const fov = Math.PI / 4; | |
| const proj = m4.perspective(fov, canvasAspect, 0.1, 100); | |
| const camDist = 1.0 / Math.tan(fov / 2) + 0.1; | |
| let rotYAngle = 0, rotXAngle = 0, dragging = false, lastMX = 0, lastMY = 0; | |
| glCanvas.addEventListener('mousedown', e => { dragging = true; lastMX = e.clientX; lastMY = e.clientY; glCanvas.style.cursor = 'grabbing'; }); | |
| window.addEventListener('mouseup', () => { dragging = false; glCanvas.style.cursor = 'grab'; }); | |
| window.addEventListener('mousemove', e => { | |
| if (!dragging) return; | |
| rotYAngle += (e.clientX - lastMX) * 0.006; rotXAngle += (e.clientY - lastMY) * 0.006; | |
| rotXAngle = Math.max(-Math.PI / 3, Math.min(Math.PI / 3, rotXAngle)); | |
| lastMX = e.clientX; lastMY = e.clientY; | |
| }); | |
| glCanvas.addEventListener('touchstart', e => { dragging = true; lastMX = e.touches[0].clientX; lastMY = e.touches[0].clientY; }, { passive: true }); | |
| window.addEventListener('touchend', () => { dragging = false; }); | |
| window.addEventListener('touchmove', e => { | |
| if (!dragging) return; | |
| rotYAngle += (e.touches[0].clientX - lastMX) * 0.006; rotXAngle += (e.touches[0].clientY - lastMY) * 0.006; | |
| rotXAngle = Math.max(-Math.PI / 3, Math.min(Math.PI / 3, rotXAngle)); | |
| lastMX = e.touches[0].clientX; lastMY = e.touches[0].clientY; | |
| }, { passive: true }); | |
| const startTime = performance.now(); | |
| const INTRO_MS = 1000; | |
| function easeOut(t) { return 1 - Math.pow(1 - t, 3); } | |
| function draw() { | |
| gl.viewport(0, 0, glCanvas.width, glCanvas.height); | |
| gl.clearColor(0, 0, 0, 0); | |
| gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); | |
| const t = Math.min((performance.now() - startTime) / INTRO_MS, 1); | |
| const spread = Z_SPREAD * easeOut(t); | |
| const view = m4.translate(0, 0, -camDist); | |
| const rot = m4.mul(m4.rotY(rotYAngle), m4.rotX(rotXAngle)); | |
| const vp = m4.mul(proj, m4.mul(view, rot)); | |
| // Compute Z positions for each layer | |
| const zPositions = []; | |
| for (let lvl = 0; lvl < FINAL_LAYERS; lvl++) { | |
| zPositions.push(FINAL_LAYERS > 1 ? (lvl / (FINAL_LAYERS - 1)) * spread - spread / 2 : 0); | |
| } | |
| for (let lvl = 0; lvl < FINAL_LAYERS; lvl++) { | |
| const zFront = zPositions[lvl]; | |
| // Draw wall geometry between this layer and the one behind | |
| if (lvl > 0 && wallVertCounts[lvl] > 0) { | |
| const zBack = zPositions[lvl - 1]; | |
| gl.useProgram(wallProg); | |
| gl.bindBuffer(gl.ARRAY_BUFFER, wallVBOs[lvl]); | |
| gl.enableVertexAttribArray(wallAPos); | |
| gl.vertexAttribPointer(wallAPos, 3, gl.FLOAT, false, 24, 0); | |
| gl.enableVertexAttribArray(wallAColor); | |
| gl.vertexAttribPointer(wallAColor, 3, gl.FLOAT, false, 24, 12); | |
| gl.uniformMatrix4fv(wallUMVP, false, vp); | |
| gl.uniform1f(wallUZFront, zFront); | |
| gl.uniform1f(wallUZBack, zBack); | |
| gl.drawArrays(gl.TRIANGLES, 0, wallVertCounts[lvl]); | |
| } | |
| // Draw the face quad | |
| gl.useProgram(faceProg); | |
| gl.bindBuffer(gl.ARRAY_BUFFER, faceVBO); | |
| gl.enableVertexAttribArray(faceAPos); | |
| gl.vertexAttribPointer(faceAPos, 2, gl.FLOAT, false, 16, 0); | |
| gl.enableVertexAttribArray(faceAUV); | |
| gl.vertexAttribPointer(faceAUV, 2, gl.FLOAT, false, 16, 8); | |
| const mvp = m4.mul(vp, m4.translate(0, 0, zFront)); | |
| gl.uniformMatrix4fv(faceUMVP, false, mvp); | |
| gl.activeTexture(gl.TEXTURE0); | |
| gl.bindTexture(gl.TEXTURE_2D, glTextures[lvl]); | |
| gl.uniform1i(faceUTex, 0); | |
| gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); | |
| } | |
| requestAnimationFrame(draw); | |
| } | |
| draw(); | |
| } | |