segment_test / index.js
benhorin's picture
added models
d6c214d
Raw
History Blame Contribute Delete
34.4 kB
import { pipeline, SamModel, AutoModel, AutoProcessor, RawImage } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.8.1';
// Reference the elements that we will need
const status = document.getElementById('status');
const fileUpload = document.getElementById('upload');
const imageContainer = document.getElementById('container');
const example = document.getElementById('example');
const modelSelect = document.getElementById('model-select');
const EXAMPLE_URL = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/city-streets.jpg';
// State and Model Caches
let currentRawImage = null;
let currentImageDataUrl = null;
const models = {
detr: { detector: null, samModel: null, samProcessor: null },
gelan: { model: null, processor: null, samModel: null, samProcessor: null },
segformer: { segmenter: null },
depth: { estimator: null }
};
// Listen for model changes and re-run if we have an image
modelSelect.addEventListener('change', () => {
if (currentImageDataUrl) detect(currentImageDataUrl);
});
example.addEventListener('click', (e) => {
e.preventDefault();
detect(EXAMPLE_URL);
});
const randomBtn = document.getElementById('random-img');
randomBtn.addEventListener('click', () => {
// Append timestamp to bust cache and get a new random image each time
detect(`https://picsum.photos/640/480?t=${Date.now()}`);
});
fileUpload.addEventListener('change', function (e) {
const file = e.target.files[0];
if (!file) {
return;
}
const reader = new FileReader();
reader.onload = e2 => detect(e2.target.result);
reader.readAsDataURL(file);
});
async function loadModelsForSelection() {
const selection = modelSelect.value;
if (selection === 'detr') {
if (!models.detr.detector) {
status.textContent = 'Loading DETR & SAM models...';
models.detr.detector = await pipeline('object-detection', 'Xenova/detr-resnet-50');
models.detr.samModel = await SamModel.from_pretrained('Xenova/slimsam-77-uniform');
models.detr.samProcessor = await AutoProcessor.from_pretrained('Xenova/slimsam-77-uniform');
}
} else if (selection === 'gelan') {
if (!models.gelan.model) {
status.textContent = 'Loading GELAN & SAM models...';
// Use AutoModel because YOLOv9 is not yet supported in pipeline()
models.gelan.model = await AutoModel.from_pretrained('Xenova/gelan-c_all', { dtype: 'fp32' });
models.gelan.processor = await AutoProcessor.from_pretrained('Xenova/gelan-c_all');
models.gelan.samModel = await SamModel.from_pretrained('Xenova/slimsam-77-uniform');
models.gelan.samProcessor = await AutoProcessor.from_pretrained('Xenova/slimsam-77-uniform');
}
} else if (selection === 'segformer') {
if (!models.segformer.segmenter) {
status.textContent = 'Loading Segformer model...';
models.segformer.segmenter = await pipeline('image-segmentation', 'Xenova/segformer-b0-finetuned-ade-512-512');
}
} else if (selection === 'depth') {
if (!models.depth.estimator) {
status.textContent = 'Loading Depth Estimation model...';
models.depth.estimator = await pipeline('depth-estimation', 'onnx-community/depth-anything-v2-small');
}
}
}
// Detect objects in the image
async function detect(img) {
currentImageDataUrl = img;
imageContainer.innerHTML = '';
// Use an actual img tag instead of background tricks
const imageElement = document.createElement('img');
imageElement.crossOrigin = 'anonymous';
imageElement.src = img;
imageContainer.appendChild(imageElement);
// Wait for image to load, then snapshot to data URL so all panels use the same image
await new Promise((resolve, reject) => {
if (imageElement.complete) return resolve();
imageElement.onload = resolve;
imageElement.onerror = reject;
});
const snapCanvas = document.createElement('canvas');
snapCanvas.width = imageElement.naturalWidth;
snapCanvas.height = imageElement.naturalHeight;
snapCanvas.getContext('2d').drawImage(imageElement, 0, 0);
const stableImgUrl = snapCanvas.toDataURL('image/png');
imageElement.src = stableImgUrl;
img = stableImgUrl;
await loadModelsForSelection();
status.textContent = 'Loading image...';
try {
currentRawImage = await RawImage.read(img);
} catch (err) {
console.error(err);
status.textContent = 'Failed to read image';
return;
}
const selection = modelSelect.value;
status.textContent = 'Analysing image...';
const renderWrapper = document.createElement('div');
renderWrapper.style.position = 'absolute';
renderWrapper.style.top = '0';
renderWrapper.style.left = '0';
renderWrapper.style.width = '100%';
renderWrapper.style.height = '100%';
renderWrapper.style.pointerEvents = 'none';
imageContainer.appendChild(renderWrapper);
try {
if (selection === 'detr') {
const output = await models.detr.detector(img, { threshold: 0.5, percentage: true });
status.textContent = 'Ready';
output.forEach(boxInfo => renderBox(boxInfo, renderWrapper, models.detr));
} else if (selection === 'gelan') {
// Manual AutoModel processing path for YOLOv9
const inputs = await models.gelan.processor(currentRawImage);
const { outputs } = await models.gelan.model(inputs);
const predictions = outputs.tolist();
status.textContent = 'Ready';
predictions.forEach(pred => {
const [xmin, ymin, xmax, ymax, score, id] = pred;
if (score > 0.5) {
// Convert raw coordinates to percentage
const boxPercentage = {
xmin: xmin / currentRawImage.width,
ymin: ymin / currentRawImage.height,
xmax: xmax / currentRawImage.width,
ymax: ymax / currentRawImage.height
};
const label = models.gelan.model.config.id2label[id];
renderBox({ box: boxPercentage, label }, renderWrapper, models.gelan);
}
});
} else if (selection === 'segformer') {
const output = await models.segformer.segmenter(img);
status.textContent = 'Ready';
// output is an array of objects containing { label, mask } where mask is a RawImage
output.forEach(segmentData => renderSemanticMask(segmentData, renderWrapper));
} else if (selection === 'depth') {
const { depth } = await models.depth.estimator(img);
status.textContent = 'Ready';
// Remove the default renderWrapper
imageContainer.removeChild(renderWrapper);
// Override container styles for 3-panel horizontal layout
imageContainer.style.overflow = 'visible';
imageContainer.style.display = 'flex';
imageContainer.style.flexDirection = 'row';
imageContainer.style.gap = '8px';
imageContainer.style.minWidth = '0';
imageContainer.style.minHeight = '0';
imageContainer.style.border = 'none';
imageContainer.style.maxWidth = '100%';
// Helper: create a labeled panel with an image + overlay area
function createPanel(label) {
const panel = document.createElement('div');
panel.style.cssText = 'flex:1; min-width:0; max-width:33.33%;';
const panelLabel = document.createElement('div');
panelLabel.textContent = label;
panelLabel.style.cssText = 'font-size:12px; font-weight:bold; margin-bottom:4px; color:#666;';
const wrap = document.createElement('div');
wrap.style.cssText = 'position:relative; width:100%;';
const panelImg = document.createElement('img');
panelImg.crossOrigin = 'anonymous';
panelImg.src = img;
panelImg.style.cssText = 'display:block; width:100%; height:auto;';
wrap.appendChild(panelImg);
panel.appendChild(panelLabel);
panel.appendChild(wrap);
imageContainer.appendChild(panel);
return { panel, wrap, panelImg };
}
// Panel 1: Original image (reuse the already-appended img)
const origPanel = document.createElement('div');
origPanel.style.cssText = 'flex:1; min-width:0; max-width:33.33%;';
const origLabel = document.createElement('div');
origLabel.textContent = 'Original';
origLabel.style.cssText = 'font-size:12px; font-weight:bold; margin-bottom:4px; color:#666;';
const origWrap = document.createElement('div');
origWrap.style.cssText = 'position:relative; width:100%;';
const existingImg = imageContainer.querySelector('img');
existingImg.style.cssText = 'display:block; width:100%; height:auto;';
origWrap.appendChild(existingImg);
origPanel.appendChild(origLabel);
origPanel.appendChild(origWrap);
imageContainer.appendChild(origPanel);
// Panel 2: Greyscale depth overlay
const { wrap: gsWrap, panelImg: gsImg } = createPanel('Depth Map');
gsImg.style.visibility = 'hidden';
renderGreyscaleDepth(depth, gsWrap);
// Panel 3: 3D layers
const { wrap: wrap3d, panelImg: img3d } = createPanel('3D Layers');
// Wait for image to load so container has dimensions for WebGL canvas
if (img3d.complete) {
renderDepthMap(depth, wrap3d);
} else {
img3d.onload = () => renderDepthMap(depth, wrap3d);
}
}
} catch (e) {
console.error(e);
status.textContent = 'Error during analysis';
}
}
// Draw the output from Segformer
function renderSemanticMask({ label, mask }, container) {
// Generate a random color for the mask
const color = '#' + Math.floor(Math.random() * 0xFFFFFF).toString(16).padStart(6, 0);
const r = parseInt(color.slice(1, 3), 16);
const g = parseInt(color.slice(3, 5), 16);
const b = parseInt(color.slice(5, 7), 16);
const canvas = document.createElement('canvas');
canvas.width = mask.width;
canvas.height = mask.height;
const ctx = canvas.getContext('2d');
const imgData = ctx.createImageData(mask.width, mask.height);
const maskSize = mask.width * mask.height;
for (let i = 0; i < maskSize; ++i) {
const val = mask.data[i];
if (val > 128) {
imgData.data[i * 4 + 0] = r;
imgData.data[i * 4 + 1] = g;
imgData.data[i * 4 + 2] = b;
imgData.data[i * 4 + 3] = 150;
} else {
imgData.data[i * 4 + 0] = 0;
imgData.data[i * 4 + 1] = 0;
imgData.data[i * 4 + 2] = 0;
imgData.data[i * 4 + 3] = 0;
}
}
ctx.putImageData(imgData, 0, 0);
canvas.style.position = 'absolute';
canvas.style.top = '0';
canvas.style.left = '0';
canvas.style.width = '100%';
canvas.style.height = '100%';
let firstIdx = mask.data.findIndex(v => v > 128);
if (firstIdx !== -1) {
const labelElement = document.createElement('span');
labelElement.textContent = label;
labelElement.className = 'bounding-box-label';
labelElement.style.backgroundColor = color;
labelElement.style.display = 'block';
const labelY = Math.floor(firstIdx / mask.width);
const labelX = firstIdx % mask.width;
labelElement.style.left = (labelX / mask.width * 100) + '%';
labelElement.style.top = (labelY / mask.height * 100) + '%';
container.appendChild(labelElement);
}
container.appendChild(canvas);
}
// Render a bounding box and label on the image
function renderBox({ box, label }, container, activeModelCache) {
const { xmax, xmin, ymax, ymin } = box;
const color = '#' + Math.floor(Math.random() * 0xFFFFFF).toString(16).padStart(6, 0);
const boxElement = document.createElement('div');
boxElement.className = 'bounding-box';
Object.assign(boxElement.style, {
borderColor: color,
left: 100 * xmin + '%',
top: 100 * ymin + '%',
width: 100 * (xmax - xmin) + '%',
height: 100 * (ymax - ymin) + '%',
cursor: 'crosshair',
zIndex: 10,
pointerEvents: 'auto',
});
const labelElement = document.createElement('span');
labelElement.textContent = label;
labelElement.className = 'bounding-box-label';
labelElement.style.backgroundColor = color;
labelElement.style.display = 'none';
boxElement.appendChild(labelElement);
container.appendChild(boxElement);
boxElement.addEventListener('click', async (e) => {
e.stopPropagation();
if (!currentRawImage) return;
status.textContent = `Generating mask for ${label}...`;
const originalXmin = xmin * currentRawImage.width;
const originalYmin = ymin * currentRawImage.height;
const originalXmax = xmax * currentRawImage.width;
const originalYmax = ymax * currentRawImage.height;
const centerX = (originalXmin + originalXmax) / 2;
const centerY = (originalYmin + originalYmax) / 2;
const input_boxes = [[[originalXmin, originalYmin, originalXmax, originalYmax]]];
const input_points = [[[centerX, centerY]]];
try {
const inputs = await activeModelCache.samProcessor(currentRawImage, { input_boxes, input_points });
const modelInputs = { ...inputs };
delete modelInputs.input_boxes;
const outputs = await activeModelCache.samModel(modelInputs);
const masks = await activeModelCache.samProcessor.post_process_masks(outputs.pred_masks, inputs.original_sizes, inputs.reshaped_input_sizes);
const scores = outputs.iou_scores.data;
let bestScoreIdx = 0;
let maxScore = -Infinity;
for (let i = 0; i < scores.length; i++) {
if (scores[i] > maxScore) {
maxScore = scores[i];
bestScoreIdx = i;
}
}
const H = masks[0].dims[2];
const W = masks[0].dims[3];
const rawMaskData = masks[0].data;
const maskSize = H * W;
const canvas = document.createElement('canvas');
canvas.width = W;
canvas.height = H;
const ctx = canvas.getContext('2d');
const imgData = ctx.createImageData(W, H);
const r = parseInt(color.slice(1, 3), 16);
const g = parseInt(color.slice(3, 5), 16);
const b = parseInt(color.slice(5, 7), 16);
const channelOffset = bestScoreIdx * maskSize;
for (let i = 0; i < maskSize; ++i) {
const val = rawMaskData[channelOffset + i];
if (val) {
imgData.data[i * 4 + 0] = r;
imgData.data[i * 4 + 1] = g;
imgData.data[i * 4 + 2] = b;
imgData.data[i * 4 + 3] = 150;
} else {
imgData.data[i * 4 + 0] = 0;
imgData.data[i * 4 + 1] = 0;
imgData.data[i * 4 + 2] = 0;
imgData.data[i * 4 + 3] = 0;
}
}
ctx.putImageData(imgData, 0, 0);
canvas.style.position = 'absolute';
canvas.style.top = '0';
canvas.style.left = '0';
canvas.style.width = '100%';
canvas.style.height = '100%';
canvas.style.pointerEvents = 'none';
canvas.style.zIndex = '5';
container.appendChild(canvas);
labelElement.style.display = 'block';
status.textContent = 'Ready';
} catch (err) {
console.error(err);
status.textContent = 'Error during segmentation';
}
});
}
// ── Greyscale Depth Overlay (Xenova-style) ──────────────────────────
function renderGreyscaleDepth(depthImage, container) {
const W = depthImage.width;
const H = depthImage.height;
const pixelCount = W * H;
const depthData = depthImage.data;
let mn = Infinity, mx = -Infinity;
for (let i = 0; i < pixelCount; i++) {
if (depthData[i] < mn) mn = depthData[i];
if (depthData[i] > mx) mx = depthData[i];
}
const rng = mx - mn || 1;
const rgba = new Uint8ClampedArray(4 * pixelCount);
for (let i = 0; i < pixelCount; i++) {
const idx = 4 * i;
rgba[idx] = 255; // Red channel
rgba[idx + 3] = Math.round(255 * (1 - (depthData[i] - mn) / rng));
}
const canvas = document.createElement('canvas');
canvas.width = W;
canvas.height = H;
canvas.getContext('2d').putImageData(new ImageData(rgba, W, H), 0, 0);
Object.assign(canvas.style, {
position: 'absolute', top: '0', left: '0',
width: '100%', height: '100%', pointerEvents: 'none', zIndex: '10'
});
container.appendChild(canvas);
}
// ── Per-Pixel Depth Layer Renderer ──────────────────────────────────
function renderDepthMap(depthImage, container) {
const Z_SPREAD = 1.6;
const NUM_LAYERS = 4;
const W = depthImage.width;
const H = depthImage.height;
const pixelCount = W * H;
const depthData = depthImage.data;
// ── 1. Normalize depth to 0..1 ──
let dMin = Infinity, dMax = -Infinity;
for (let i = 0; i < pixelCount; i++) {
if (depthData[i] < dMin) dMin = depthData[i];
if (depthData[i] > dMax) dMax = depthData[i];
}
const dRange = dMax - dMin || 1;
const norm = new Float32Array(pixelCount);
for (let i = 0; i < pixelCount; i++) {
norm[i] = (depthData[i] - dMin) / dRange;
}
// ── 2. Build histogram of per-pixel depths, find natural boundaries ──
const HIST_BINS = 256;
const histogram = new Float32Array(HIST_BINS);
for (let i = 0; i < pixelCount; i++) {
const bin = Math.min(HIST_BINS - 1, Math.floor(norm[i] * HIST_BINS));
histogram[bin]++;
}
// Smooth histogram with a wide Gaussian kernel
const smoothed = new Float32Array(HIST_BINS);
const KERNEL_RADIUS = 8;
for (let i = 0; i < HIST_BINS; i++) {
let sum = 0, wt = 0;
for (let k = -KERNEL_RADIUS; k <= KERNEL_RADIUS; k++) {
const idx = Math.max(0, Math.min(HIST_BINS - 1, i + k));
const w = Math.exp(-0.5 * (k / (KERNEL_RADIUS / 2)) ** 2);
sum += histogram[idx] * w;
wt += w;
}
smoothed[i] = sum / wt;
}
// Find valleys (local minima) in smoothed histogram
const valleys = [];
for (let i = 2; i < HIST_BINS - 2; i++) {
if (smoothed[i] <= smoothed[i - 1] && smoothed[i] <= smoothed[i + 1] &&
smoothed[i] < smoothed[i - 2] && smoothed[i] < smoothed[i + 2]) {
valleys.push({ bin: i, value: smoothed[i] });
}
}
// Sort by depth of valley (lowest count = best separator)
valleys.sort((a, b) => a.value - b.value);
// Pick the best (NUM_LAYERS-1) valleys as cut points
const numCuts = NUM_LAYERS - 1;
let cutBins;
if (valleys.length >= numCuts) {
cutBins = valleys.slice(0, numCuts).map(v => v.bin).sort((a, b) => a - b);
} else {
// Fallback: uniform cuts
cutBins = [];
for (let i = 1; i < NUM_LAYERS; i++) {
cutBins.push(Math.round((i / NUM_LAYERS) * HIST_BINS));
}
}
// Convert bin indices to depth boundaries
const boundaries = [0, ...cutBins.map(b => b / HIST_BINS), 1.01];
console.log(`Depth: ${NUM_LAYERS} layers, boundaries: [${boundaries.map(b => b.toFixed(2)).join(', ')}]`);
// ── 3. Assign each pixel to a layer ──
const pixelLayer = new Uint8Array(pixelCount);
for (let i = 0; i < pixelCount; i++) {
const d = norm[i];
for (let l = 0; l < NUM_LAYERS; l++) {
if (d >= boundaries[l] && d < boundaries[l + 1]) {
pixelLayer[i] = l;
break;
}
}
}
// Merge small layers into the one above (nearer)
const MIN_PIXEL_RATIO = 0.05; // layers with < 5% of pixels get merged
const layerCounts = new Uint32Array(NUM_LAYERS);
for (let i = 0; i < pixelCount; i++) layerCounts[pixelLayer[i]]++;
const mergeMap = new Uint8Array(NUM_LAYERS);
for (let l = 0; l < NUM_LAYERS; l++) mergeMap[l] = l;
for (let l = 0; l < NUM_LAYERS; l++) {
if (layerCounts[l] < pixelCount * MIN_PIXEL_RATIO) {
// Merge into the layer above (l+1), or below if it's the last
const target = l < NUM_LAYERS - 1 ? l + 1 : l - 1;
if (target >= 0) mergeMap[l] = mergeMap[target];
}
}
// Apply merge and remap to contiguous indices
const usedLayers = [...new Set(Array.from(mergeMap))].sort((a, b) => a - b);
const remapTable = new Uint8Array(NUM_LAYERS);
usedLayers.forEach((old, idx) => remapTable[old] = idx);
for (let i = 0; i < pixelCount; i++) {
pixelLayer[i] = remapTable[mergeMap[pixelLayer[i]]];
}
const FINAL_LAYERS = usedLayers.length;
console.log(`After merge: ${FINAL_LAYERS} layers (from ${NUM_LAYERS})`);
// ── 4. Get original image pixels ──
const tmpCanvas = document.createElement('canvas');
tmpCanvas.width = W; tmpCanvas.height = H;
const tmpCtx = tmpCanvas.getContext('2d');
const imgEl = container.querySelector('img');
tmpCtx.drawImage(imgEl, 0, 0, W, H);
const origPixels = tmpCtx.getImageData(0, 0, W, H).data;
imgEl.style.visibility = 'hidden';
// ── 5. Create masked textures per layer + build wall geometry ──
const levelTextures = [];
const aspect = W / H;
// Per-layer: collect wall quad vertices (position xyz + color rgb, 6 floats/vertex)
const wallVertArrays = []; // wallVertArrays[layer] = Float32Array of wall verts
for (let layer = 0; layer < FINAL_LAYERS; layer++) {
const rgba = new Uint8Array(W * H * 4);
const wallVerts = []; // temp array to collect wall vertex data
for (let i = 0; i < pixelCount; i++) {
if (pixelLayer[i] !== layer) continue;
rgba[i * 4] = origPixels[i * 4];
rgba[i * 4 + 1] = origPixels[i * 4 + 1];
rgba[i * 4 + 2] = origPixels[i * 4 + 2];
rgba[i * 4 + 3] = 255;
// Only build walls for layers that have a layer behind them
if (layer === 0) continue;
const px = i % W, py = (i / W) | 0;
const r = origPixels[i * 4] / 255;
const g = origPixels[i * 4 + 1] / 255;
const b = origPixels[i * 4 + 2] / 255;
// Map pixel bounds to world coords
const xL = aspect * (2 * px / W - 1);
const xR = aspect * (2 * (px + 1) / W - 1);
const yT = 1 - 2 * py / H;
const yB = 1 - 2 * (py + 1) / H;
// Check each face - if neighbor is a different layer, create a wall quad
// Wall quad: z=0 is front (this layer), z=1 is back (layer behind)
// Each quad = 2 triangles = 6 vertices, each vertex = (x,y,z, r,g,b)
// Right wall
if (px < W - 1 && pixelLayer[i + 1] !== layer) {
wallVerts.push(xR, yT, 0, r, g, b, xR, yT, 1, r, g, b, xR, yB, 0, r, g, b);
wallVerts.push(xR, yB, 0, r, g, b, xR, yT, 1, r, g, b, xR, yB, 1, r, g, b);
}
// Left wall
if (px > 0 && pixelLayer[i - 1] !== layer) {
wallVerts.push(xL, yT, 0, r, g, b, xL, yB, 0, r, g, b, xL, yT, 1, r, g, b);
wallVerts.push(xL, yB, 0, r, g, b, xL, yB, 1, r, g, b, xL, yT, 1, r, g, b);
}
// Bottom wall
if (py < H - 1 && pixelLayer[i + W] !== layer) {
wallVerts.push(xL, yB, 0, r, g, b, xR, yB, 1, r, g, b, xR, yB, 0, r, g, b);
wallVerts.push(xL, yB, 0, r, g, b, xL, yB, 1, r, g, b, xR, yB, 1, r, g, b);
}
// Top wall
if (py > 0 && pixelLayer[i - W] !== layer) {
wallVerts.push(xL, yT, 0, r, g, b, xR, yT, 0, r, g, b, xR, yT, 1, r, g, b);
wallVerts.push(xL, yT, 0, r, g, b, xR, yT, 1, r, g, b, xL, yT, 1, r, g, b);
}
}
levelTextures.push(rgba);
wallVertArrays.push(new Float32Array(wallVerts));
}
console.log(`Wall geometry: ${wallVertArrays.map((a, i) => `L${i}:${a.length / 6} verts`).join(', ')}`);
// ── 6. WebGL 3D Render ──
const glCanvas = document.createElement('canvas');
const rect = container.getBoundingClientRect();
glCanvas.width = rect.width * devicePixelRatio;
glCanvas.height = rect.height * devicePixelRatio;
Object.assign(glCanvas.style, {
position: 'absolute', top: '0', left: '0',
width: '100%', height: '100%', zIndex: '20', pointerEvents: 'auto', cursor: 'grab'
});
container.appendChild(glCanvas);
const gl = glCanvas.getContext('webgl', { alpha: true, premultipliedAlpha: false });
if (!gl) { console.error('WebGL not supported'); return; }
gl.enable(gl.BLEND);
gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
gl.enable(gl.DEPTH_TEST);
gl.depthFunc(gl.LEQUAL);
function compileShader(src, type) {
const s = gl.createShader(type); gl.shaderSource(s, src); gl.compileShader(s);
if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) { console.error(gl.getShaderInfoLog(s)); return null; }
return s;
}
// ── Face shader (textured quads) ──
const faceProg = gl.createProgram();
gl.attachShader(faceProg, compileShader(`
attribute vec2 aPos; attribute vec2 aUV; uniform mat4 uMVP; varying vec2 vUV;
void main() { vUV = aUV; gl_Position = uMVP * vec4(aPos, 0.0, 1.0); }`, gl.VERTEX_SHADER));
gl.attachShader(faceProg, compileShader(`
precision mediump float; varying vec2 vUV; uniform sampler2D uTex;
void main() { vec4 c = texture2D(uTex, vUV); if (c.a < 0.01) discard; gl_FragColor = c; }`, gl.FRAGMENT_SHADER));
gl.linkProgram(faceProg);
const faceAPos = gl.getAttribLocation(faceProg, 'aPos');
const faceAUV = gl.getAttribLocation(faceProg, 'aUV');
const faceUMVP = gl.getUniformLocation(faceProg, 'uMVP');
const faceUTex = gl.getUniformLocation(faceProg, 'uTex');
const faceVerts = new Float32Array([-aspect, 1, 0, 0, aspect, 1, 1, 0, -aspect, -1, 0, 1, aspect, -1, 1, 1]);
const faceVBO = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, faceVBO);
gl.bufferData(gl.ARRAY_BUFFER, faceVerts, gl.STATIC_DRAW);
// Upload face textures
const glTextures = [];
for (let lvl = 0; lvl < FINAL_LAYERS; lvl++) {
const tex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, tex);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, W, H, 0, gl.RGBA, gl.UNSIGNED_BYTE, levelTextures[lvl]);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
glTextures.push(tex);
}
// ── Wall shader (solid colored geometry) ──
const wallProg = gl.createProgram();
gl.attachShader(wallProg, compileShader(`
attribute vec3 aWallPos; attribute vec3 aWallColor;
uniform mat4 uMVP; uniform float uZFront; uniform float uZBack;
varying vec3 vColor;
void main() {
float z = mix(uZFront, uZBack, aWallPos.z);
gl_Position = uMVP * vec4(aWallPos.xy, z, 1.0);
vColor = aWallColor;
}`, gl.VERTEX_SHADER));
gl.attachShader(wallProg, compileShader(`
precision mediump float; varying vec3 vColor;
void main() { gl_FragColor = vec4(vColor, 1.0); }`, gl.FRAGMENT_SHADER));
gl.linkProgram(wallProg);
const wallAPos = gl.getAttribLocation(wallProg, 'aWallPos');
const wallAColor = gl.getAttribLocation(wallProg, 'aWallColor');
const wallUMVP = gl.getUniformLocation(wallProg, 'uMVP');
const wallUZFront = gl.getUniformLocation(wallProg, 'uZFront');
const wallUZBack = gl.getUniformLocation(wallProg, 'uZBack');
// Upload wall geometry buffers
const wallVBOs = [];
const wallVertCounts = [];
for (let lvl = 0; lvl < FINAL_LAYERS; lvl++) {
const vbo = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, vbo);
gl.bufferData(gl.ARRAY_BUFFER, wallVertArrays[lvl], gl.STATIC_DRAW);
wallVBOs.push(vbo);
wallVertCounts.push(wallVertArrays[lvl].length / 6); // 6 floats per vertex
}
// Matrix helpers
const m4 = {
perspective(fovY, asp, near, far) {
const f = 1 / Math.tan(fovY / 2), nf = 1 / (near - far);
return new Float32Array([f / asp, 0, 0, 0, 0, f, 0, 0, 0, 0, (far + near) * nf, -1, 0, 0, 2 * far * near * nf, 0]);
},
rotY(a) { const c = Math.cos(a), s = Math.sin(a); return new Float32Array([c, 0, s, 0, 0, 1, 0, 0, -s, 0, c, 0, 0, 0, 0, 1]); },
rotX(a) { const c = Math.cos(a), s = Math.sin(a); return new Float32Array([1, 0, 0, 0, 0, c, -s, 0, 0, s, c, 0, 0, 0, 0, 1]); },
translate(x, y, z) { return new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, x, y, z, 1]); },
mul(a, b) {
const o = new Float32Array(16);
for (let i = 0; i < 4; i++) for (let j = 0; j < 4; j++) {
o[j * 4 + i] = 0; for (let k = 0; k < 4; k++) o[j * 4 + i] += a[k * 4 + i] * b[j * 4 + k];
}
return o;
}
};
const canvasAspect = glCanvas.width / glCanvas.height;
const fov = Math.PI / 4;
const proj = m4.perspective(fov, canvasAspect, 0.1, 100);
const camDist = 1.0 / Math.tan(fov / 2) + 0.1;
let rotYAngle = 0, rotXAngle = 0, dragging = false, lastMX = 0, lastMY = 0;
glCanvas.addEventListener('mousedown', e => { dragging = true; lastMX = e.clientX; lastMY = e.clientY; glCanvas.style.cursor = 'grabbing'; });
window.addEventListener('mouseup', () => { dragging = false; glCanvas.style.cursor = 'grab'; });
window.addEventListener('mousemove', e => {
if (!dragging) return;
rotYAngle += (e.clientX - lastMX) * 0.006; rotXAngle += (e.clientY - lastMY) * 0.006;
rotXAngle = Math.max(-Math.PI / 3, Math.min(Math.PI / 3, rotXAngle));
lastMX = e.clientX; lastMY = e.clientY;
});
glCanvas.addEventListener('touchstart', e => { dragging = true; lastMX = e.touches[0].clientX; lastMY = e.touches[0].clientY; }, { passive: true });
window.addEventListener('touchend', () => { dragging = false; });
window.addEventListener('touchmove', e => {
if (!dragging) return;
rotYAngle += (e.touches[0].clientX - lastMX) * 0.006; rotXAngle += (e.touches[0].clientY - lastMY) * 0.006;
rotXAngle = Math.max(-Math.PI / 3, Math.min(Math.PI / 3, rotXAngle));
lastMX = e.touches[0].clientX; lastMY = e.touches[0].clientY;
}, { passive: true });
const startTime = performance.now();
const INTRO_MS = 1000;
function easeOut(t) { return 1 - Math.pow(1 - t, 3); }
function draw() {
gl.viewport(0, 0, glCanvas.width, glCanvas.height);
gl.clearColor(0, 0, 0, 0);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
const t = Math.min((performance.now() - startTime) / INTRO_MS, 1);
const spread = Z_SPREAD * easeOut(t);
const view = m4.translate(0, 0, -camDist);
const rot = m4.mul(m4.rotY(rotYAngle), m4.rotX(rotXAngle));
const vp = m4.mul(proj, m4.mul(view, rot));
// Compute Z positions for each layer
const zPositions = [];
for (let lvl = 0; lvl < FINAL_LAYERS; lvl++) {
zPositions.push(FINAL_LAYERS > 1 ? (lvl / (FINAL_LAYERS - 1)) * spread - spread / 2 : 0);
}
for (let lvl = 0; lvl < FINAL_LAYERS; lvl++) {
const zFront = zPositions[lvl];
// Draw wall geometry between this layer and the one behind
if (lvl > 0 && wallVertCounts[lvl] > 0) {
const zBack = zPositions[lvl - 1];
gl.useProgram(wallProg);
gl.bindBuffer(gl.ARRAY_BUFFER, wallVBOs[lvl]);
gl.enableVertexAttribArray(wallAPos);
gl.vertexAttribPointer(wallAPos, 3, gl.FLOAT, false, 24, 0);
gl.enableVertexAttribArray(wallAColor);
gl.vertexAttribPointer(wallAColor, 3, gl.FLOAT, false, 24, 12);
gl.uniformMatrix4fv(wallUMVP, false, vp);
gl.uniform1f(wallUZFront, zFront);
gl.uniform1f(wallUZBack, zBack);
gl.drawArrays(gl.TRIANGLES, 0, wallVertCounts[lvl]);
}
// Draw the face quad
gl.useProgram(faceProg);
gl.bindBuffer(gl.ARRAY_BUFFER, faceVBO);
gl.enableVertexAttribArray(faceAPos);
gl.vertexAttribPointer(faceAPos, 2, gl.FLOAT, false, 16, 0);
gl.enableVertexAttribArray(faceAUV);
gl.vertexAttribPointer(faceAUV, 2, gl.FLOAT, false, 16, 8);
const mvp = m4.mul(vp, m4.translate(0, 0, zFront));
gl.uniformMatrix4fv(faceUMVP, false, mvp);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, glTextures[lvl]);
gl.uniform1i(faceUTex, 0);
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
}
requestAnimationFrame(draw);
}
draw();
}