import * as ort from "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.21.0/dist/ort.min.mjs"; const CLASS_NAMES = ["fire", "smoke", "fire extinguisher"]; const CLS_REMAP = [0, 2, 1]; const CLASS_COLORS = { fire: "#EF4444", smoke: "#64748B", "fire extinguisher": "#DC2626", }; const CONFIG = { confThres: [0.15, 0.3, 0.23], bonus: [0.02, 0.1, 0.1], iouThres: 0.55, crossIouThresh: 0.8, maxDet: 150, useTta: false, minBoxArea: 14 * 14, minSide: 8, maxAspectRatio: 8.0, maxBoxAreaRatio: 0.95, smokeMergeOverlap: 0.8, fireMergeOverlap: 1.01, fireSuppressOverlap: 0.88, fireColorFilterMaxConf: 0.45, fireExtColorFilterMaxConf: 0.4, colorFilterMinSaturation: 0.06, }; function safeDim(value, fallback) { return Number.isInteger(value) && value > 0 ? value : fallback; } function remapClassId(clsId) { return CLS_REMAP[clsId] ?? clsId; } function clipBoxes(boxes, imageW, imageH) { for (const box of boxes) { box[0] = Math.min(Math.max(box[0], 0), imageW - 1); box[1] = Math.min(Math.max(box[1], 0), imageH - 1); box[2] = Math.min(Math.max(box[2], 0), imageW - 1); box[3] = Math.min(Math.max(box[3], 0), imageH - 1); } return boxes; } function hardNms(boxes, scores, iouThresh) { const n = boxes.length; if (n === 0) return []; const order = scores .map((score, index) => [score, index]) .sort((a, b) => b[0] - a[0]) .map((pair) => pair[1]); const keep = []; while (order.length > 0) { const i = order.shift(); keep.push(i); if (order.length === 0) break; const rest = []; const boxI = boxes[i]; const areaI = Math.max(0, boxI[2] - boxI[0]) * Math.max(0, boxI[3] - boxI[1]); for (const j of order) { const boxJ = boxes[j]; const xx1 = Math.max(boxI[0], boxJ[0]); const yy1 = Math.max(boxI[1], boxJ[1]); const xx2 = Math.min(boxI[2], boxJ[2]); const yy2 = Math.min(boxI[3], boxJ[3]); const inter = Math.max(0, xx2 - xx1) * Math.max(0, yy2 - yy1); const areaJ = Math.max(0, boxJ[2] - boxJ[0]) * Math.max(0, boxJ[3] - boxJ[1]); const iou = inter / (areaI + areaJ - inter + 1e-7); if (iou <= iouThresh) rest.push(j); } order.length = 0; order.push(...rest); } return keep; } function perClassHardNms(boxes, scores, clsIds, iouThresh) { if (boxes.length === 0) return []; const classes = [...new Set(clsIds)]; const allKeep = []; for (const cls of classes) { const indices = clsIds.map((id, i) => (id === cls ? i : -1)).filter((i) => i >= 0); const classBoxes = indices.map((i) => boxes[i]); const classScores = indices.map((i) => scores[i]); const keep = hardNms(classBoxes, classScores, iouThresh); allKeep.push(...keep.map((k) => indices[k])); } allKeep.sort((a, b) => a - b); return allKeep; } function confFilterMask(scores, clsIds) { const keep = scores.map((score, i) => score >= CONFIG.confThres[clsIds[i]]); const classes = [...new Set(clsIds)]; for (const c of classes) { if (CONFIG.bonus[c] <= 0) continue; const indices = clsIds.map((id, i) => (id === c ? i : -1)).filter((i) => i >= 0); if (indices.some((i) => keep[i])) continue; let top = indices[0]; for (const i of indices) { if (scores[i] > scores[top]) top = i; } if (scores[top] >= CONFIG.confThres[c] - CONFIG.bonus[c]) keep[top] = true; } return keep; } function filterSaneBoxes(boxes, scores, clsIds, origW, origH) { const imageArea = origW * origH; const keptBoxes = []; const keptScores = []; const keptCls = []; for (let i = 0; i < boxes.length; i += 1) { const [x1, y1, x2, y2] = boxes[i]; const bw = x2 - x1; const bh = y2 - y1; if (bw <= 0 || bh <= 0) continue; if (bw < CONFIG.minSide || bh < CONFIG.minSide) continue; const area = bw * bh; if (area < CONFIG.minBoxArea) continue; if (area > CONFIG.maxBoxAreaRatio * imageArea) continue; const ar = Math.max(bw / Math.max(bh, 1e-6), bh / Math.max(bw, 1e-6)); if (ar > CONFIG.maxAspectRatio) continue; keptBoxes.push(boxes[i]); keptScores.push(scores[i]); keptCls.push(clsIds[i]); } return { boxes: keptBoxes, scores: keptScores, clsIds: keptCls }; } function crossClassDedup(boxes, scores, clsIds, iouThresh) { const n = boxes.length; if (n <= 1) return { boxes, scores, clsIds }; const areas = boxes.map(([x1, y1, x2, y2]) => Math.max(0, x2 - x1) * Math.max(0, y2 - y1)); const margins = scores.map((s, i) => s - CONFIG.confThres[clsIds[i]]); const order = [...Array(n).keys()].sort((a, b) => { if (margins[b] !== margins[a]) return margins[b] - margins[a]; return areas[b] - areas[a]; }); const suppressed = new Array(n).fill(false); const keep = []; for (const i of order) { if (suppressed[i]) continue; keep.push(i); const bi = boxes[i]; const areaI = Math.max(1e-7, (bi[2] - bi[0]) * (bi[3] - bi[1])); for (let j = 0; j < n; j += 1) { if (j === i || suppressed[j]) continue; const bj = boxes[j]; const xx1 = Math.max(bi[0], bj[0]); const yy1 = Math.max(bi[1], bj[1]); const xx2 = Math.min(bi[2], bj[2]); const yy2 = Math.min(bi[3], bj[3]); const inter = Math.max(0, xx2 - xx1) * Math.max(0, yy2 - yy1); const iou = inter / (areaI + areas[j] - inter + 1e-7); if (iou > iouThresh) suppressed[j] = true; } } return { boxes: keep.map((i) => boxes[i]), scores: keep.map((i) => scores[i]), clsIds: keep.map((i) => clsIds[i]), }; } function mergeClassBoxes(boxes, scores, clsIds, targetCls, overlap) { if (overlap > 1.0) return { boxes, scores, clsIds }; const idx = clsIds.map((id, i) => (id === targetCls ? i : -1)).filter((i) => i >= 0); if (idx.length <= 1) return { boxes, scores, clsIds }; let sb = idx.map((i) => [...boxes[i]]); let ss = idx.map((i) => scores[i]); let mergedAny = true; while (mergedAny && sb.length > 1) { mergedAny = false; for (let i = 0; i < sb.length; i += 1) { for (let j = i + 1; j < sb.length; j += 1) { const a = sb[i]; const b = sb[j]; const ix1 = Math.max(a[0], b[0]); const iy1 = Math.max(a[1], b[1]); const ix2 = Math.min(a[2], b[2]); const iy2 = Math.min(a[3], b[3]); const inter = Math.max(0, ix2 - ix1) * Math.max(0, iy2 - iy1); const areaA = Math.max(0, a[2] - a[0]) * Math.max(0, a[3] - a[1]); const areaB = Math.max(0, b[2] - b[0]) * Math.max(0, b[3] - b[1]); if (inter / (Math.min(areaA, areaB) + 1e-7) >= overlap) { sb[i] = [Math.min(a[0], b[0]), Math.min(a[1], b[1]), Math.max(a[2], b[2]), Math.max(a[3], b[3])]; ss[i] = Math.max(ss[i], ss[j]); sb.splice(j, 1); ss.splice(j, 1); mergedAny = true; break; } } if (mergedAny) break; } } const other = []; const otherScores = []; const otherCls = []; for (let i = 0; i < boxes.length; i += 1) { if (clsIds[i] !== targetCls) { other.push(boxes[i]); otherScores.push(scores[i]); otherCls.push(clsIds[i]); } } return { boxes: [...other, ...sb], scores: [...otherScores, ...ss], clsIds: [...otherCls, ...new Array(sb.length).fill(targetCls)], }; } function suppressContainedLowerConf(boxes, scores, clsIds, targetCls, overlap) { if (overlap > 1.0) return { boxes, scores, clsIds }; const idx = clsIds.map((id, i) => (id === targetCls ? i : -1)).filter((i) => i >= 0); if (idx.length <= 1) return { boxes, scores, clsIds }; const order = [...idx].sort((a, b) => scores[b] - scores[a]); const remove = new Set(); for (let a = 0; a < order.length; a += 1) { const i = order[a]; if (remove.has(i)) continue; const bi = boxes[i]; const areaI = Math.max(1e-7, (bi[2] - bi[0]) * (bi[3] - bi[1])); for (let b = a + 1; b < order.length; b += 1) { const j = order[b]; if (remove.has(j)) continue; const bj = boxes[j]; const ix1 = Math.max(bi[0], bj[0]); const iy1 = Math.max(bi[1], bj[1]); const ix2 = Math.min(bi[2], bj[2]); const iy2 = Math.min(bi[3], bj[3]); const inter = Math.max(0, ix2 - ix1) * Math.max(0, iy2 - iy1); if (inter <= 0) continue; const areaJ = Math.max(1e-7, (bj[2] - bj[0]) * (bj[3] - bj[1])); if (inter / (Math.min(areaI, areaJ) + 1e-7) >= overlap) remove.add(j); } } if (remove.size === 0) return { boxes, scores, clsIds }; const keep = boxes.map((_, i) => !remove.has(i)); return { boxes: boxes.filter((_, i) => keep[i]), scores: scores.filter((_, i) => keep[i]), clsIds: clsIds.filter((_, i) => keep[i]), }; } function mergeSameClassBoxes(boxes, scores, clsIds) { let state = mergeClassBoxes(boxes, scores, clsIds, CLASS_NAMES.indexOf("smoke"), CONFIG.smokeMergeOverlap); state = mergeClassBoxes(state.boxes, state.scores, state.clsIds, CLASS_NAMES.indexOf("fire"), CONFIG.fireMergeOverlap); state = suppressContainedLowerConf(state.boxes, state.scores, state.clsIds, CLASS_NAMES.indexOf("fire"), CONFIG.fireSuppressOverlap); return state; } function perViewPipeline(boxes, scores, clsIds) { if (boxes.length > 1) { const keep = perClassHardNms(boxes, scores, clsIds, CONFIG.iouThres); boxes = keep.map((i) => boxes[i]); scores = keep.map((i) => scores[i]); clsIds = keep.map((i) => clsIds[i]); } if (scores.length > CONFIG.maxDet) { const order = scores.map((s, i) => [s, i]).sort((a, b) => b[0] - a[0]).slice(0, CONFIG.maxDet).map((p) => p[1]); boxes = order.map((i) => boxes[i]); scores = order.map((i) => scores[i]); clsIds = order.map((i) => clsIds[i]); } if (boxes.length > 1) { const deduped = crossClassDedup(boxes, scores, clsIds, CONFIG.crossIouThresh); boxes = deduped.boxes; scores = deduped.scores; clsIds = deduped.clsIds; } if (boxes.length > 1) { const merged = mergeSameClassBoxes(boxes, scores, clsIds); boxes = merged.boxes; scores = merged.scores; clsIds = merged.clsIds; } return { boxes, scores, clsIds }; } function toBoundingBoxes(boxes, scores, clsIds) { const out = []; for (let i = 0; i < boxes.length; i += 1) { const [x1, y1, x2, y2] = boxes[i]; if (x2 <= x1 || y2 <= y1) continue; out.push({ x1: Math.floor(x1), y1: Math.floor(y1), x2: Math.ceil(x2), y2: Math.ceil(y2), cls_id: clsIds[i], conf: scores[i], }); } return out; } function letterboxCanvas(image, newW, newH) { const srcW = image.width; const srcH = image.height; const ratio = Math.min(newW / srcW, newH / srcH); const resizedW = Math.round(srcW * ratio); const resizedH = Math.round(srcH * ratio); const resized = document.createElement("canvas"); resized.width = resizedW; resized.height = resizedH; resized.getContext("2d").drawImage(image, 0, 0, resizedW, resizedH); const padW = (newW - resizedW) / 2; const padH = (newH - resizedH) / 2; const canvas = document.createElement("canvas"); canvas.width = newW; canvas.height = newH; const ctx = canvas.getContext("2d"); ctx.fillStyle = "rgb(114,114,114)"; ctx.fillRect(0, 0, newW, newH); ctx.drawImage(resized, padW, padH); return { canvas, ratio, pad: [padW, padH] }; } function imageToTensor(canvas, inputW, inputH) { const { data } = canvas.getContext("2d").getImageData(0, 0, inputW, inputH); const tensor = new Float32Array(1 * 3 * inputH * inputW); const plane = inputH * inputW; for (let y = 0; y < inputH; y += 1) { for (let x = 0; x < inputW; x += 1) { const i = (y * inputW + x) * 4; const offset = y * inputW + x; tensor[offset] = data[i] / 255; tensor[plane + offset] = data[i + 1] / 255; tensor[2 * plane + offset] = data[i + 2] / 255; } } return tensor; } function transpose(matrix) { const rows = matrix.length; const cols = matrix[0].length; const out = Array.from({ length: cols }, () => new Array(rows)); for (let r = 0; r < rows; r += 1) { for (let c = 0; c < cols; c += 1) { out[c][r] = matrix[r][c]; } } return out; } function tensorToNestedArray(tensor) { const data = Array.from(tensor.data); const dims = tensor.dims; if (dims.length === 2) { const [rows, cols] = dims; const out = []; for (let r = 0; r < rows; r += 1) out.push(data.slice(r * cols, (r + 1) * cols)); return out; } if (dims.length === 3) { const [batch, dim1, dim2] = dims; const out = []; const stride = dim1 * dim2; for (let b = 0; b < batch; b += 1) { const batchData = data.slice(b * stride, (b + 1) * stride); const rows = []; for (let r = 0; r < dim1; r += 1) rows.push(batchData.slice(r * dim2, (r + 1) * dim2)); out.push(rows); } return out; } throw new Error(`Unsupported output tensor shape: ${dims.join("x")}`); } function decodeFinalDets(preds, ratio, pad, origW, origH) { let rows = preds; if (preds.length === 1 && Array.isArray(preds[0]) && Array.isArray(preds[0][0])) { rows = preds[0]; } const rawScores = []; const rawCls = []; for (const row of rows) { if (row.length < 6) continue; rawScores.push(row[4]); rawCls.push(remapClassId(Math.round(row[5]))); } const mask = confFilterMask(rawScores, rawCls); let boxes = []; let scores = []; let clsIds = []; for (let i = 0; i < rows.length; i += 1) { if (!mask[i] || rows[i].length < 6) continue; boxes.push(rows[i].slice(0, 4)); scores.push(rows[i][4]); clsIds.push(rawCls[i]); } if (boxes.length === 0) return []; const [padW, padH] = pad; boxes = boxes.map(([x1, y1, x2, y2]) => [ (x1 - padW) / ratio, (y1 - padH) / ratio, (x2 - padW) / ratio, (y2 - padH) / ratio, ]); clipBoxes(boxes, origW, origH); let filtered = filterSaneBoxes(boxes, scores, clsIds, origW, origH); if (filtered.boxes.length === 0) return []; const piped = perViewPipeline(filtered.boxes, filtered.scores, filtered.clsIds); return toBoundingBoxes(piped.boxes, piped.scores, piped.clsIds); } function decodeRawYolo(preds, ratio, pad, origW, origH) { let rows = preds[0]; if (rows.length <= 16 && rows[0].length > rows.length) rows = transpose(rows); const rawScores = []; const rawCls = []; const boxesXywh = []; for (const row of rows) { if (row.length < 5) continue; const tail = row.slice(4); let score; let clsId; if (tail.length === 1) { score = tail[0]; clsId = 0; } else { clsId = tail.indexOf(Math.max(...tail)); score = tail[clsId]; } clsId = remapClassId(clsId); rawScores.push(score); rawCls.push(clsId); boxesXywh.push(row.slice(0, 4)); } const mask = confFilterMask(rawScores, rawCls); let boxes = []; let scores = []; let clsIds = []; for (let i = 0; i < boxesXywh.length; i += 1) { if (!mask[i]) continue; const [x, y, w, h] = boxesXywh[i]; boxes.push([x - w / 2, y - h / 2, x + w / 2, y + h / 2]); scores.push(rawScores[i]); clsIds.push(rawCls[i]); } if (boxes.length === 0) return []; const [padW, padH] = pad; boxes = boxes.map(([x1, y1, x2, y2]) => [ (x1 - padW) / ratio, (y1 - padH) / ratio, (x2 - padW) / ratio, (y2 - padH) / ratio, ]); clipBoxes(boxes, origW, origH); let filtered = filterSaneBoxes(boxes, scores, clsIds, origW, origH); if (filtered.boxes.length === 0) return []; const piped = perViewPipeline(filtered.boxes, filtered.scores, filtered.clsIds); return toBoundingBoxes(piped.boxes, piped.scores, piped.clsIds); } function postprocess(output, ratio, pad, origW, origH) { const preds = tensorToNestedArray(output); if (output.dims.length === 2 && output.dims[1] >= 6) { return decodeFinalDets(preds, ratio, pad, origW, origH); } if (output.dims.length === 3 && output.dims[0] === 1 && output.dims[2] >= 6) { return decodeFinalDets(preds, ratio, pad, origW, origH); } return decodeRawYolo(preds, ratio, pad, origW, origH); } function getRoiData(image, box) { const canvas = document.createElement("canvas"); const w = image.width; const h = image.height; const x1 = Math.max(0, Math.floor(box.x1)); const y1 = Math.max(0, Math.floor(box.y1)); const x2 = Math.min(w, Math.ceil(box.x2)); const y2 = Math.min(h, Math.ceil(box.y2)); if (x2 <= x1 || y2 <= y1) return null; canvas.width = x2 - x1; canvas.height = y2 - y1; const ctx = canvas.getContext("2d"); ctx.drawImage(image, x1, y1, x2 - x1, y2 - y1, 0, 0, x2 - x1, y2 - y1); return ctx.getImageData(0, 0, x2 - x1, y2 - y1).data; } function roiIsNearGrayscale(data) { let satSum = 0; const pixels = data.length / 4; for (let i = 0; i < data.length; i += 4) { const r = data[i]; const g = data[i + 1]; const b = data[i + 2]; const mx = Math.max(r, g, b); const mn = Math.min(r, g, b); satSum += (mx - mn) / (mx + 1e-6); } return satSum / pixels < CONFIG.colorFilterMinSaturation; } function passesFireColor(data) { let meanR = 0; let meanG = 0; let maxRgb = 0; let brightCount = 0; let warmCount = 0; const pixels = data.length / 4; for (let i = 0; i < data.length; i += 4) { const r = data[i]; const g = data[i + 1]; const b = data[i + 2]; meanR += r; meanG += g; maxRgb = Math.max(maxRgb, r, g, b); if (Math.max(r, g, b) >= 150) brightCount += 1; if (r > g + 10 && r > b + 10) warmCount += 1; } meanR /= pixels; meanG /= pixels; const brightFrac = brightCount / pixels; const warmFrac = warmCount / pixels; if (maxRgb >= 200 && brightFrac >= 0.01) return true; if (warmFrac >= 0.05 && (maxRgb >= 120 || meanR >= 120 || warmFrac >= 0.15)) return true; if (brightFrac >= 0.12 && meanR - meanG >= 2) return true; return false; } function passesFireExtColor(data) { let redDom = 0; let meanR = 0; let meanG = 0; const pixels = data.length / 4; for (let i = 0; i < data.length; i += 4) { const r = data[i]; const g = data[i + 1]; const b = data[i + 2]; meanR += r; meanG += g; if (r > g + 10 && r > b + 10) redDom += 1; } meanR /= pixels; meanG /= pixels; if (redDom / pixels >= 0.03) return true; return meanR - meanG >= 0 && meanR >= 50; } function filterLowConfByColor(image, boxes) { const clsFire = CLASS_NAMES.indexOf("fire"); const clsExt = CLASS_NAMES.indexOf("fire extinguisher"); const out = []; for (const box of boxes) { const checkFire = box.cls_id === clsFire && box.conf <= CONFIG.fireColorFilterMaxConf; const checkExt = box.cls_id === clsExt && box.conf <= CONFIG.fireExtColorFilterMaxConf; if (!checkFire && !checkExt) { out.push(box); continue; } const data = getRoiData(image, box); if (!data || roiIsNearGrayscale(data)) { out.push(box); continue; } if (checkFire && !passesFireColor(data)) continue; if (checkExt && !passesFireExtColor(data)) continue; out.push(box); } return out; } export class FireDetector { constructor() { this.session = null; this.inputName = null; this.outputNames = null; this.inputHeight = 1280; this.inputWidth = 1280; } async load(modelUrl) { ort.env.wasm.wasmPaths = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.21.0/dist/"; this.session = await ort.InferenceSession.create(modelUrl, { executionProviders: ["wasm"], }); this.inputName = this.session.inputNames[0]; this.outputNames = this.session.outputNames; const shape = this.session.inputs.get(this.inputName).dims; this.inputHeight = safeDim(shape[2], 1280); this.inputWidth = safeDim(shape[3], 1280); } async predictSingle(image) { const origW = image.width; const origH = image.height; const { canvas, ratio, pad } = letterboxCanvas(image, this.inputWidth, this.inputHeight); const tensorData = imageToTensor(canvas, this.inputWidth, this.inputHeight); const inputTensor = new ort.Tensor("float32", tensorData, [1, 3, this.inputHeight, this.inputWidth]); const outputs = await this.session.run({ [this.inputName]: inputTensor }); return postprocess(outputs[this.outputNames[0]], ratio, pad, origW, origH); } async predictImage(image) { const boxes = await this.predictSingle(image); return filterLowConfByColor(image, boxes); } } export function className(clsId) { return CLASS_NAMES[clsId] ?? "unknown"; } export function drawBoxes(ctx, boxes) { ctx.lineWidth = 2; ctx.font = "600 12px Inter, system-ui, sans-serif"; for (const box of boxes) { const label = CLASS_NAMES[box.cls_id] ?? "unknown"; const color = CLASS_COLORS[label] ?? "#EF4444"; const w = box.x2 - box.x1; const h = box.y2 - box.y1; ctx.strokeStyle = color; ctx.strokeRect(box.x1, box.y1, w, h); const text = `${label.toUpperCase()} ${(box.conf * 100).toFixed(1)}%`; const textWidth = ctx.measureText(text).width; const y = Math.max(box.y1, 18); ctx.fillStyle = color; ctx.fillRect(box.x1, y - 18, textWidth + 10, 18); ctx.fillStyle = "#ffffff"; ctx.fillText(text, box.x1 + 5, y - 4); } }