Spaces:
Runtime error
Runtime error
Claude commited on
feat: crop inspection to selected object bbox with 20% padding
Browse filesInstead of sending the entire frame to inspection modules, all API
calls now pass the selected track_id so the backend crops to the
object's bounding box with 20% surrounding context:
- fetchFrame: passes track_id + padding=0.20 query params
- fetchDepth + _fetchDepthJson: passes track_id for bbox-scoped depth
- fetchSuperRes: padding updated from 0.15 to 0.20 for consistency
- _loadAndRender: passes trackId to frame and depth fetchers
The backend already supported track_id cropping on all these endpoints
— this change wires the frontend to use it. Clicking a bbox or track
card now inspects only the detected object region, not the full frame.
https://claude.ai/code/session_01XQ1edVcrdcMErbKF53r1aF
frontend/js/api/inspection-api.js
CHANGED
|
@@ -2,14 +2,20 @@
|
|
| 2 |
APP.api.inspection = {};
|
| 3 |
|
| 4 |
/**
|
| 5 |
-
* Fetch a raw video frame as an Image element.
|
| 6 |
* @param {string} jobId
|
| 7 |
* @param {number} frameIdx
|
|
|
|
|
|
|
| 8 |
* @returns {Promise<HTMLImageElement>}
|
| 9 |
*/
|
| 10 |
-
APP.api.inspection.fetchFrame = async function (jobId, frameIdx) {
|
| 11 |
const base = APP.core.state.hf.baseUrl;
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
const resp = await fetch(url);
|
| 14 |
if (!resp.ok) throw new Error(`Frame fetch failed: ${resp.status}`);
|
| 15 |
|
|
@@ -66,15 +72,19 @@ APP.api.inspection.generateMask = async function (jobId, frameIdx, trackId) {
|
|
| 66 |
};
|
| 67 |
|
| 68 |
/**
|
| 69 |
-
* Fetch depth map for a specific frame.
|
| 70 |
* @param {string} jobId
|
| 71 |
* @param {number} frameIdx
|
|
|
|
| 72 |
* @returns {Promise<Object>} { width, height, min, max, data: Float32Array }
|
| 73 |
*/
|
| 74 |
-
APP.api.inspection.fetchDepth = async function (jobId, frameIdx) {
|
| 75 |
const base = APP.core.state.hf.baseUrl;
|
| 76 |
// Try binary format first; fall back to JSON if CORS strips custom headers
|
| 77 |
-
|
|
|
|
|
|
|
|
|
|
| 78 |
const resp = await fetch(url);
|
| 79 |
if (!resp.ok) throw new Error(`Depth fetch failed: ${resp.status}`);
|
| 80 |
|
|
@@ -92,7 +102,7 @@ APP.api.inspection.fetchDepth = async function (jobId, frameIdx) {
|
|
| 92 |
// If CORS stripped headers, infer dimensions from data length
|
| 93 |
if (isNaN(w) || isNaN(h)) {
|
| 94 |
// Fall back to JSON format
|
| 95 |
-
return await APP.api.inspection._fetchDepthJson(jobId, frameIdx);
|
| 96 |
}
|
| 97 |
|
| 98 |
return {
|
|
@@ -112,9 +122,12 @@ APP.api.inspection.fetchDepth = async function (jobId, frameIdx) {
|
|
| 112 |
/**
|
| 113 |
* Fallback: fetch depth in JSON format if raw binary headers are unavailable.
|
| 114 |
*/
|
| 115 |
-
APP.api.inspection._fetchDepthJson = async function (jobId, frameIdx) {
|
| 116 |
const base = APP.core.state.hf.baseUrl;
|
| 117 |
-
|
|
|
|
|
|
|
|
|
|
| 118 |
const resp = await fetch(url);
|
| 119 |
if (!resp.ok) throw new Error(`Depth (JSON) fetch failed: ${resp.status}`);
|
| 120 |
const json = await resp.json();
|
|
@@ -184,7 +197,7 @@ APP.api.inspection.fetchSuperRes = async function (jobId, frameIdx, trackId, sca
|
|
| 184 |
const resp = await fetch(`${base}/inspect/super-resolve/${jobId}/${frameIdx}`, {
|
| 185 |
method: "POST",
|
| 186 |
headers: { "Content-Type": "application/json" },
|
| 187 |
-
body: JSON.stringify({ track_id: trackId, scale: scale, padding: 0.
|
| 188 |
});
|
| 189 |
|
| 190 |
if (resp.status === 202) {
|
|
|
|
| 2 |
APP.api.inspection = {};
|
| 3 |
|
| 4 |
/**
|
| 5 |
+
* Fetch a raw video frame as an Image element, optionally cropped to a track.
|
| 6 |
* @param {string} jobId
|
| 7 |
* @param {number} frameIdx
|
| 8 |
+
* @param {string} [trackId] — if provided, crops to the track's bbox + padding
|
| 9 |
+
* @param {number} [padding=0.20] — padding ratio around the bbox
|
| 10 |
* @returns {Promise<HTMLImageElement>}
|
| 11 |
*/
|
| 12 |
+
APP.api.inspection.fetchFrame = async function (jobId, frameIdx, trackId, padding) {
|
| 13 |
const base = APP.core.state.hf.baseUrl;
|
| 14 |
+
let url = `${base}/inspect/frame/${jobId}/${frameIdx}`;
|
| 15 |
+
if (trackId) {
|
| 16 |
+
const p = padding != null ? padding : 0.20;
|
| 17 |
+
url += `?track_id=${encodeURIComponent(trackId)}&padding=${p}`;
|
| 18 |
+
}
|
| 19 |
const resp = await fetch(url);
|
| 20 |
if (!resp.ok) throw new Error(`Frame fetch failed: ${resp.status}`);
|
| 21 |
|
|
|
|
| 72 |
};
|
| 73 |
|
| 74 |
/**
|
| 75 |
+
* Fetch depth map for a specific frame, optionally cropped to a track.
|
| 76 |
* @param {string} jobId
|
| 77 |
* @param {number} frameIdx
|
| 78 |
+
* @param {string} [trackId] — if provided, crops depth to the track's bbox
|
| 79 |
* @returns {Promise<Object>} { width, height, min, max, data: Float32Array }
|
| 80 |
*/
|
| 81 |
+
APP.api.inspection.fetchDepth = async function (jobId, frameIdx, trackId) {
|
| 82 |
const base = APP.core.state.hf.baseUrl;
|
| 83 |
// Try binary format first; fall back to JSON if CORS strips custom headers
|
| 84 |
+
let url = `${base}/inspect/depth/${jobId}/${frameIdx}?format=raw`;
|
| 85 |
+
if (trackId) {
|
| 86 |
+
url += `&track_id=${encodeURIComponent(trackId)}`;
|
| 87 |
+
}
|
| 88 |
const resp = await fetch(url);
|
| 89 |
if (!resp.ok) throw new Error(`Depth fetch failed: ${resp.status}`);
|
| 90 |
|
|
|
|
| 102 |
// If CORS stripped headers, infer dimensions from data length
|
| 103 |
if (isNaN(w) || isNaN(h)) {
|
| 104 |
// Fall back to JSON format
|
| 105 |
+
return await APP.api.inspection._fetchDepthJson(jobId, frameIdx, trackId);
|
| 106 |
}
|
| 107 |
|
| 108 |
return {
|
|
|
|
| 122 |
/**
|
| 123 |
* Fallback: fetch depth in JSON format if raw binary headers are unavailable.
|
| 124 |
*/
|
| 125 |
+
APP.api.inspection._fetchDepthJson = async function (jobId, frameIdx, trackId) {
|
| 126 |
const base = APP.core.state.hf.baseUrl;
|
| 127 |
+
let url = `${base}/inspect/depth/${jobId}/${frameIdx}?format=json`;
|
| 128 |
+
if (trackId) {
|
| 129 |
+
url += `&track_id=${encodeURIComponent(trackId)}`;
|
| 130 |
+
}
|
| 131 |
const resp = await fetch(url);
|
| 132 |
if (!resp.ok) throw new Error(`Depth (JSON) fetch failed: ${resp.status}`);
|
| 133 |
const json = await resp.json();
|
|
|
|
| 197 |
const resp = await fetch(`${base}/inspect/super-resolve/${jobId}/${frameIdx}`, {
|
| 198 |
method: "POST",
|
| 199 |
headers: { "Content-Type": "application/json" },
|
| 200 |
+
body: JSON.stringify({ track_id: trackId, scale: scale, padding: 0.20 })
|
| 201 |
});
|
| 202 |
|
| 203 |
if (resp.status === 202) {
|
frontend/js/ui/inspection.js
CHANGED
|
@@ -272,9 +272,10 @@ APP.ui.inspection._loadAndRender = async function () {
|
|
| 272 |
|
| 273 |
try {
|
| 274 |
// --- Step 1: Ensure we have the base frame image (shared by most modes) ---
|
|
|
|
| 275 |
if (!state.inspection._frameImg && mode !== "3d") {
|
| 276 |
APP.ui.inspection._setLoading(true, mode);
|
| 277 |
-
const frameImg = await api.fetchFrame(jobId, frameIdx);
|
| 278 |
state.inspection.frameImageUrl = frameImg.src;
|
| 279 |
state.inspection._frameImg = frameImg;
|
| 280 |
}
|
|
@@ -296,7 +297,7 @@ APP.ui.inspection._loadAndRender = async function () {
|
|
| 296 |
break;
|
| 297 |
|
| 298 |
case "depth":
|
| 299 |
-
cache.depth = await api.fetchDepth(jobId, frameIdx);
|
| 300 |
break;
|
| 301 |
|
| 302 |
case "attention":
|
|
|
|
| 272 |
|
| 273 |
try {
|
| 274 |
// --- Step 1: Ensure we have the base frame image (shared by most modes) ---
|
| 275 |
+
// Fetch cropped to the selected track's bbox with 20% padding
|
| 276 |
if (!state.inspection._frameImg && mode !== "3d") {
|
| 277 |
APP.ui.inspection._setLoading(true, mode);
|
| 278 |
+
const frameImg = await api.fetchFrame(jobId, frameIdx, trackId, 0.20);
|
| 279 |
state.inspection.frameImageUrl = frameImg.src;
|
| 280 |
state.inspection._frameImg = frameImg;
|
| 281 |
}
|
|
|
|
| 297 |
break;
|
| 298 |
|
| 299 |
case "depth":
|
| 300 |
+
cache.depth = await api.fetchDepth(jobId, frameIdx, trackId);
|
| 301 |
break;
|
| 302 |
|
| 303 |
case "attention":
|