Claude commited on
Commit
e3398cb
·
unverified ·
1 Parent(s): 29602e0

feat: crop inspection to selected object bbox with 20% padding

Browse files

Instead of sending the entire frame to inspection modules, all API
calls now pass the selected track_id so the backend crops to the
object's bounding box with 20% surrounding context:

- fetchFrame: passes track_id + padding=0.20 query params
- fetchDepth + _fetchDepthJson: passes track_id for bbox-scoped depth
- fetchSuperRes: padding updated from 0.15 to 0.20 for consistency
- _loadAndRender: passes trackId to frame and depth fetchers

The backend already supported track_id cropping on all these endpoints
— this change wires the frontend to use it. Clicking a bbox or track
card now inspects only the detected object region, not the full frame.

https://claude.ai/code/session_01XQ1edVcrdcMErbKF53r1aF

frontend/js/api/inspection-api.js CHANGED
@@ -2,14 +2,20 @@
2
  APP.api.inspection = {};
3
 
4
  /**
5
- * Fetch a raw video frame as an Image element.
6
  * @param {string} jobId
7
  * @param {number} frameIdx
 
 
8
  * @returns {Promise<HTMLImageElement>}
9
  */
10
- APP.api.inspection.fetchFrame = async function (jobId, frameIdx) {
11
  const base = APP.core.state.hf.baseUrl;
12
- const url = `${base}/inspect/frame/${jobId}/${frameIdx}`;
 
 
 
 
13
  const resp = await fetch(url);
14
  if (!resp.ok) throw new Error(`Frame fetch failed: ${resp.status}`);
15
 
@@ -66,15 +72,19 @@ APP.api.inspection.generateMask = async function (jobId, frameIdx, trackId) {
66
  };
67
 
68
  /**
69
- * Fetch depth map for a specific frame.
70
  * @param {string} jobId
71
  * @param {number} frameIdx
 
72
  * @returns {Promise<Object>} { width, height, min, max, data: Float32Array }
73
  */
74
- APP.api.inspection.fetchDepth = async function (jobId, frameIdx) {
75
  const base = APP.core.state.hf.baseUrl;
76
  // Try binary format first; fall back to JSON if CORS strips custom headers
77
- const url = `${base}/inspect/depth/${jobId}/${frameIdx}?format=raw`;
 
 
 
78
  const resp = await fetch(url);
79
  if (!resp.ok) throw new Error(`Depth fetch failed: ${resp.status}`);
80
 
@@ -92,7 +102,7 @@ APP.api.inspection.fetchDepth = async function (jobId, frameIdx) {
92
  // If CORS stripped headers, infer dimensions from data length
93
  if (isNaN(w) || isNaN(h)) {
94
  // Fall back to JSON format
95
- return await APP.api.inspection._fetchDepthJson(jobId, frameIdx);
96
  }
97
 
98
  return {
@@ -112,9 +122,12 @@ APP.api.inspection.fetchDepth = async function (jobId, frameIdx) {
112
  /**
113
  * Fallback: fetch depth in JSON format if raw binary headers are unavailable.
114
  */
115
- APP.api.inspection._fetchDepthJson = async function (jobId, frameIdx) {
116
  const base = APP.core.state.hf.baseUrl;
117
- const url = `${base}/inspect/depth/${jobId}/${frameIdx}?format=json`;
 
 
 
118
  const resp = await fetch(url);
119
  if (!resp.ok) throw new Error(`Depth (JSON) fetch failed: ${resp.status}`);
120
  const json = await resp.json();
@@ -184,7 +197,7 @@ APP.api.inspection.fetchSuperRes = async function (jobId, frameIdx, trackId, sca
184
  const resp = await fetch(`${base}/inspect/super-resolve/${jobId}/${frameIdx}`, {
185
  method: "POST",
186
  headers: { "Content-Type": "application/json" },
187
- body: JSON.stringify({ track_id: trackId, scale: scale, padding: 0.15 })
188
  });
189
 
190
  if (resp.status === 202) {
 
2
  APP.api.inspection = {};
3
 
4
  /**
5
+ * Fetch a raw video frame as an Image element, optionally cropped to a track.
6
  * @param {string} jobId
7
  * @param {number} frameIdx
8
+ * @param {string} [trackId] — if provided, crops to the track's bbox + padding
9
+ * @param {number} [padding=0.20] — padding ratio around the bbox
10
  * @returns {Promise<HTMLImageElement>}
11
  */
12
+ APP.api.inspection.fetchFrame = async function (jobId, frameIdx, trackId, padding) {
13
  const base = APP.core.state.hf.baseUrl;
14
+ let url = `${base}/inspect/frame/${jobId}/${frameIdx}`;
15
+ if (trackId) {
16
+ const p = padding != null ? padding : 0.20;
17
+ url += `?track_id=${encodeURIComponent(trackId)}&padding=${p}`;
18
+ }
19
  const resp = await fetch(url);
20
  if (!resp.ok) throw new Error(`Frame fetch failed: ${resp.status}`);
21
 
 
72
  };
73
 
74
  /**
75
+ * Fetch depth map for a specific frame, optionally cropped to a track.
76
  * @param {string} jobId
77
  * @param {number} frameIdx
78
+ * @param {string} [trackId] — if provided, crops depth to the track's bbox
79
  * @returns {Promise<Object>} { width, height, min, max, data: Float32Array }
80
  */
81
+ APP.api.inspection.fetchDepth = async function (jobId, frameIdx, trackId) {
82
  const base = APP.core.state.hf.baseUrl;
83
  // Try binary format first; fall back to JSON if CORS strips custom headers
84
+ let url = `${base}/inspect/depth/${jobId}/${frameIdx}?format=raw`;
85
+ if (trackId) {
86
+ url += `&track_id=${encodeURIComponent(trackId)}`;
87
+ }
88
  const resp = await fetch(url);
89
  if (!resp.ok) throw new Error(`Depth fetch failed: ${resp.status}`);
90
 
 
102
  // If CORS stripped headers, infer dimensions from data length
103
  if (isNaN(w) || isNaN(h)) {
104
  // Fall back to JSON format
105
+ return await APP.api.inspection._fetchDepthJson(jobId, frameIdx, trackId);
106
  }
107
 
108
  return {
 
122
  /**
123
  * Fallback: fetch depth in JSON format if raw binary headers are unavailable.
124
  */
125
+ APP.api.inspection._fetchDepthJson = async function (jobId, frameIdx, trackId) {
126
  const base = APP.core.state.hf.baseUrl;
127
+ let url = `${base}/inspect/depth/${jobId}/${frameIdx}?format=json`;
128
+ if (trackId) {
129
+ url += `&track_id=${encodeURIComponent(trackId)}`;
130
+ }
131
  const resp = await fetch(url);
132
  if (!resp.ok) throw new Error(`Depth (JSON) fetch failed: ${resp.status}`);
133
  const json = await resp.json();
 
197
  const resp = await fetch(`${base}/inspect/super-resolve/${jobId}/${frameIdx}`, {
198
  method: "POST",
199
  headers: { "Content-Type": "application/json" },
200
+ body: JSON.stringify({ track_id: trackId, scale: scale, padding: 0.20 })
201
  });
202
 
203
  if (resp.status === 202) {
frontend/js/ui/inspection.js CHANGED
@@ -272,9 +272,10 @@ APP.ui.inspection._loadAndRender = async function () {
272
 
273
  try {
274
  // --- Step 1: Ensure we have the base frame image (shared by most modes) ---
 
275
  if (!state.inspection._frameImg && mode !== "3d") {
276
  APP.ui.inspection._setLoading(true, mode);
277
- const frameImg = await api.fetchFrame(jobId, frameIdx);
278
  state.inspection.frameImageUrl = frameImg.src;
279
  state.inspection._frameImg = frameImg;
280
  }
@@ -296,7 +297,7 @@ APP.ui.inspection._loadAndRender = async function () {
296
  break;
297
 
298
  case "depth":
299
- cache.depth = await api.fetchDepth(jobId, frameIdx);
300
  break;
301
 
302
  case "attention":
 
272
 
273
  try {
274
  // --- Step 1: Ensure we have the base frame image (shared by most modes) ---
275
+ // Fetch cropped to the selected track's bbox with 20% padding
276
  if (!state.inspection._frameImg && mode !== "3d") {
277
  APP.ui.inspection._setLoading(true, mode);
278
+ const frameImg = await api.fetchFrame(jobId, frameIdx, trackId, 0.20);
279
  state.inspection.frameImageUrl = frameImg.src;
280
  state.inspection._frameImg = frameImg;
281
  }
 
297
  break;
298
 
299
  case "depth":
300
+ cache.depth = await api.fetchDepth(jobId, frameIdx, trackId);
301
  break;
302
 
303
  case "attention":