Upload 51 files

src/components/MultiSourceCaptioningView.tsx

@@ -6,7 +6,7 @@ const MODES = ["Webcam", "URL", "File"] as const;
 type Mode = typeof MODES[number];
 
 const EXAMPLE_VIDEO_URL = "/space/videos/1.mp4";
-const EXAMPLE_PROMPT = "Detect all birds in the image. For each bird, output a JSON array of objects with fields: 'label' (string) and 'bbox_2d' ([x1, y1, x2, y2]) where coordinates are in pixel values. This is only an example use your own values (Do not use these values [100, 50, 200, 300]) Example: [{\"label\": \"bird\", \"bbox_2d\": [100, 50, 200, 300]}]";
+const EXAMPLE_PROMPT = "Detect all birds in the image. For each bird, output a JSON array of objects with fields: 'label' (string) and 'bbox_2d' ([x1, y1, x2, y2]) where coordinates are in pixel values. Example: [{\"label\": \"bird\", \"bbox_2d\": [x1, y1, x2, y2]}]";
 
 function parseFlatBoxArray(arr: any[]): { label: string, bbox_2d: number[] }[] {
   if (typeof arr[0] === "string" && Array.isArray(arr[1])) {
@@ -62,6 +62,19 @@ function isVideoFile(file: File) {
   return file.type.startsWith("video/");
 }
 
+function denormalizeBox(box: number[], width: number, height: number) {
+  // If all values are between 0 and 1, treat as normalized
+  if (box.length === 4 && box.every(v => v >= 0 && v <= 1)) {
+    return [
+      box[0] * width,
+      box[1] * height,
+      box[2] * width,
+      box[3] * height
+    ];
+  }
+  return box;
+}
+
 export default function MultiSourceCaptioningView() {
   const [mode, setMode] = useState<Mode>("File");
   const [videoUrl, setVideoUrl] = useState<string>(EXAMPLE_VIDEO_URL);
@@ -337,7 +350,12 @@ export default function MultiSourceCaptioningView() {
       if (boxHistory.length > 0) {
         const scaleX = canvas.width / overlayVideo.videoWidth;
         const scaleY = canvas.height / overlayVideo.videoHeight;
-        drawBoundingBoxesOnCanvas(ctx, boxHistory, { color: "#FF00FF", lineWidth: 4, font: "20px Arial", scaleX, scaleY });
+        // Denormalize boxes if needed
+        const denormalizedBoxes = boxHistory.map(b => ({
+          ...b,
+          bbox_2d: denormalizeBox(b.bbox_2d, canvas.width, canvas.height)
+        }));
+        drawBoundingBoxesOnCanvas(ctx, denormalizedBoxes, { color: "#FF00FF", lineWidth: 4, font: "20px Arial", scaleX, scaleY });
       }
     };
     draw();