Update agents/agent.py

agents/agent.py

@@ -23,7 +23,7 @@ class CellposeAgent:
     def attach_images_callback(step_log: ActionStep, agent: ToolCallingAgent) -> None:
         """
         Callback to attach actual PIL images for VLM inspection.
-        Images are automatically resized to reduce token consumption.
+        Images are automatically resized and compressed to reduce token consumption.
         """
         if not isinstance(step_log, ActionStep):
             return
@@ -31,16 +31,42 @@ class CellposeAgent:
         if not step_log.observations:
             return
     
-        def resize_image(img: Image.Image, max_size: int = 1024) -> Image.Image:
-            """Resize image maintaining aspect ratio, max dimension = max_size."""
-            if max(img.size) <= max_size:
-                return img
-        
-            ratio = max_size / max(img.size)
-            new_size = tuple(int(dim * ratio) for dim in img.size)
-            resized = img.resize(new_size, Image.Resampling.LANCZOS)
-            print(f"    Resized {img.size} → {resized.size}")
-            return resized
+        def resize_and_compress_image(img: Image.Image, max_size: int = 512, quality: int = 75) -> Image.Image:
+            """
+            Resize and compress image to reduce payload size.
+            
+            Args:
+                img: Input PIL Image
+                max_size: Maximum dimension (width or height)
+                quality: JPEG quality (1-95, lower = smaller file)
+            
+            Returns:
+                Compressed PIL Image
+            """
+            # Convert to RGB if needed (JPEG doesn't support RGBA)
+            if img.mode in ('RGBA', 'LA', 'P'):
+                background = Image.new('RGB', img.size, (255, 255, 255))
+                if img.mode == 'P':
+                    img = img.convert('RGBA')
+                background.paste(img, mask=img.split()[-1] if img.mode in ('RGBA', 'LA') else None)
+                img = background
+            elif img.mode != 'RGB':
+                img = img.convert('RGB')
+            
+            # Resize maintaining aspect ratio
+            if max(img.size) > max_size:
+                ratio = max_size / max(img.size)
+                new_size = tuple(int(dim * ratio) for dim in img.size)
+                img = img.resize(new_size, Image.Resampling.LANCZOS)
+            
+            # Compress using JPEG encoding
+            buffer = BytesIO()
+            img.save(buffer, format='JPEG', quality=quality, optimize=True)
+            buffer.seek(0)
+            compressed_img = Image.open(buffer)
+            
+            print(f"    Resized and compressed to {compressed_img.size}, quality={quality}")
+            return compressed_img
     
         try:
             obs_data = json.loads(step_log.observations)
@@ -52,25 +78,24 @@ class CellposeAgent:
             
                 try:
                     img = Image.open(image_path)
-                    resized_img = resize_image(img)
+                    compressed_img = resize_and_compress_image(img, max_size=512, quality=75)
                 
-                    # Attach resized PIL Image
-                    step_log.observations_images = [resized_img]
+                    # Attach compressed PIL Image
+                    step_log.observations_images = [compressed_img]
                     
                     # Keep metadata for context
                     obs_data["image_info"] = {
                         "original_dimensions": f"{img.size[0]}x{img.size[1]} pixels",
-                        "resized_dimensions": f"{resized_img.size[0]}x{resized_img.size[1]} pixels",
-                        "mode": resized_img.mode,
-                        "note": "Image attached for visual inspection (resized for efficiency)"
+                        "processed_dimensions": f"{compressed_img.size[0]}x{compressed_img.size[1]} pixels",
+                        "mode": compressed_img.mode,
+                        "note": "Image compressed for API efficiency (JPEG quality=75)"
                     }
                     step_log.observations = json.dumps(obs_data, indent=2)
-                    print(f"[Callback] ✓ Attached resized image for VLM inspection")
+                    print(f"[Callback] ✓ Attached compressed image for VLM inspection")
                 except Exception as e:
                     print(f"[Callback] Error attaching image: {e}")
         
             # Pattern 2: Multiple images from refine_segmentation
-            
             elif obs_data.get("status") == "ready_for_visual_analysis":
                 paths = obs_data.get("image_paths", {})
                 original = paths.get("original")
@@ -82,21 +107,21 @@ class CellposeAgent:
                         orig_img = Image.open(original)
                         seg_img = Image.open(segmented)
                     
-                        # Resize both images
-                        resized_orig = resize_image(orig_img)
-                        resized_seg = resize_image(seg_img)
+                        # Compress both images
+                        compressed_orig = resize_and_compress_image(orig_img, max_size=512, quality=75)
+                        compressed_seg = resize_and_compress_image(seg_img, max_size=512, quality=75)
                     
-                        # Attach both resized images as list
-                        step_log.observations_images = [resized_orig, resized_seg]
+                        # Attach both compressed images as list
+                        step_log.observations_images = [compressed_orig, compressed_seg]
                     
                         obs_data["images_info"] = {
                             "image_order": ["original", "segmented"],
                             "original_size": f"{orig_img.size[0]}x{orig_img.size[1]}",
-                            "resized_size": f"{resized_orig.size[0]}x{resized_orig.size[1]}",
-                            "note": "Both images attached for visual comparison (resized for efficiency)"
+                            "processed_size": f"{compressed_orig.size[0]}x{compressed_orig.size[1]}",
+                            "note": "Both images compressed for API efficiency (JPEG quality=75)"
                         }
                         step_log.observations = json.dumps(obs_data, indent=2)
-                        print(f"[Callback] ✓ Attached both resized images for VLM inspection")
+                        print(f"[Callback] ✓ Attached both compressed images for VLM inspection")
                     except Exception as e:
                         print(f"[Callback] Error attaching images: {e}")