import React, { useEffect, useState, useRef, useImperativeHandle, forwardRef } from 'react'; import { Wrench, Trash2, Circle as CircleIcon, Hexagon, Square as SquareIcon, ChevronDown, ChevronUp, Sparkles, Loader } from 'lucide-react'; type ShapeType = 'rect' | 'circle' | 'polygon'; interface Point { x: number; y: number; } interface Annotation { id: string; type: ShapeType; x: number; y: number; width: number; height: number; color: string; label?: string; points?: Point[]; source?: 'manual' | 'ai'; identified?: boolean; accepted?: boolean; } interface AceticAnnotatorProps { imageUrl?: string; imageUrls?: string[]; onAnnotationsChange?: (annotations: Annotation[]) => void; } export interface AceticAnnotatorHandle { addAIAnnotations: (aiAnnotations: Annotation[]) => void; setImageIndex: (index: number) => void; getCurrentImageIndex: () => number; clearAIAnnotations: () => void; clearAllAnnotations: () => void; resetViewport: () => void; waitForImageReady: () => Promise; runAIAssist: () => Promise; } const AceticAnnotatorComponent = forwardRef(({ imageUrl, imageUrls, onAnnotationsChange }, ref) => { const canvasRef = useRef(null); const containerRef = useRef(null); const imageReadyResolveRef = useRef<(() => void) | null>(null); const emptyAnnotationsRef = useRef([]); const cachedImagesRef = useRef>({}); const pendingDrawAnimationRef = useRef(null); const lastDrawStateRef = useRef(''); // Store annotations per image index const [annotationsByImage, setAnnotationsByImage] = useState>({}); const [tool, setTool] = useState('rect'); const [color, setColor] = useState('#05998c'); const [labelInput, setLabelInput] = useState(''); const [isDrawing, setIsDrawing] = useState(false); const [startPoint, setStartPoint] = useState(null); const [currentAnnotation, setCurrentAnnotation] = useState(null); const [imageLoaded, setImageLoaded] = useState(false); const [imageDimensions, setImageDimensions] = useState({ width: 0, height: 0 }); const [polygonPoints, setPolygonPoints] = useState([]); const [selectedImageIndex, setSelectedImageIndex] = useState(0); const [isAnnotationsOpen, setIsAnnotationsOpen] = useState(true); const [isLabelDropdownOpen, setIsLabelDropdownOpen] = useState(false); // Edit state for annotation list const [editingId, setEditingId] = useState(null); const [editLabel, setEditLabel] = useState(''); const [editColor, setEditColor] = useState('#05998c'); // Annotation metadata state const [annotationIdentified, setAnnotationIdentified] = useState>({}); const [annotationAccepted, setAnnotationAccepted] = useState>({}); // AI state const [isAILoading, setIsAILoading] = useState(false); const [aiError, setAIError] = useState(null); const [isAIAssistEnabled, setIsAIAssistEnabled] = useState(false); // Get annotations for current image const annotations = annotationsByImage[selectedImageIndex] ?? emptyAnnotationsRef.current; // Helper to update annotations for current image const setAnnotations = (updater: Annotation[] | ((prev: Annotation[]) => Annotation[])) => { setAnnotationsByImage(prev => { const currentAnnotations = prev[selectedImageIndex] || []; const newAnnotations = typeof updater === 'function' ? updater(currentAnnotations) : updater; return { ...prev, [selectedImageIndex]: newAnnotations }; }); }; // Predefined label options for Acetic Acid step (ONLY CHANGE from ImageAnnotator) const labelOptions = [ 'Acetowhite', 'Mosaic', 'Punctation', 'Blood Vessel', 'Border' ]; // Filter labels based on input const filteredLabels = labelOptions.filter(label => label.toLowerCase().includes(labelInput.toLowerCase()) ); // Helper function to get bounding box from polygon points const getBoundsFromPoints = (points: any[]) => { if (!points || !Array.isArray(points) || points.length === 0) { console.warn('⚠️ Invalid points array in getBoundsFromPoints:', points); return { x: 0, y: 0, width: 0, height: 0 }; } // Handle both array [x, y] and object {x, y} formats const getX = (p: any) => Array.isArray(p) ? p[0] : (p?.x ?? 0); const getY = (p: any) => Array.isArray(p) ? p[1] : (p?.y ?? 0); const xs = points.map(getX).filter(x => typeof x === 'number' && !isNaN(x)); const ys = points.map(getY).filter(y => typeof y === 'number' && !isNaN(y)); if (xs.length === 0 || ys.length === 0) { console.warn('⚠️ No valid coordinates in points:', points); return { x: 0, y: 0, width: 0, height: 0 }; } const minX = Math.min(...xs); const minY = Math.min(...ys); const maxX = Math.max(...xs); const maxY = Math.max(...ys); return { x: minX, y: minY, width: maxX - minX, height: maxY - minY }; }; // AI Assist - Run acetowhite contour detection on current image const runAIAssist = async () => { const images = imageUrls || (imageUrl ? [imageUrl] : []); if (images.length === 0) { setAIError('No images available to analyze'); return; } const currentImageUrl = images[selectedImageIndex]; setIsAILoading(true); setAIError(null); try { // Ensure image is loaded first if (!imageLoaded || imageDimensions.width === 0 || imageDimensions.height === 0) { console.log('⏳ Waiting for image to load before AI processing...'); await new Promise((resolve) => { imageReadyResolveRef.current = resolve; }); } const response = await fetch(currentImageUrl); const blob = await response.blob(); const formData = new FormData(); formData.append('file', blob, 'image.jpg'); formData.append('conf_threshold', '0.4'); console.log('🚀 Sending to backend with image dimensions:', imageDimensions); const backendResponse = await fetch('/api/infer-aw-contour', { method: 'POST', body: formData, }); if (!backendResponse.ok) { throw new Error(`Backend error: ${backendResponse.statusText}`); } const result = await backendResponse.json(); console.log('📥 Backend response received'); if (result.status !== 'success') { throw new Error(result.error || 'AI inference failed'); } if (!result.contours || !Array.isArray(result.contours)) { console.warn('⚠️ Invalid contours format:', result.contours); setAIError('Invalid response format from server'); setIsAILoading(false); return; } const contours = result.contours; if (contours.length === 0) { console.log('ℹ️ No contours detected'); setAIError('No contours detected for this image'); setIsAILoading(false); return; } console.log(`✅ Processing ${contours.length} contour(s)`); const aiAnnotations = contours .filter((contour: any) => contour !== null && contour !== undefined) .map((contour: any, idx: number) => { let points: { x: number; y: number }[]; let area: number | undefined; let confidence: number | undefined; if (!contour) { console.warn(`⚠️ Contour ${idx} is null or undefined`); return null; } // Handle two formats: // 1. OLD: contour is an array of [x, y] pairs: [[x,y], [x,y], ...] // 2. NEW: contour is an object: {points: [[x,y], ...], area: ..., confidence: ...} if (Array.isArray(contour)) { // OLD FORMAT: contour is directly an array of points try { // Use points directly as-is from backend points = contour as any[]; // Handle OpenCV format: [[[x, y]], [[x, y]], ...] (nested arrays) if (Array.isArray(contour[0]) && Array.isArray(contour[0][0])) { points = contour.map((p: any) => { if (Array.isArray(p) && Array.isArray(p[0])) { return p[0]; // Extract [x, y] from [[x, y]] } return p; }); } } catch (e) { console.warn(`⚠️ Error processing contour ${idx}:`, e); return null; } area = undefined; confidence = undefined; } else if (contour && typeof contour === 'object' && contour.points && Array.isArray(contour.points)) { // NEW FORMAT: contour is an object with points property try { // Use points directly as-is from backend points = contour.points as any[]; // Handle OpenCV format: [[[x, y]], [[x, y]], ...] (nested arrays) if (Array.isArray(contour.points[0]) && Array.isArray(contour.points[0][0])) { points = contour.points.map((p: any) => { if (Array.isArray(p) && Array.isArray(p[0])) { return p[0]; // Extract [x, y] from [[x, y]] } return p; }); } } catch (e) { console.warn(`⚠️ Error processing contour points ${idx}:`, e); return null; } area = contour.area; confidence = contour.confidence; } else { console.warn(`⚠️ Invalid contour format at index ${idx}:`, contour); return null; } if (!points || points.length < 3) { console.warn(`⚠️ Contour ${idx} has insufficient points (${points?.length || 0})`); return null; } // Normalize point scale if backend returns normalized (0-1) or percent (0-100) coordinates let scaledPoints = points; if (imageDimensions.width > 0 && imageDimensions.height > 0) { // Helper to extract x from various formats const getX = (p: any) => { if (Array.isArray(p) && typeof p[0] === 'number') return p[0]; if (Array.isArray(p) && Array.isArray(p[0])) return p[0][0]; if (typeof p === 'object' && p !== null) return p.x; return undefined; }; // Helper to extract y from various formats const getY = (p: any) => { if (Array.isArray(p) && typeof p[0] === 'number') return p[1]; if (Array.isArray(p) && Array.isArray(p[0])) return p[0][1]; if (typeof p === 'object' && p !== null) return p.y; return undefined; }; const xValues = points.map(getX).filter(x => typeof x === 'number' && !isNaN(x)); const yValues = points.map(getY).filter(y => typeof y === 'number' && !isNaN(y)); if (xValues.length === 0 || yValues.length === 0) { // Fallback: assume [x, y] or [[x, y]] array format scaledPoints = points.map((p) => { let x, y; if (Array.isArray(p) && Array.isArray(p[0])) { x = Number(p[0][0]); y = Number(p[0][1]); } else if (Array.isArray(p) && p.length >= 2) { x = Number(p[0]); y = Number(p[1]); } else { x = NaN; y = NaN; } return { x, y }; }); } else { const maxX = Math.max(...xValues); const maxY = Math.max(...yValues); if (maxX <= 1 && maxY <= 1) { scaledPoints = points.map(p => { const x = getX(p); const y = getY(p); return { x: x * imageDimensions.width, y: y * imageDimensions.height }; }); } else if (maxX <= 100 && maxY <= 100) { scaledPoints = points.map(p => { const x = getX(p); const y = getY(p); return { x: (x / 100) * imageDimensions.width, y: (y / 100) * imageDimensions.height }; }); } else { scaledPoints = points.map(p => { const x = getX(p); const y = getY(p); return { x: Number(x), y: Number(y) }; }); } } } else { // No image dimensions yet, just convert to {x, y} format scaledPoints = points.map(p => { let x, y; if (Array.isArray(p) && Array.isArray(p[0])) { x = Number(p[0][0]); y = Number(p[0][1]); } else if (Array.isArray(p) && p.length >= 2) { x = Number(p[0]); y = Number(p[1]); } else if (typeof p === 'object' && p !== null) { x = Number(p.x); y = Number(p.y); } else { x = NaN; y = NaN; } return { x, y }; }); } const bounds = getBoundsFromPoints(scaledPoints); const annotation = { id: `ai_${Date.now()}_${idx}`, type: 'polygon' as const, x: bounds.x, y: bounds.y, width: bounds.width, height: bounds.height, color: '#05998c', label: 'Acetowhite', points: scaledPoints, source: 'ai' as const, identified: false, accepted: false, confidence: confidence, area: area }; return annotation; }) .filter(Boolean) as Annotation[]; if (aiAnnotations.length > 0) { // Replace all annotations with new AI annotations // The useEffect will automatically trigger drawCanvas when annotationsByImage updates console.log(`🎨 Setting ${aiAnnotations.length} AI annotations and triggering redraw`); setAnnotations(aiAnnotations); } else { setAIError('No valid contours detected'); } setIsAILoading(false); } catch (error) { console.error('AI assist error:', error); setAIError(error instanceof Error ? error.message : 'Failed to run AI analysis'); setIsAILoading(false); } }; // Expose annotator methods via ref useImperativeHandle(ref, () => ({ addAIAnnotations: (aiAnnotations: Annotation[]) => { setAnnotations(aiAnnotations); }, setImageIndex: (index: number) => { setSelectedImageIndex(index); }, getCurrentImageIndex: () => { return selectedImageIndex; }, clearAIAnnotations: () => { setAnnotations(prev => prev.filter(ann => ann.source !== 'ai')); }, clearAllAnnotations: () => { setAnnotations([]); setPolygonPoints([]); setCurrentAnnotation(null); }, resetViewport: () => { setSelectedImageIndex(0); setAnnotationsByImage({}); // Clear all images' annotations setPolygonPoints([]); setCurrentAnnotation(null); }, runAIAssist: async () => { await runAIAssist(); }, waitForImageReady: () => { return new Promise((resolve) => { if (imageLoaded) { console.log('✅ Image already ready'); resolve(); } else { console.log('⏳ Waiting for image to load'); imageReadyResolveRef.current = resolve; } }); } })); const images = imageUrls || (imageUrl ? [imageUrl] : []); const currentImageUrl = images[selectedImageIndex]; // Reset drawing state when switching images (but keep annotations per-image) useEffect(() => { // Clear temporary drawing state only setPolygonPoints([]); setCurrentAnnotation(null); setAnnotationIdentified({}); setAnnotationAccepted({}); setImageLoaded(false); // Immediately clear the canvas (will be redrawn with correct image's annotations) const canvas = canvasRef.current; if (canvas) { const ctx = canvas.getContext('2d'); if (ctx) { ctx.clearRect(0, 0, canvas.width, canvas.height); } } }, [selectedImageIndex]); useEffect(() => { const img = new Image(); img.src = currentImageUrl; img.onload = () => { setImageDimensions({ width: img.width, height: img.height }); setImageLoaded(true); // Resolve waitForImageReady promise if pending if (imageReadyResolveRef.current) { imageReadyResolveRef.current(); imageReadyResolveRef.current = null; } // drawCanvas will be called by useEffect when imageLoaded changes }; }, [currentImageUrl]); useEffect(() => { if (imageLoaded) drawCanvas(); }, [annotationsByImage, selectedImageIndex, currentAnnotation, polygonPoints, imageLoaded, imageDimensions]); useEffect(() => { if (onAnnotationsChange) onAnnotationsChange(annotations); }, [annotations, onAnnotationsChange]); // Cleanup pending draw on unmount useEffect(() => { return () => { if (pendingDrawAnimationRef.current) { cancelAnimationFrame(pendingDrawAnimationRef.current); } }; }, []); const clearAnnotations = () => setAnnotations([]); const deleteLastAnnotation = () => setAnnotations(prev => prev.slice(0, -1)); const getCanvasCoordinates = (e: React.MouseEvent) => { const canvas = canvasRef.current; if (!canvas) return { x: 0, y: 0 }; const rect = canvas.getBoundingClientRect(); const scaleX = imageDimensions.width / canvas.width || 1; const scaleY = imageDimensions.height / canvas.height || 1; return { x: (e.clientX - rect.left) * scaleX, y: (e.clientY - rect.top) * scaleY }; }; const startDrawing = (point: Point) => { setStartPoint(point); setIsDrawing(true); }; const finishDrawingRectOrCircle = () => { if (currentAnnotation && (currentAnnotation.width > 5 || currentAnnotation.height > 5)) { const ann: Annotation = { ...currentAnnotation, id: Date.now().toString(), label: labelInput, source: 'manual', identified: false }; setAnnotations(prev => [...prev, ann]); } setIsDrawing(false); setStartPoint(null); setCurrentAnnotation(null); }; const handleMouseDown = (e: React.MouseEvent) => { const p = getCanvasCoordinates(e); if (tool === 'polygon') { // add point to polygon setPolygonPoints(prev => [...prev, p]); return; } startDrawing(p); }; const handleMouseMove = (e: React.MouseEvent) => { if (tool === 'polygon') return; // polygon preview handled by polygonPoints if (!isDrawing || !startPoint) return; const p = getCanvasCoordinates(e); const w = p.x - startPoint.x; const h = p.y - startPoint.y; const ann: Annotation = { id: 'temp', type: tool, x: w > 0 ? startPoint.x : p.x, y: h > 0 ? startPoint.y : p.y, width: Math.abs(w), height: Math.abs(h), color, label: labelInput }; setCurrentAnnotation(ann); }; const handleMouseUp = () => { if (tool === 'polygon') return; if (isDrawing) finishDrawingRectOrCircle(); }; const finishPolygon = () => { if (polygonPoints.length < 3) return; const bounds = getBoundsFromPoints(polygonPoints); const ann: Annotation = { id: Date.now().toString(), type: 'polygon', x: bounds.x, y: bounds.y, width: bounds.width, height: bounds.height, color, label: labelInput, points: polygonPoints, source: 'manual', identified: false }; setAnnotations(prev => [...prev, ann]); setPolygonPoints([]); setLabelInput(''); }; const cancelPolygon = () => setPolygonPoints([]); const drawCanvas = () => { const canvas = canvasRef.current; const container = containerRef.current; if (!canvas || !container) return; // Cancel previous animation frame if pending if (pendingDrawAnimationRef.current) { cancelAnimationFrame(pendingDrawAnimationRef.current); } // Use requestAnimationFrame for smooth, optimized drawing pendingDrawAnimationRef.current = requestAnimationFrame(() => { performDraw(); }); }; const performDraw = () => { const canvas = canvasRef.current; const container = containerRef.current; if (!canvas || !container) return; const ctx = canvas.getContext('2d'); if (!ctx) return; // Create state key to prevent redundant draws const currentAnnotations = annotationsByImage[selectedImageIndex] || []; const stateKey = `${selectedImageIndex}-${currentAnnotations.length}-${polygonPoints.length}-${currentAnnotation?.id}`; if (lastDrawStateRef.current === stateKey && canvas.width > 0) { return; // Skip if nothing changed } lastDrawStateRef.current = stateKey; // Get or create cached image let img = cachedImagesRef.current[currentImageUrl]; if (!img) { img = new Image(); img.src = currentImageUrl; cachedImagesRef.current[currentImageUrl] = img; } // If image not loaded yet, wait for it if (!img.complete) { img.onload = () => performDraw(); return; } // Resize canvas if needed const containerWidth = container.clientWidth; const aspectRatio = img.height / img.width || 1; const canvasHeight = containerWidth * aspectRatio; if (canvas.width !== containerWidth || canvas.height !== canvasHeight) { canvas.width = containerWidth; canvas.height = canvasHeight; } // Draw image ctx.clearRect(0, 0, canvas.width, canvas.height); ctx.drawImage(img, 0, 0, canvas.width, canvas.height); // Draw saved annotations currentAnnotations.forEach((a) => { drawAnnotation(ctx, a, canvas.width, canvas.height); }); // Draw polygon being created if (polygonPoints.length > 0) { drawPreviewPolygon(ctx, polygonPoints, canvas.width, canvas.height, color); } // Draw current temp annotation if (currentAnnotation) { drawAnnotation(ctx, currentAnnotation, canvas.width, canvas.height); } pendingDrawAnimationRef.current = null; }; const drawPreviewPolygon = (ctx: CanvasRenderingContext2D, pts: Point[], canvasWidth: number, canvasHeight: number, col: string) => { const scaleX = canvasWidth / imageDimensions.width || 1; const scaleY = canvasHeight / imageDimensions.height || 1; ctx.strokeStyle = col; ctx.lineWidth = 2; ctx.setLineDash([6, 4]); ctx.beginPath(); pts.forEach((p, i) => { const x = p.x * scaleX; const y = p.y * scaleY; if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y); }); ctx.stroke(); ctx.setLineDash([]); }; const drawAnnotation = (ctx: CanvasRenderingContext2D, annotation: Annotation, canvasWidth: number, canvasHeight: number) => { const scaleX = imageDimensions.width > 0 ? canvasWidth / imageDimensions.width : 1; const scaleY = imageDimensions.height > 0 ? canvasHeight / imageDimensions.height : 1; ctx.strokeStyle = annotation.color; ctx.lineWidth = 3; ctx.setLineDash([]); if (annotation.type === 'rect') { ctx.strokeRect(annotation.x * scaleX, annotation.y * scaleY, annotation.width * scaleX, annotation.height * scaleY); ctx.fillStyle = annotation.color + '20'; ctx.fillRect(annotation.x * scaleX, annotation.y * scaleY, annotation.width * scaleX, annotation.height * scaleY); } else if (annotation.type === 'circle') { const cx = (annotation.x + annotation.width / 2) * scaleX; const cy = (annotation.y + annotation.height / 2) * scaleY; const r = Math.max(annotation.width * scaleX, annotation.height * scaleY) / 2; ctx.beginPath(); ctx.arc(cx, cy, r, 0, Math.PI * 2); ctx.stroke(); ctx.fillStyle = annotation.color + '20'; ctx.fill(); } else if (annotation.type === 'polygon' && annotation.points && Array.isArray(annotation.points) && annotation.points.length > 0) { console.log(`🎨 Drawing polygon with ${annotation.points.length} points, scale: ${scaleX}x${scaleY}`); ctx.beginPath(); let validPointsCount = 0; annotation.points.forEach((p, i) => { if (p && typeof p.x === 'number' && typeof p.y === 'number' && Number.isFinite(p.x) && Number.isFinite(p.y)) { const x = p.x * scaleX; const y = p.y * scaleY; if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y); validPointsCount++; } else { console.warn(`⚠️ Invalid point at index ${i}:`, p); } }); if (validPointsCount < 3) { console.warn(`⚠️ Polygon has only ${validPointsCount} valid points, skipping draw`); return; } ctx.closePath(); ctx.stroke(); ctx.fillStyle = annotation.color + '20'; ctx.fill(); console.log(`✅ Polygon drawn with ${validPointsCount} points`); } }; // list editing const startEdit = (ann: Annotation) => { setEditingId(ann.id); setEditLabel(ann.label || ''); setEditColor(ann.color || '#05998c'); }; const saveEdit = () => { setAnnotations(prev => prev.map(a => a.id === editingId ? { ...a, label: editLabel, color: editColor } : a)); setEditingId(null); }; const deleteAnnotation = (id: string) => setAnnotations(prev => prev.filter(a => a.id !== id)); const handleAIAssistToggle = async () => { if (isAILoading) return; if (isAIAssistEnabled) { setIsAIAssistEnabled(false); return; } setIsAIAssistEnabled(true); await runAIAssist(); }; const getShapeTypeName = (type: ShapeType): string => { const typeMap: Record = { 'rect': 'Rectangle', 'circle': 'Circle', 'polygon': 'Polygon' }; return typeMap[type] || type; }; return (
{/* Image Selection - Show if multiple images */} {images.length > 1 && (
{images.map((imgUrl, idx) => ( ))}
)} {/* Tab Switcher with AI Assist Button */}
{/* Annotate Section */} <>
Tools
setLabelInput(e.target.value)} onFocus={() => setIsLabelDropdownOpen(true)} onBlur={() => setTimeout(() => setIsLabelDropdownOpen(false), 200)} className="px-3 py-1 border rounded text-xs md:text-sm w-48" /> {isLabelDropdownOpen && filteredLabels.length > 0 && (
{filteredLabels.map((label, idx) => ( ))}
)}
setColor(e.target.value)} className="w-10 h-8 p-0 border rounded" />
{/* AI Error Message */} {aiError && (
{aiError}
)}
{tool === 'polygon' && (
Polygon points: {polygonPoints.length}
)} {/* Annotations Table */}
{isAnnotationsOpen && (
{annotations.length === 0 ? (

No annotations yet. Draw on the image to create annotations.

) : ( {annotations.map((a) => ( {/* Annotation Column - shown in color it was drawn */} {/* Identified Checkbox Column */} {/* Source Column (Manual/AI) */} {/* Action Column - Accept/Reject for AI, Edit/Delete for Manual */} ))}
Annotation Identified Source Action
{editingId === a.id ? (
setEditColor(e.target.value)} className="w-6 h-6 rounded p-0 border cursor-pointer" /> setEditLabel(e.target.value)} className="px-2 py-1 border rounded text-xs flex-1" placeholder="Label" />
) : (
{a.label || '(no label)'}
{getShapeTypeName(a.type)}
)}
{ setAnnotationIdentified(prev => ({ ...prev, [a.id]: e.target.checked })); }} className="w-4 h-4 rounded border-gray-300 cursor-pointer" /> {a.source === 'ai' ? 'AI' : 'Manual'}
{editingId === a.id ? ( <> ) : a.source === 'ai' ? ( <> {annotationAccepted[a.id] === true ? (
✓ Accepted
) : annotationAccepted[a.id] === false ? ( ✕ Rejected ) : ( <> )} ) : ( <> )}
)}
)}
); }); AceticAnnotatorComponent.displayName = 'AceticAnnotator'; export const AceticAnnotator = AceticAnnotatorComponent; export type { AceticAnnotatorProps, Annotation };