Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -10,6 +10,7 @@ from PIL import Image, ImageOps, ImageDraw
|
|
| 10 |
from roboflow import Roboflow
|
| 11 |
from gradio_client import Client
|
| 12 |
import gradio as gr
|
|
|
|
| 13 |
|
| 14 |
# -------------------------------------------------------------------------
|
| 15 |
# 🧠 Fix for Gradio schema bug ("TypeError: argument of type 'bool' is not iterable")
|
|
@@ -35,15 +36,12 @@ def _safe_json_schema_to_python_type(schema, defs=None):
|
|
| 35 |
try:
|
| 36 |
if isinstance(schema, dict) and "anyOf" in schema:
|
| 37 |
types = [s.get("type") for s in schema["anyOf"] if isinstance(s, dict)]
|
| 38 |
-
# Handle the common case that causes the crash
|
| 39 |
if set(types) == {"string", "null"}:
|
| 40 |
return "Optional[str]"
|
| 41 |
-
# Default back to the original function
|
| 42 |
return gu._json_schema_to_python_type_original(schema, defs)
|
| 43 |
except Exception:
|
| 44 |
return "UnknownType"
|
| 45 |
|
| 46 |
-
# Backup and patch the function safely
|
| 47 |
if not hasattr(gu, "_json_schema_to_python_type_original"):
|
| 48 |
gu._json_schema_to_python_type_original = gu._json_schema_to_python_type
|
| 49 |
gu._json_schema_to_python_type = _safe_json_schema_to_python_type
|
|
@@ -76,11 +74,10 @@ logging.basicConfig(
|
|
| 76 |
# -------------------------------------------------------------------------
|
| 77 |
# 🤖 Roboflow configuration
|
| 78 |
# -------------------------------------------------------------------------
|
| 79 |
-
ROBOFLOW_API_KEY = "u5LX112EBlNmzYoofvPL"
|
| 80 |
PROJECT_NAME = "model_verification_project"
|
| 81 |
VERSION_NUMBER = 2
|
| 82 |
|
| 83 |
-
# Force environment variable to override cached API key
|
| 84 |
os.environ["ROBOFLOW_API_KEY"] = ROBOFLOW_API_KEY
|
| 85 |
|
| 86 |
# -------------------------------------------------------------------------
|
|
@@ -88,15 +85,13 @@ os.environ["ROBOFLOW_API_KEY"] = ROBOFLOW_API_KEY
|
|
| 88 |
# -------------------------------------------------------------------------
|
| 89 |
HANDWRITING_MODEL_ENDPOINT = "3morrrrr/Handwriting_Model_Inf"
|
| 90 |
|
| 91 |
-
# Cached
|
| 92 |
_handwriting_client = None
|
| 93 |
|
| 94 |
def get_handwriting_client(max_retries=5, retry_delay=3):
|
| 95 |
"""
|
| 96 |
-
Lazily initialize and cache the handwriting Client.
|
| 97 |
-
|
| 98 |
-
Retries a few times in case the Hugging Face Space is cold-starting,
|
| 99 |
-
to avoid crashing the whole app on startup.
|
| 100 |
"""
|
| 101 |
global _handwriting_client
|
| 102 |
if _handwriting_client is not None:
|
|
@@ -133,7 +128,10 @@ DEBUG_DIR = os.path.join(tempfile.gettempdir(), "debug_images")
|
|
| 133 |
os.makedirs(DEBUG_DIR, exist_ok=True)
|
| 134 |
|
| 135 |
logging.info(f"Debug images stored in: {DEBUG_DIR}")
|
| 136 |
-
logging.info(
|
|
|
|
|
|
|
|
|
|
| 137 |
logging.info(f"Using handwriting model endpoint: {HANDWRITING_MODEL_ENDPOINT}")
|
| 138 |
|
| 139 |
# -------------------------------------------------------------------------
|
|
@@ -171,6 +169,53 @@ def save_debug_image(image, filename, text=None):
|
|
| 171 |
logging.debug(f"Saved debug image: {path}")
|
| 172 |
return path
|
| 173 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
# -------------------------------------------------------------------------
|
| 175 |
# 🧠 Load Roboflow models
|
| 176 |
# -------------------------------------------------------------------------
|
|
@@ -178,18 +223,12 @@ rf = Roboflow(api_key=ROBOFLOW_API_KEY)
|
|
| 178 |
project = rf.workspace().project(PROJECT_NAME)
|
| 179 |
model = project.version(VERSION_NUMBER).model
|
| 180 |
|
| 181 |
-
#
|
|
|
|
|
|
|
| 182 |
def detect_paper_angle(image, bounding_box):
|
| 183 |
"""
|
| 184 |
-
Detect the angle of a paper document within the given bounding box
|
| 185 |
-
optimized for white paper detection.
|
| 186 |
-
|
| 187 |
-
Parameters:
|
| 188 |
-
- image: PIL Image or numpy array of the full image
|
| 189 |
-
- bounding_box: Tuple of (x1, y1, x2, y2) coordinates
|
| 190 |
-
|
| 191 |
-
Returns:
|
| 192 |
-
- angle: The detected angle in degrees
|
| 193 |
"""
|
| 194 |
x1, y1, x2, y2 = bounding_box
|
| 195 |
|
|
@@ -202,7 +241,6 @@ def detect_paper_angle(image, bounding_box):
|
|
| 202 |
# Crop the region of interest (ROI)
|
| 203 |
roi = image_np[y1:y2, x1:x2]
|
| 204 |
|
| 205 |
-
# Create a debug image
|
| 206 |
if DEBUG:
|
| 207 |
debug_roi = Image.fromarray(roi)
|
| 208 |
save_debug_image(debug_roi, f"paper_roi_{int(time.time())}.png",
|
|
@@ -214,61 +252,46 @@ def detect_paper_angle(image, bounding_box):
|
|
| 214 |
else:
|
| 215 |
gray = roi
|
| 216 |
|
| 217 |
-
# Save the grayscale image for debugging
|
| 218 |
if DEBUG:
|
| 219 |
cv2.imwrite(os.path.join(DEBUG_DIR, f"gray_paper_{int(time.time())}.png"), gray)
|
| 220 |
|
| 221 |
-
# Method 1:
|
| 222 |
try:
|
| 223 |
-
# Apply adaptive thresholding to handle varying lighting conditions
|
| 224 |
-
# This is particularly effective for white paper
|
| 225 |
binary = cv2.adaptiveThreshold(
|
| 226 |
gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
|
| 227 |
cv2.THRESH_BINARY_INV, 11, 2
|
| 228 |
)
|
| 229 |
|
| 230 |
-
# Save binary image for debugging
|
| 231 |
if DEBUG:
|
| 232 |
cv2.imwrite(os.path.join(DEBUG_DIR, f"binary_paper_{int(time.time())}.png"), binary)
|
| 233 |
|
| 234 |
-
# Find contours in the binary image
|
| 235 |
contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
| 236 |
|
| 237 |
-
if contours
|
| 238 |
-
# Sort contours by area (largest first)
|
| 239 |
contours = sorted(contours, key=cv2.contourArea, reverse=True)
|
| 240 |
-
|
| 241 |
-
# Find the largest contour that has a reasonable area
|
| 242 |
-
# (to avoid small noise contours)
|
| 243 |
-
min_area_ratio = 0.05 # Minimum 5% of ROI area
|
| 244 |
roi_area = gray.shape[0] * gray.shape[1]
|
| 245 |
valid_contours = [c for c in contours if cv2.contourArea(c) > roi_area * min_area_ratio]
|
| 246 |
|
| 247 |
if valid_contours:
|
| 248 |
largest_contour = valid_contours[0]
|
| 249 |
|
| 250 |
-
# Create a debug image showing detected contour
|
| 251 |
if DEBUG:
|
| 252 |
contour_debug = np.zeros_like(binary)
|
| 253 |
cv2.drawContours(contour_debug, [largest_contour], 0, 255, 2)
|
| 254 |
cv2.imwrite(os.path.join(DEBUG_DIR, f"paper_contour_{int(time.time())}.png"), contour_debug)
|
| 255 |
|
| 256 |
-
# Get the minimum area rectangle that bounds the contour
|
| 257 |
rect = cv2.minAreaRect(largest_contour)
|
| 258 |
box = cv2.boxPoints(rect)
|
| 259 |
box = np.int0(box)
|
| 260 |
|
| 261 |
-
# Create a debug image with rectangle
|
| 262 |
if DEBUG:
|
| 263 |
rect_debug = roi.copy() if len(roi.shape) == 3 else cv2.cvtColor(roi, cv2.COLOR_GRAY2RGB)
|
| 264 |
cv2.drawContours(rect_debug, [box], 0, (0, 0, 255), 2)
|
| 265 |
cv2.imwrite(os.path.join(DEBUG_DIR, f"paper_rect_{int(time.time())}.png"), rect_debug)
|
| 266 |
|
| 267 |
-
# Extract the angle from the rectangle
|
| 268 |
center, (width, height), angle = rect
|
| 269 |
|
| 270 |
-
# Adjust angle for consistent orientation
|
| 271 |
-
# OpenCV's minAreaRect returns angles in (-90, 0]
|
| 272 |
if width < height:
|
| 273 |
angle += 90
|
| 274 |
|
|
@@ -277,37 +300,28 @@ def detect_paper_angle(image, bounding_box):
|
|
| 277 |
except Exception as e:
|
| 278 |
logging.warning(f"Error in adaptive threshold method: {str(e)}")
|
| 279 |
|
| 280 |
-
# Method
|
| 281 |
try:
|
| 282 |
-
# Apply Gaussian blur to reduce noise
|
| 283 |
blurred = cv2.GaussianBlur(gray, (5, 5), 0)
|
| 284 |
-
|
| 285 |
-
# Use automatic thresholding to determine Canny parameters
|
| 286 |
median = np.median(blurred)
|
| 287 |
lower = int(max(0, (1.0 - 0.33) * median))
|
| 288 |
upper = int(min(255, (1.0 + 0.33) * median))
|
| 289 |
-
|
| 290 |
-
# Apply edge detection with dynamic thresholds
|
| 291 |
edges = cv2.Canny(blurred, lower, upper)
|
| 292 |
|
| 293 |
-
# Save edges image for debugging
|
| 294 |
if DEBUG:
|
| 295 |
cv2.imwrite(os.path.join(DEBUG_DIR, f"canny_edges_{int(time.time())}.png"), edges)
|
| 296 |
|
| 297 |
-
# Dilate edges to connect broken lines
|
| 298 |
kernel = np.ones((3, 3), np.uint8)
|
| 299 |
dilated_edges = cv2.dilate(edges, kernel, iterations=1)
|
| 300 |
|
| 301 |
-
# Find lines using Hough Line Transform with more sensitive parameters
|
| 302 |
lines = cv2.HoughLinesP(
|
| 303 |
dilated_edges, 1, np.pi/180,
|
| 304 |
-
threshold=50,
|
| 305 |
-
minLineLength=max(roi.shape[0], roi.shape[1]) // 10,
|
| 306 |
-
maxLineGap=20
|
| 307 |
)
|
| 308 |
|
| 309 |
if lines is not None and len(lines) > 0:
|
| 310 |
-
# Draw all detected lines for debugging
|
| 311 |
if DEBUG:
|
| 312 |
lines_debug = roi.copy() if len(roi.shape) == 3 else cv2.cvtColor(roi, cv2.COLOR_GRAY2RGB)
|
| 313 |
for line in lines:
|
|
@@ -315,7 +329,6 @@ def detect_paper_angle(image, bounding_box):
|
|
| 315 |
cv2.line(lines_debug, (x1_l, y1_l), (x2_l, y2_l), (0, 255, 255), 2)
|
| 316 |
cv2.imwrite(os.path.join(DEBUG_DIR, f"hough_lines_{int(time.time())}.png"), lines_debug)
|
| 317 |
|
| 318 |
-
# Find the longest line
|
| 319 |
longest_line = max(
|
| 320 |
lines,
|
| 321 |
key=lambda line: np.linalg.norm(
|
|
@@ -324,13 +337,10 @@ def detect_paper_angle(image, bounding_box):
|
|
| 324 |
)
|
| 325 |
x1_l, y1_l, x2_l, y2_l = longest_line[0]
|
| 326 |
|
| 327 |
-
# Calculate the angle of the line
|
| 328 |
dx = x2_l - x1_l
|
| 329 |
dy = y2_l - y1_l
|
| 330 |
angle = degrees(atan2(dy, dx))
|
| 331 |
|
| 332 |
-
# Normalize angle to be between -45 and 45 degrees
|
| 333 |
-
# (assuming paper is roughly rectangular)
|
| 334 |
if angle > 45:
|
| 335 |
angle -= 90
|
| 336 |
elif angle < -45:
|
|
@@ -341,22 +351,20 @@ def detect_paper_angle(image, bounding_box):
|
|
| 341 |
except Exception as e:
|
| 342 |
logging.warning(f"Error in Hough lines method: {str(e)}")
|
| 343 |
|
| 344 |
-
# If all methods fail, return 0 (no rotation)
|
| 345 |
logging.warning("All paper angle detection methods failed, defaulting to 0 degrees")
|
| 346 |
return 0
|
| 347 |
|
| 348 |
-
#
|
|
|
|
|
|
|
| 349 |
def extract_text_from_handwriting(image_path):
|
| 350 |
try:
|
| 351 |
-
# Create a copy of the image in a temporary location
|
| 352 |
temp_dir = tempfile.mkdtemp()
|
| 353 |
temp_image_path = os.path.join(temp_dir, "trimmed_handwriting.png")
|
| 354 |
debug_image_path = os.path.join(temp_dir, "debug_extraction.png")
|
| 355 |
|
| 356 |
-
# Open the image
|
| 357 |
img = Image.open(image_path).convert("RGBA")
|
| 358 |
|
| 359 |
-
# Save the original for debugging
|
| 360 |
if DEBUG:
|
| 361 |
debug_img = img.copy()
|
| 362 |
draw = ImageDraw.Draw(debug_img)
|
|
@@ -367,57 +375,46 @@ def extract_text_from_handwriting(image_path):
|
|
| 367 |
)
|
| 368 |
debug_img.save(os.path.join(DEBUG_DIR, "original_handwriting.png"))
|
| 369 |
|
| 370 |
-
# Get the original dimensions
|
| 371 |
original_width, original_height = img.width, img.height
|
| 372 |
|
| 373 |
-
# Get the bounding box of non-zero areas (text content)
|
| 374 |
gray_img = img.convert('L')
|
| 375 |
-
|
| 376 |
-
thresh = 240 # Higher threshold to catch light text
|
| 377 |
binary_img = gray_img.point(lambda p: p < thresh and 255)
|
| 378 |
-
# Get bounding box of text
|
| 379 |
bbox = ImageOps.invert(binary_img).getbbox()
|
| 380 |
|
| 381 |
text_dimensions = {}
|
| 382 |
text_dimensions['original'] = {'width': original_width, 'height': original_height}
|
| 383 |
|
| 384 |
if bbox:
|
| 385 |
-
|
| 386 |
-
padding = 20 # Increased padding
|
| 387 |
left, upper, right, lower = bbox
|
| 388 |
|
| 389 |
-
# Calculate non-whitespace area dimensions
|
| 390 |
text_width = right - left
|
| 391 |
text_height = lower - upper
|
| 392 |
|
| 393 |
-
# Add debug info
|
| 394 |
text_dimensions['text_only'] = {'width': text_width, 'height': text_height}
|
| 395 |
text_dimensions['text_percentage'] = {
|
| 396 |
'width': (text_width / original_width) * 100,
|
| 397 |
'height': (text_height / original_height) * 100
|
| 398 |
}
|
| 399 |
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
|
|
|
| 405 |
|
| 406 |
-
# Crop the image to the bounding box
|
| 407 |
trimmed_img = img.crop(bbox)
|
| 408 |
trimmed_img.save(temp_image_path)
|
| 409 |
|
| 410 |
-
# Final trimmed dimensions
|
| 411 |
trimmed_width, trimmed_height = trimmed_img.width, trimmed_img.height
|
| 412 |
text_dimensions['trimmed'] = {'width': trimmed_width, 'height': trimmed_height}
|
| 413 |
|
| 414 |
-
# Create a debug image showing the extraction
|
| 415 |
if DEBUG:
|
| 416 |
debug_img = img.copy()
|
| 417 |
draw = ImageDraw.Draw(debug_img)
|
| 418 |
-
# Draw original bounding box
|
| 419 |
draw.rectangle(bbox, outline=(255, 0, 0, 255), width=2)
|
| 420 |
-
# Add text annotation
|
| 421 |
draw.text(
|
| 422 |
(bbox[0], bbox[1] - 15),
|
| 423 |
(
|
|
@@ -428,13 +425,11 @@ def extract_text_from_handwriting(image_path):
|
|
| 428 |
fill=(255, 0, 0, 255)
|
| 429 |
)
|
| 430 |
debug_img.save(debug_image_path)
|
| 431 |
-
# Save for reference
|
| 432 |
debug_img.save(os.path.join(DEBUG_DIR, "text_extraction.png"))
|
| 433 |
|
| 434 |
logging.debug(f"Text extraction: {text_dimensions}")
|
| 435 |
return temp_image_path, temp_dir, text_dimensions
|
| 436 |
else:
|
| 437 |
-
# If no content found, just return the original
|
| 438 |
shutil.copy(image_path, temp_image_path)
|
| 439 |
text_dimensions['error'] = "No text content detected"
|
| 440 |
logging.warning("No text content detected in handwriting image")
|
|
@@ -443,27 +438,26 @@ def extract_text_from_handwriting(image_path):
|
|
| 443 |
logging.error(f"Error extracting text from image: {str(e)}")
|
| 444 |
return image_path, None, {'error': str(e)}
|
| 445 |
|
| 446 |
-
#
|
|
|
|
|
|
|
| 447 |
def process_image(image, text, style, bias, color, stroke_width):
|
| 448 |
-
temp_dirs = []
|
| 449 |
|
| 450 |
try:
|
| 451 |
timestamp = int(time.time())
|
| 452 |
|
| 453 |
-
# Save input image for reference
|
| 454 |
input_debug_path = os.path.join(DEBUG_DIR, f"{timestamp}_input.jpg")
|
| 455 |
image.save(input_debug_path)
|
| 456 |
|
| 457 |
-
#
|
| 458 |
rf_local = Roboflow(api_key=ROBOFLOW_API_KEY)
|
| 459 |
project_local = rf_local.workspace().project(PROJECT_NAME)
|
| 460 |
model_local = project_local.version(VERSION_NUMBER).model
|
| 461 |
|
| 462 |
-
# Save input image temporarily
|
| 463 |
input_image_path = "/tmp/input_image.jpg"
|
| 464 |
image.save(input_image_path)
|
| 465 |
|
| 466 |
-
# Perform inference to detect papers
|
| 467 |
prediction = model_local.predict(input_image_path, confidence=70, overlap=50).json()
|
| 468 |
num_papers = len(prediction['predictions'])
|
| 469 |
logging.debug(f"Detected {num_papers} papers")
|
|
@@ -471,37 +465,36 @@ def process_image(image, text, style, bias, color, stroke_width):
|
|
| 471 |
if num_papers == 0:
|
| 472 |
logging.error("No papers detected in the image")
|
| 473 |
return None
|
| 474 |
-
|
| 475 |
-
# Format text
|
| 476 |
-
if
|
| 477 |
obj0 = prediction['predictions'][0]
|
| 478 |
paper_width = obj0['width']
|
| 479 |
-
|
| 480 |
-
# Calculate usable width (accounting for padding)
|
| 481 |
padding_x = int(paper_width * 0.1)
|
| 482 |
usable_width = paper_width - 2 * padding_x
|
| 483 |
-
|
| 484 |
-
# Format text to fit within paper boundaries
|
| 485 |
formatted_text = format_text_for_paper(text, usable_width)
|
| 486 |
logging.debug(f"Formatted text for paper width {usable_width}px: \n{formatted_text}")
|
| 487 |
else:
|
| 488 |
formatted_text = text
|
| 489 |
logging.debug("No papers detected, using original text")
|
| 490 |
|
| 491 |
-
#
|
| 492 |
logging.debug(f"Calling handwriting model with formatted text: '{formatted_text}'")
|
| 493 |
handwriting_client = get_handwriting_client()
|
| 494 |
result = handwriting_client.predict(
|
| 495 |
-
formatted_text,
|
| 496 |
-
style,
|
| 497 |
-
bias,
|
| 498 |
-
color,
|
| 499 |
-
stroke_width,
|
| 500 |
api_name="/generate_handwriting_wrapper"
|
| 501 |
)
|
| 502 |
-
|
| 503 |
-
svg_content,
|
| 504 |
-
logging.debug(f"
|
|
|
|
|
|
|
|
|
|
| 505 |
|
| 506 |
# Save original handwriting for reference
|
| 507 |
orig_hw_debug_path = os.path.join(DEBUG_DIR, f"{timestamp}_original_handwriting.png")
|
|
@@ -511,55 +504,46 @@ def process_image(image, text, style, bias, color, stroke_width):
|
|
| 511 |
except Exception as e:
|
| 512 |
logging.error(f"Error saving original handwriting: {str(e)}")
|
| 513 |
|
| 514 |
-
#
|
| 515 |
trimmed_path, temp_dir, text_dimensions = extract_text_from_handwriting(png_path)
|
| 516 |
if temp_dir:
|
| 517 |
temp_dirs.append(temp_dir)
|
| 518 |
-
|
| 519 |
-
# Log text dimensions
|
| 520 |
logging.debug(f"Handwriting dimensions: {text_dimensions}")
|
| 521 |
|
| 522 |
-
# Load the trimmed handwriting image
|
| 523 |
handwriting_img = Image.open(trimmed_path).convert("RGBA")
|
| 524 |
logging.debug(f"Loaded trimmed handwriting image: {handwriting_img.width}x{handwriting_img.height}")
|
| 525 |
|
| 526 |
-
# Save trimmed handwriting for reference
|
| 527 |
trimmed_hw_debug_path = os.path.join(DEBUG_DIR, f"{timestamp}_trimmed_handwriting.png")
|
| 528 |
handwriting_img.save(trimmed_hw_debug_path)
|
| 529 |
|
| 530 |
-
# Convert the input image to RGBA for processing
|
| 531 |
pil_image = image.convert("RGBA")
|
| 532 |
|
| 533 |
-
# Create a debug image showing detected papers
|
| 534 |
debug_image = pil_image.copy()
|
| 535 |
debug_draw = ImageDraw.Draw(debug_image)
|
| 536 |
|
| 537 |
-
#
|
| 538 |
for i, obj in enumerate(prediction['predictions']):
|
| 539 |
-
# Get paper dimensions
|
| 540 |
paper_width = obj['width']
|
| 541 |
paper_height = obj['height']
|
| 542 |
|
| 543 |
-
# Log paper dimensions
|
| 544 |
logging.debug(f"Paper {i+1} dimensions: {paper_width}x{paper_height} at position ({obj['x']}, {obj['y']})")
|
| 545 |
|
| 546 |
-
# Add padding (20%)
|
| 547 |
padding_x = int(paper_width * 0.20)
|
| 548 |
padding_y = int(paper_height * 0.20)
|
| 549 |
|
| 550 |
-
# Calculate available area for text
|
| 551 |
box_width = paper_width - 2 * padding_x
|
| 552 |
box_height = paper_height - 2 * padding_y
|
| 553 |
|
| 554 |
-
# Calculate text box coordinates
|
| 555 |
x1 = int(obj['x'] - paper_width / 2 + padding_x)
|
| 556 |
y1 = int(obj['y'] - paper_height / 2 + padding_y)
|
| 557 |
x2 = int(obj['x'] + paper_width / 2 - padding_x)
|
| 558 |
y2 = int(obj['y'] + paper_height / 2 - padding_y)
|
| 559 |
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
|
|
|
| 563 |
debug_draw.rectangle(paper_box, outline=(0, 255, 0, 255), width=3)
|
| 564 |
debug_draw.text(
|
| 565 |
(paper_box[0][0], paper_box[0][1] - 15),
|
|
@@ -567,7 +551,6 @@ def process_image(image, text, style, bias, color, stroke_width):
|
|
| 567 |
fill=(0, 255, 0, 255)
|
| 568 |
)
|
| 569 |
|
| 570 |
-
# Draw usable area on debug image
|
| 571 |
usable_box = [(x1, y1), (x2, y2)]
|
| 572 |
debug_draw.rectangle(usable_box, outline=(255, 255, 0, 255), width=2)
|
| 573 |
debug_draw.text(
|
|
@@ -576,20 +559,17 @@ def process_image(image, text, style, bias, color, stroke_width):
|
|
| 576 |
fill=(255, 255, 0, 255)
|
| 577 |
)
|
| 578 |
|
| 579 |
-
# Paper coordinates for detecting the actual paper orientation
|
| 580 |
paper_x1 = int(obj['x'] - paper_width / 2)
|
| 581 |
paper_y1 = int(obj['y'] - paper_height / 2)
|
| 582 |
paper_x2 = int(obj['x'] + paper_width / 2)
|
| 583 |
paper_y2 = int(obj['y'] + paper_height / 2)
|
| 584 |
|
| 585 |
-
# Detect the actual paper angle (not just the bounding box)
|
| 586 |
angle = detect_paper_angle(
|
| 587 |
np.array(image),
|
| 588 |
(paper_x1, paper_y1, paper_x2, paper_y2)
|
| 589 |
)
|
| 590 |
logging.debug(f"Paper {i+1} angle: {angle} degrees")
|
| 591 |
|
| 592 |
-
# Add a debug visualization of the detected angle
|
| 593 |
debug_draw.line(
|
| 594 |
[
|
| 595 |
(obj['x'], obj['y']),
|
|
@@ -607,34 +587,24 @@ def process_image(image, text, style, bias, color, stroke_width):
|
|
| 607 |
fill=(255, 0, 0, 255)
|
| 608 |
)
|
| 609 |
|
| 610 |
-
# Calculate the initial size while maintaining aspect ratio
|
| 611 |
handwriting_aspect = handwriting_img.width / handwriting_img.height
|
| 612 |
|
| 613 |
-
# Start with the full usable width
|
| 614 |
target_width = box_width
|
| 615 |
-
|
| 616 |
-
# Apply scale factor to make text larger (but don't exceed 2x usable width)
|
| 617 |
target_width = min(int(target_width * TEXT_SCALE_FACTOR), box_width * 2)
|
| 618 |
-
|
| 619 |
-
# Calculate height based on aspect ratio
|
| 620 |
target_height = int(target_width / handwriting_aspect)
|
| 621 |
|
| 622 |
-
# If too tall, constrain by height
|
| 623 |
if target_height > box_height:
|
| 624 |
target_height = box_height
|
| 625 |
target_width = int(target_height * handwriting_aspect)
|
| 626 |
|
| 627 |
-
# Ensure we're not making text too small
|
| 628 |
min_width = int(box_width * MIN_WIDTH_PERCENTAGE)
|
| 629 |
if target_width < min_width:
|
| 630 |
target_width = min_width
|
| 631 |
target_height = int(target_width / handwriting_aspect)
|
| 632 |
-
# Check height again
|
| 633 |
if target_height > box_height:
|
| 634 |
target_height = box_height
|
| 635 |
target_width = int(target_height * handwriting_aspect)
|
| 636 |
|
| 637 |
-
# Log sizing calculations
|
| 638 |
logging.debug(
|
| 639 |
f"Paper {i+1} usable area: {box_width}x{box_height}"
|
| 640 |
)
|
|
@@ -645,7 +615,6 @@ def process_image(image, text, style, bias, color, stroke_width):
|
|
| 645 |
f"(scale factor={TEXT_SCALE_FACTOR})"
|
| 646 |
)
|
| 647 |
|
| 648 |
-
# Draw text area on debug image
|
| 649 |
text_center_x = x1 + box_width // 2
|
| 650 |
text_center_y = y1 + box_height // 2
|
| 651 |
text_box = [
|
|
@@ -659,37 +628,30 @@ def process_image(image, text, style, bias, color, stroke_width):
|
|
| 659 |
fill=(255, 0, 255, 255)
|
| 660 |
)
|
| 661 |
|
| 662 |
-
# Resize the handwriting with the calculated dimensions
|
| 663 |
resized_handwriting = handwriting_img.resize(
|
| 664 |
(target_width, target_height),
|
| 665 |
Image.LANCZOS
|
| 666 |
)
|
| 667 |
|
| 668 |
-
# Save resized handwriting for reference
|
| 669 |
resized_hw_debug_path = os.path.join(
|
| 670 |
DEBUG_DIR,
|
| 671 |
f"{timestamp}_resized_handwriting_{i+1}.png"
|
| 672 |
)
|
| 673 |
resized_handwriting.save(resized_hw_debug_path)
|
| 674 |
|
| 675 |
-
# Create a transparent layer for the handwriting
|
| 676 |
handwriting_layer = Image.new("RGBA", pil_image.size, (0, 0, 0, 0))
|
| 677 |
|
| 678 |
-
# Center handwriting on paper
|
| 679 |
paste_x = x1 + (box_width - target_width) // 2
|
| 680 |
paste_y = y1 + (box_height - target_height) // 2
|
| 681 |
|
| 682 |
-
# Paste handwriting onto layer
|
| 683 |
handwriting_layer.paste(resized_handwriting, (paste_x, paste_y), resized_handwriting)
|
| 684 |
|
| 685 |
-
# Add to debug image
|
| 686 |
debug_paste_box = [
|
| 687 |
(paste_x, paste_y),
|
| 688 |
(paste_x + target_width, paste_y + target_height)
|
| 689 |
]
|
| 690 |
debug_draw.rectangle(debug_paste_box, outline=(0, 0, 255, 255), width=1)
|
| 691 |
|
| 692 |
-
# Create another debug visualization showing rotation center and angle
|
| 693 |
rotation_debug_path = os.path.join(
|
| 694 |
DEBUG_DIR,
|
| 695 |
f"{timestamp}_rotation_paper_{i+1}.png"
|
|
@@ -720,37 +682,30 @@ def process_image(image, text, style, bias, color, stroke_width):
|
|
| 720 |
)
|
| 721 |
rotation_debug.save(rotation_debug_path)
|
| 722 |
|
| 723 |
-
# Rotate to match paper angle
|
| 724 |
rotated_layer = handwriting_layer.rotate(
|
| 725 |
-angle,
|
| 726 |
resample=Image.BICUBIC,
|
| 727 |
center=(obj['x'], obj['y'])
|
| 728 |
)
|
| 729 |
|
| 730 |
-
# Composite onto original image
|
| 731 |
pil_image = Image.alpha_composite(pil_image, rotated_layer)
|
| 732 |
|
| 733 |
-
# Save debug image
|
| 734 |
debug_path = os.path.join(DEBUG_DIR, f"{timestamp}_debug_overlay.png")
|
| 735 |
debug_image.save(debug_path)
|
| 736 |
logging.debug(f"Saved debug overlay image to {debug_path}")
|
| 737 |
|
| 738 |
-
# Save final result
|
| 739 |
output_path = "/tmp/output_image.png"
|
| 740 |
pil_image.convert("RGB").save(output_path)
|
| 741 |
|
| 742 |
-
# Also save a copy to debug directory for reference
|
| 743 |
final_debug_path = os.path.join(DEBUG_DIR, f"{timestamp}_final_output.png")
|
| 744 |
pil_image.save(final_debug_path)
|
| 745 |
|
| 746 |
-
# Clean up temporary directories
|
| 747 |
for dir_path in temp_dirs:
|
| 748 |
try:
|
| 749 |
shutil.rmtree(dir_path)
|
| 750 |
except Exception as e:
|
| 751 |
logging.warning(f"Failed to clean up temporary directory {dir_path}: {str(e)}")
|
| 752 |
|
| 753 |
-
# Create a comprehensive debug report
|
| 754 |
debug_report = {
|
| 755 |
'timestamp': timestamp,
|
| 756 |
'input_image': input_debug_path,
|
|
@@ -779,17 +734,14 @@ def process_image(image, text, style, bias, color, stroke_width):
|
|
| 779 |
'final_output': final_debug_path
|
| 780 |
}
|
| 781 |
|
| 782 |
-
# Log the debug report
|
| 783 |
logging.debug(f"Debug report: {debug_report}")
|
| 784 |
|
| 785 |
-
# Return paths to debug images along with the output path
|
| 786 |
return {
|
| 787 |
'output_path': output_path,
|
| 788 |
'debug_report': debug_report
|
| 789 |
}
|
| 790 |
|
| 791 |
except Exception as e:
|
| 792 |
-
# Clean up temporary directories
|
| 793 |
for dir_path in temp_dirs:
|
| 794 |
try:
|
| 795 |
shutil.rmtree(dir_path)
|
|
@@ -799,10 +751,12 @@ def process_image(image, text, style, bias, color, stroke_width):
|
|
| 799 |
logging.error(f"Error: {str(e)}")
|
| 800 |
raise
|
| 801 |
|
| 802 |
-
#
|
|
|
|
|
|
|
| 803 |
def gradio_process(image, text, style, bias, color, stroke_width, text_size):
|
| 804 |
global TEXT_SCALE_FACTOR
|
| 805 |
-
TEXT_SCALE_FACTOR = text_size
|
| 806 |
|
| 807 |
if image is None:
|
| 808 |
return None, None, "Please upload an image with paper."
|
|
@@ -818,11 +772,9 @@ def gradio_process(image, text, style, bias, color, stroke_width, text_size):
|
|
| 818 |
output_path = result['output_path']
|
| 819 |
debug_report = result['debug_report']
|
| 820 |
|
| 821 |
-
# Generate a detailed message about the process
|
| 822 |
debug_msg = f"Processing complete!\n\n"
|
| 823 |
debug_msg += f"Debug information in: {DEBUG_DIR}\n"
|
| 824 |
|
| 825 |
-
# Add information about handwriting dimensions
|
| 826 |
if 'text_dimensions' in debug_report:
|
| 827 |
td = debug_report['text_dimensions']
|
| 828 |
if 'original' in td:
|
|
@@ -834,7 +786,6 @@ def gradio_process(image, text, style, bias, color, stroke_width, text_size):
|
|
| 834 |
if 'trimmed' in td:
|
| 835 |
debug_msg += f"Trimmed size: {td['trimmed']['width']}x{td['trimmed']['height']} px\n"
|
| 836 |
|
| 837 |
-
# Add information about paper
|
| 838 |
if 'paper_dimensions' in debug_report and len(debug_report['paper_dimensions']) > 0:
|
| 839 |
paper = debug_report['paper_dimensions'][0]
|
| 840 |
debug_msg += f"Detected paper: {paper['width']}x{paper['height']} px\n"
|
|
@@ -847,7 +798,9 @@ def gradio_process(image, text, style, bias, color, stroke_width, text_size):
|
|
| 847 |
logging.exception("Processing error")
|
| 848 |
return None, None, f"Error: {str(e)}"
|
| 849 |
|
| 850 |
-
#
|
|
|
|
|
|
|
| 851 |
interface = gr.Interface(
|
| 852 |
fn=gradio_process,
|
| 853 |
inputs=[
|
|
@@ -872,6 +825,5 @@ interface = gr.Interface(
|
|
| 872 |
)
|
| 873 |
)
|
| 874 |
|
| 875 |
-
# Launch app
|
| 876 |
if __name__ == "__main__":
|
| 877 |
interface.launch(share=True)
|
|
|
|
| 10 |
from roboflow import Roboflow
|
| 11 |
from gradio_client import Client
|
| 12 |
import gradio as gr
|
| 13 |
+
import requests # <-- for downloading PNG from URL
|
| 14 |
|
| 15 |
# -------------------------------------------------------------------------
|
| 16 |
# 🧠 Fix for Gradio schema bug ("TypeError: argument of type 'bool' is not iterable")
|
|
|
|
| 36 |
try:
|
| 37 |
if isinstance(schema, dict) and "anyOf" in schema:
|
| 38 |
types = [s.get("type") for s in schema["anyOf"] if isinstance(s, dict)]
|
|
|
|
| 39 |
if set(types) == {"string", "null"}:
|
| 40 |
return "Optional[str]"
|
|
|
|
| 41 |
return gu._json_schema_to_python_type_original(schema, defs)
|
| 42 |
except Exception:
|
| 43 |
return "UnknownType"
|
| 44 |
|
|
|
|
| 45 |
if not hasattr(gu, "_json_schema_to_python_type_original"):
|
| 46 |
gu._json_schema_to_python_type_original = gu._json_schema_to_python_type
|
| 47 |
gu._json_schema_to_python_type = _safe_json_schema_to_python_type
|
|
|
|
| 74 |
# -------------------------------------------------------------------------
|
| 75 |
# 🤖 Roboflow configuration
|
| 76 |
# -------------------------------------------------------------------------
|
| 77 |
+
ROBOFLOW_API_KEY = "u5LX112EBlNmzYoofvPL"
|
| 78 |
PROJECT_NAME = "model_verification_project"
|
| 79 |
VERSION_NUMBER = 2
|
| 80 |
|
|
|
|
| 81 |
os.environ["ROBOFLOW_API_KEY"] = ROBOFLOW_API_KEY
|
| 82 |
|
| 83 |
# -------------------------------------------------------------------------
|
|
|
|
| 85 |
# -------------------------------------------------------------------------
|
| 86 |
HANDWRITING_MODEL_ENDPOINT = "3morrrrr/Handwriting_Model_Inf"
|
| 87 |
|
| 88 |
+
# Cached handwriting client
|
| 89 |
_handwriting_client = None
|
| 90 |
|
| 91 |
def get_handwriting_client(max_retries=5, retry_delay=3):
|
| 92 |
"""
|
| 93 |
+
Lazily initialize and cache the handwriting Client with retries.
|
| 94 |
+
Avoids crashing the app if the Space is waking up / slow.
|
|
|
|
|
|
|
| 95 |
"""
|
| 96 |
global _handwriting_client
|
| 97 |
if _handwriting_client is not None:
|
|
|
|
| 128 |
os.makedirs(DEBUG_DIR, exist_ok=True)
|
| 129 |
|
| 130 |
logging.info(f"Debug images stored in: {DEBUG_DIR}")
|
| 131 |
+
logging.info(
|
| 132 |
+
f"Using Roboflow project '{PROJECT_NAME}' (v{VERSION_NUMBER}) "
|
| 133 |
+
f"with API key ending in {ROBOFLOW_API_KEY[-4:]}"
|
| 134 |
+
)
|
| 135 |
logging.info(f"Using handwriting model endpoint: {HANDWRITING_MODEL_ENDPOINT}")
|
| 136 |
|
| 137 |
# -------------------------------------------------------------------------
|
|
|
|
| 169 |
logging.debug(f"Saved debug image: {path}")
|
| 170 |
return path
|
| 171 |
|
| 172 |
+
def ensure_local_png(png_output):
|
| 173 |
+
"""
|
| 174 |
+
Handle Gradio / HF output for the PNG:
|
| 175 |
+
- If it's a path string, return it.
|
| 176 |
+
- If it's a dict, use .path or .url.
|
| 177 |
+
- If it's a URL, download it to a temp file.
|
| 178 |
+
"""
|
| 179 |
+
if png_output is None:
|
| 180 |
+
raise ValueError("Handwriting model returned no PNG output (None).")
|
| 181 |
+
|
| 182 |
+
png_path = None
|
| 183 |
+
|
| 184 |
+
# Case 1: plain string path
|
| 185 |
+
if isinstance(png_output, str):
|
| 186 |
+
png_path = png_output
|
| 187 |
+
|
| 188 |
+
# Case 2: dict from Gradio output: {path, url, ...}
|
| 189 |
+
elif isinstance(png_output, dict):
|
| 190 |
+
png_path = png_output.get("path") or png_output.get("url")
|
| 191 |
+
|
| 192 |
+
else:
|
| 193 |
+
raise ValueError(f"Unexpected PNG output type: {type(png_output)}")
|
| 194 |
+
|
| 195 |
+
if not png_path:
|
| 196 |
+
raise ValueError(f"PNG output from handwriting model is missing a path/url: {png_output}")
|
| 197 |
+
|
| 198 |
+
# If already a local file path
|
| 199 |
+
if os.path.exists(png_path):
|
| 200 |
+
return png_path
|
| 201 |
+
|
| 202 |
+
# If it's a URL, download it
|
| 203 |
+
if isinstance(png_path, str) and png_path.startswith("http"):
|
| 204 |
+
logging.debug(f"Downloading PNG from URL: {png_path}")
|
| 205 |
+
temp_png = os.path.join(tempfile.gettempdir(), f"handwriting_{int(time.time())}.png")
|
| 206 |
+
try:
|
| 207 |
+
r = requests.get(png_path, stream=True, timeout=30)
|
| 208 |
+
r.raise_for_status()
|
| 209 |
+
with open(temp_png, "wb") as f:
|
| 210 |
+
shutil.copyfileobj(r.raw, f)
|
| 211 |
+
logging.debug(f"Downloaded PNG to {temp_png}")
|
| 212 |
+
return temp_png
|
| 213 |
+
except Exception as e:
|
| 214 |
+
raise RuntimeError(f"Failed to download PNG from URL: {e}")
|
| 215 |
+
|
| 216 |
+
# Any other weird case
|
| 217 |
+
raise ValueError(f"Invalid PNG path returned: {png_path}")
|
| 218 |
+
|
| 219 |
# -------------------------------------------------------------------------
|
| 220 |
# 🧠 Load Roboflow models
|
| 221 |
# -------------------------------------------------------------------------
|
|
|
|
| 223 |
project = rf.workspace().project(PROJECT_NAME)
|
| 224 |
model = project.version(VERSION_NUMBER).model
|
| 225 |
|
| 226 |
+
# -------------------------------------------------------------------------
|
| 227 |
+
# 📐 Detect paper angle
|
| 228 |
+
# -------------------------------------------------------------------------
|
| 229 |
def detect_paper_angle(image, bounding_box):
|
| 230 |
"""
|
| 231 |
+
Detect the angle of a paper document within the given bounding box.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 232 |
"""
|
| 233 |
x1, y1, x2, y2 = bounding_box
|
| 234 |
|
|
|
|
| 241 |
# Crop the region of interest (ROI)
|
| 242 |
roi = image_np[y1:y2, x1:x2]
|
| 243 |
|
|
|
|
| 244 |
if DEBUG:
|
| 245 |
debug_roi = Image.fromarray(roi)
|
| 246 |
save_debug_image(debug_roi, f"paper_roi_{int(time.time())}.png",
|
|
|
|
| 252 |
else:
|
| 253 |
gray = roi
|
| 254 |
|
|
|
|
| 255 |
if DEBUG:
|
| 256 |
cv2.imwrite(os.path.join(DEBUG_DIR, f"gray_paper_{int(time.time())}.png"), gray)
|
| 257 |
|
| 258 |
+
# Method 1: adaptive thresholding
|
| 259 |
try:
|
|
|
|
|
|
|
| 260 |
binary = cv2.adaptiveThreshold(
|
| 261 |
gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
|
| 262 |
cv2.THRESH_BINARY_INV, 11, 2
|
| 263 |
)
|
| 264 |
|
|
|
|
| 265 |
if DEBUG:
|
| 266 |
cv2.imwrite(os.path.join(DEBUG_DIR, f"binary_paper_{int(time.time())}.png"), binary)
|
| 267 |
|
|
|
|
| 268 |
contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
| 269 |
|
| 270 |
+
if contours:
|
|
|
|
| 271 |
contours = sorted(contours, key=cv2.contourArea, reverse=True)
|
| 272 |
+
min_area_ratio = 0.05
|
|
|
|
|
|
|
|
|
|
| 273 |
roi_area = gray.shape[0] * gray.shape[1]
|
| 274 |
valid_contours = [c for c in contours if cv2.contourArea(c) > roi_area * min_area_ratio]
|
| 275 |
|
| 276 |
if valid_contours:
|
| 277 |
largest_contour = valid_contours[0]
|
| 278 |
|
|
|
|
| 279 |
if DEBUG:
|
| 280 |
contour_debug = np.zeros_like(binary)
|
| 281 |
cv2.drawContours(contour_debug, [largest_contour], 0, 255, 2)
|
| 282 |
cv2.imwrite(os.path.join(DEBUG_DIR, f"paper_contour_{int(time.time())}.png"), contour_debug)
|
| 283 |
|
|
|
|
| 284 |
rect = cv2.minAreaRect(largest_contour)
|
| 285 |
box = cv2.boxPoints(rect)
|
| 286 |
box = np.int0(box)
|
| 287 |
|
|
|
|
| 288 |
if DEBUG:
|
| 289 |
rect_debug = roi.copy() if len(roi.shape) == 3 else cv2.cvtColor(roi, cv2.COLOR_GRAY2RGB)
|
| 290 |
cv2.drawContours(rect_debug, [box], 0, (0, 0, 255), 2)
|
| 291 |
cv2.imwrite(os.path.join(DEBUG_DIR, f"paper_rect_{int(time.time())}.png"), rect_debug)
|
| 292 |
|
|
|
|
| 293 |
center, (width, height), angle = rect
|
| 294 |
|
|
|
|
|
|
|
| 295 |
if width < height:
|
| 296 |
angle += 90
|
| 297 |
|
|
|
|
| 300 |
except Exception as e:
|
| 301 |
logging.warning(f"Error in adaptive threshold method: {str(e)}")
|
| 302 |
|
| 303 |
+
# Method 2: Canny + Hough lines
|
| 304 |
try:
|
|
|
|
| 305 |
blurred = cv2.GaussianBlur(gray, (5, 5), 0)
|
|
|
|
|
|
|
| 306 |
median = np.median(blurred)
|
| 307 |
lower = int(max(0, (1.0 - 0.33) * median))
|
| 308 |
upper = int(min(255, (1.0 + 0.33) * median))
|
|
|
|
|
|
|
| 309 |
edges = cv2.Canny(blurred, lower, upper)
|
| 310 |
|
|
|
|
| 311 |
if DEBUG:
|
| 312 |
cv2.imwrite(os.path.join(DEBUG_DIR, f"canny_edges_{int(time.time())}.png"), edges)
|
| 313 |
|
|
|
|
| 314 |
kernel = np.ones((3, 3), np.uint8)
|
| 315 |
dilated_edges = cv2.dilate(edges, kernel, iterations=1)
|
| 316 |
|
|
|
|
| 317 |
lines = cv2.HoughLinesP(
|
| 318 |
dilated_edges, 1, np.pi/180,
|
| 319 |
+
threshold=50,
|
| 320 |
+
minLineLength=max(roi.shape[0], roi.shape[1]) // 10,
|
| 321 |
+
maxLineGap=20
|
| 322 |
)
|
| 323 |
|
| 324 |
if lines is not None and len(lines) > 0:
|
|
|
|
| 325 |
if DEBUG:
|
| 326 |
lines_debug = roi.copy() if len(roi.shape) == 3 else cv2.cvtColor(roi, cv2.COLOR_GRAY2RGB)
|
| 327 |
for line in lines:
|
|
|
|
| 329 |
cv2.line(lines_debug, (x1_l, y1_l), (x2_l, y2_l), (0, 255, 255), 2)
|
| 330 |
cv2.imwrite(os.path.join(DEBUG_DIR, f"hough_lines_{int(time.time())}.png"), lines_debug)
|
| 331 |
|
|
|
|
| 332 |
longest_line = max(
|
| 333 |
lines,
|
| 334 |
key=lambda line: np.linalg.norm(
|
|
|
|
| 337 |
)
|
| 338 |
x1_l, y1_l, x2_l, y2_l = longest_line[0]
|
| 339 |
|
|
|
|
| 340 |
dx = x2_l - x1_l
|
| 341 |
dy = y2_l - y1_l
|
| 342 |
angle = degrees(atan2(dy, dx))
|
| 343 |
|
|
|
|
|
|
|
| 344 |
if angle > 45:
|
| 345 |
angle -= 90
|
| 346 |
elif angle < -45:
|
|
|
|
| 351 |
except Exception as e:
|
| 352 |
logging.warning(f"Error in Hough lines method: {str(e)}")
|
| 353 |
|
|
|
|
| 354 |
logging.warning("All paper angle detection methods failed, defaulting to 0 degrees")
|
| 355 |
return 0
|
| 356 |
|
| 357 |
+
# -------------------------------------------------------------------------
|
| 358 |
+
# ✂ Trim whitespace from handwriting image
|
| 359 |
+
# -------------------------------------------------------------------------
|
| 360 |
def extract_text_from_handwriting(image_path):
|
| 361 |
try:
|
|
|
|
| 362 |
temp_dir = tempfile.mkdtemp()
|
| 363 |
temp_image_path = os.path.join(temp_dir, "trimmed_handwriting.png")
|
| 364 |
debug_image_path = os.path.join(temp_dir, "debug_extraction.png")
|
| 365 |
|
|
|
|
| 366 |
img = Image.open(image_path).convert("RGBA")
|
| 367 |
|
|
|
|
| 368 |
if DEBUG:
|
| 369 |
debug_img = img.copy()
|
| 370 |
draw = ImageDraw.Draw(debug_img)
|
|
|
|
| 375 |
)
|
| 376 |
debug_img.save(os.path.join(DEBUG_DIR, "original_handwriting.png"))
|
| 377 |
|
|
|
|
| 378 |
original_width, original_height = img.width, img.height
|
| 379 |
|
|
|
|
| 380 |
gray_img = img.convert('L')
|
| 381 |
+
thresh = 240
|
|
|
|
| 382 |
binary_img = gray_img.point(lambda p: p < thresh and 255)
|
|
|
|
| 383 |
bbox = ImageOps.invert(binary_img).getbbox()
|
| 384 |
|
| 385 |
text_dimensions = {}
|
| 386 |
text_dimensions['original'] = {'width': original_width, 'height': original_height}
|
| 387 |
|
| 388 |
if bbox:
|
| 389 |
+
padding = 20
|
|
|
|
| 390 |
left, upper, right, lower = bbox
|
| 391 |
|
|
|
|
| 392 |
text_width = right - left
|
| 393 |
text_height = lower - upper
|
| 394 |
|
|
|
|
| 395 |
text_dimensions['text_only'] = {'width': text_width, 'height': text_height}
|
| 396 |
text_dimensions['text_percentage'] = {
|
| 397 |
'width': (text_width / original_width) * 100,
|
| 398 |
'height': (text_height / original_height) * 100
|
| 399 |
}
|
| 400 |
|
| 401 |
+
bbox = (
|
| 402 |
+
max(0, left-padding),
|
| 403 |
+
max(0, upper-padding),
|
| 404 |
+
min(img.width, right+padding),
|
| 405 |
+
min(img.height, lower+padding)
|
| 406 |
+
)
|
| 407 |
|
|
|
|
| 408 |
trimmed_img = img.crop(bbox)
|
| 409 |
trimmed_img.save(temp_image_path)
|
| 410 |
|
|
|
|
| 411 |
trimmed_width, trimmed_height = trimmed_img.width, trimmed_img.height
|
| 412 |
text_dimensions['trimmed'] = {'width': trimmed_width, 'height': trimmed_height}
|
| 413 |
|
|
|
|
| 414 |
if DEBUG:
|
| 415 |
debug_img = img.copy()
|
| 416 |
draw = ImageDraw.Draw(debug_img)
|
|
|
|
| 417 |
draw.rectangle(bbox, outline=(255, 0, 0, 255), width=2)
|
|
|
|
| 418 |
draw.text(
|
| 419 |
(bbox[0], bbox[1] - 15),
|
| 420 |
(
|
|
|
|
| 425 |
fill=(255, 0, 0, 255)
|
| 426 |
)
|
| 427 |
debug_img.save(debug_image_path)
|
|
|
|
| 428 |
debug_img.save(os.path.join(DEBUG_DIR, "text_extraction.png"))
|
| 429 |
|
| 430 |
logging.debug(f"Text extraction: {text_dimensions}")
|
| 431 |
return temp_image_path, temp_dir, text_dimensions
|
| 432 |
else:
|
|
|
|
| 433 |
shutil.copy(image_path, temp_image_path)
|
| 434 |
text_dimensions['error'] = "No text content detected"
|
| 435 |
logging.warning("No text content detected in handwriting image")
|
|
|
|
| 438 |
logging.error(f"Error extracting text from image: {str(e)}")
|
| 439 |
return image_path, None, {'error': str(e)}
|
| 440 |
|
| 441 |
+
# -------------------------------------------------------------------------
|
| 442 |
+
# 🖼 Main processing function
|
| 443 |
+
# -------------------------------------------------------------------------
|
| 444 |
def process_image(image, text, style, bias, color, stroke_width):
|
| 445 |
+
temp_dirs = []
|
| 446 |
|
| 447 |
try:
|
| 448 |
timestamp = int(time.time())
|
| 449 |
|
|
|
|
| 450 |
input_debug_path = os.path.join(DEBUG_DIR, f"{timestamp}_input.jpg")
|
| 451 |
image.save(input_debug_path)
|
| 452 |
|
| 453 |
+
# Roboflow detection
|
| 454 |
rf_local = Roboflow(api_key=ROBOFLOW_API_KEY)
|
| 455 |
project_local = rf_local.workspace().project(PROJECT_NAME)
|
| 456 |
model_local = project_local.version(VERSION_NUMBER).model
|
| 457 |
|
|
|
|
| 458 |
input_image_path = "/tmp/input_image.jpg"
|
| 459 |
image.save(input_image_path)
|
| 460 |
|
|
|
|
| 461 |
prediction = model_local.predict(input_image_path, confidence=70, overlap=50).json()
|
| 462 |
num_papers = len(prediction['predictions'])
|
| 463 |
logging.debug(f"Detected {num_papers} papers")
|
|
|
|
| 465 |
if num_papers == 0:
|
| 466 |
logging.error("No papers detected in the image")
|
| 467 |
return None
|
| 468 |
+
|
| 469 |
+
# Format text using first paper width
|
| 470 |
+
if prediction['predictions']:
|
| 471 |
obj0 = prediction['predictions'][0]
|
| 472 |
paper_width = obj0['width']
|
|
|
|
|
|
|
| 473 |
padding_x = int(paper_width * 0.1)
|
| 474 |
usable_width = paper_width - 2 * padding_x
|
|
|
|
|
|
|
| 475 |
formatted_text = format_text_for_paper(text, usable_width)
|
| 476 |
logging.debug(f"Formatted text for paper width {usable_width}px: \n{formatted_text}")
|
| 477 |
else:
|
| 478 |
formatted_text = text
|
| 479 |
logging.debug("No papers detected, using original text")
|
| 480 |
|
| 481 |
+
# Call handwriting model
|
| 482 |
logging.debug(f"Calling handwriting model with formatted text: '{formatted_text}'")
|
| 483 |
handwriting_client = get_handwriting_client()
|
| 484 |
result = handwriting_client.predict(
|
| 485 |
+
formatted_text,
|
| 486 |
+
style,
|
| 487 |
+
bias,
|
| 488 |
+
color,
|
| 489 |
+
stroke_width,
|
| 490 |
api_name="/generate_handwriting_wrapper"
|
| 491 |
)
|
| 492 |
+
|
| 493 |
+
svg_content, png_output = result
|
| 494 |
+
logging.debug(f"Handwriting model raw PNG output: {png_output}")
|
| 495 |
+
|
| 496 |
+
png_path = ensure_local_png(png_output)
|
| 497 |
+
logging.debug(f"Using PNG path: {png_path}")
|
| 498 |
|
| 499 |
# Save original handwriting for reference
|
| 500 |
orig_hw_debug_path = os.path.join(DEBUG_DIR, f"{timestamp}_original_handwriting.png")
|
|
|
|
| 504 |
except Exception as e:
|
| 505 |
logging.error(f"Error saving original handwriting: {str(e)}")
|
| 506 |
|
| 507 |
+
# Extract text and dimensions
|
| 508 |
trimmed_path, temp_dir, text_dimensions = extract_text_from_handwriting(png_path)
|
| 509 |
if temp_dir:
|
| 510 |
temp_dirs.append(temp_dir)
|
| 511 |
+
|
|
|
|
| 512 |
logging.debug(f"Handwriting dimensions: {text_dimensions}")
|
| 513 |
|
|
|
|
| 514 |
handwriting_img = Image.open(trimmed_path).convert("RGBA")
|
| 515 |
logging.debug(f"Loaded trimmed handwriting image: {handwriting_img.width}x{handwriting_img.height}")
|
| 516 |
|
|
|
|
| 517 |
trimmed_hw_debug_path = os.path.join(DEBUG_DIR, f"{timestamp}_trimmed_handwriting.png")
|
| 518 |
handwriting_img.save(trimmed_hw_debug_path)
|
| 519 |
|
|
|
|
| 520 |
pil_image = image.convert("RGBA")
|
| 521 |
|
|
|
|
| 522 |
debug_image = pil_image.copy()
|
| 523 |
debug_draw = ImageDraw.Draw(debug_image)
|
| 524 |
|
| 525 |
+
# Process each detected paper
|
| 526 |
for i, obj in enumerate(prediction['predictions']):
|
|
|
|
| 527 |
paper_width = obj['width']
|
| 528 |
paper_height = obj['height']
|
| 529 |
|
|
|
|
| 530 |
logging.debug(f"Paper {i+1} dimensions: {paper_width}x{paper_height} at position ({obj['x']}, {obj['y']})")
|
| 531 |
|
|
|
|
| 532 |
padding_x = int(paper_width * 0.20)
|
| 533 |
padding_y = int(paper_height * 0.20)
|
| 534 |
|
|
|
|
| 535 |
box_width = paper_width - 2 * padding_x
|
| 536 |
box_height = paper_height - 2 * padding_y
|
| 537 |
|
|
|
|
| 538 |
x1 = int(obj['x'] - paper_width / 2 + padding_x)
|
| 539 |
y1 = int(obj['y'] - paper_height / 2 + padding_y)
|
| 540 |
x2 = int(obj['x'] + paper_width / 2 - padding_x)
|
| 541 |
y2 = int(obj['y'] + paper_height / 2 - padding_y)
|
| 542 |
|
| 543 |
+
paper_box = [
|
| 544 |
+
(obj['x'] - paper_width/2, obj['y'] - paper_height/2),
|
| 545 |
+
(obj['x'] + paper_width/2, obj['y'] + paper_height/2)
|
| 546 |
+
]
|
| 547 |
debug_draw.rectangle(paper_box, outline=(0, 255, 0, 255), width=3)
|
| 548 |
debug_draw.text(
|
| 549 |
(paper_box[0][0], paper_box[0][1] - 15),
|
|
|
|
| 551 |
fill=(0, 255, 0, 255)
|
| 552 |
)
|
| 553 |
|
|
|
|
| 554 |
usable_box = [(x1, y1), (x2, y2)]
|
| 555 |
debug_draw.rectangle(usable_box, outline=(255, 255, 0, 255), width=2)
|
| 556 |
debug_draw.text(
|
|
|
|
| 559 |
fill=(255, 255, 0, 255)
|
| 560 |
)
|
| 561 |
|
|
|
|
| 562 |
paper_x1 = int(obj['x'] - paper_width / 2)
|
| 563 |
paper_y1 = int(obj['y'] - paper_height / 2)
|
| 564 |
paper_x2 = int(obj['x'] + paper_width / 2)
|
| 565 |
paper_y2 = int(obj['y'] + paper_height / 2)
|
| 566 |
|
|
|
|
| 567 |
angle = detect_paper_angle(
|
| 568 |
np.array(image),
|
| 569 |
(paper_x1, paper_y1, paper_x2, paper_y2)
|
| 570 |
)
|
| 571 |
logging.debug(f"Paper {i+1} angle: {angle} degrees")
|
| 572 |
|
|
|
|
| 573 |
debug_draw.line(
|
| 574 |
[
|
| 575 |
(obj['x'], obj['y']),
|
|
|
|
| 587 |
fill=(255, 0, 0, 255)
|
| 588 |
)
|
| 589 |
|
|
|
|
| 590 |
handwriting_aspect = handwriting_img.width / handwriting_img.height
|
| 591 |
|
|
|
|
| 592 |
target_width = box_width
|
|
|
|
|
|
|
| 593 |
target_width = min(int(target_width * TEXT_SCALE_FACTOR), box_width * 2)
|
|
|
|
|
|
|
| 594 |
target_height = int(target_width / handwriting_aspect)
|
| 595 |
|
|
|
|
| 596 |
if target_height > box_height:
|
| 597 |
target_height = box_height
|
| 598 |
target_width = int(target_height * handwriting_aspect)
|
| 599 |
|
|
|
|
| 600 |
min_width = int(box_width * MIN_WIDTH_PERCENTAGE)
|
| 601 |
if target_width < min_width:
|
| 602 |
target_width = min_width
|
| 603 |
target_height = int(target_width / handwriting_aspect)
|
|
|
|
| 604 |
if target_height > box_height:
|
| 605 |
target_height = box_height
|
| 606 |
target_width = int(target_height * handwriting_aspect)
|
| 607 |
|
|
|
|
| 608 |
logging.debug(
|
| 609 |
f"Paper {i+1} usable area: {box_width}x{box_height}"
|
| 610 |
)
|
|
|
|
| 615 |
f"(scale factor={TEXT_SCALE_FACTOR})"
|
| 616 |
)
|
| 617 |
|
|
|
|
| 618 |
text_center_x = x1 + box_width // 2
|
| 619 |
text_center_y = y1 + box_height // 2
|
| 620 |
text_box = [
|
|
|
|
| 628 |
fill=(255, 0, 255, 255)
|
| 629 |
)
|
| 630 |
|
|
|
|
| 631 |
resized_handwriting = handwriting_img.resize(
|
| 632 |
(target_width, target_height),
|
| 633 |
Image.LANCZOS
|
| 634 |
)
|
| 635 |
|
|
|
|
| 636 |
resized_hw_debug_path = os.path.join(
|
| 637 |
DEBUG_DIR,
|
| 638 |
f"{timestamp}_resized_handwriting_{i+1}.png"
|
| 639 |
)
|
| 640 |
resized_handwriting.save(resized_hw_debug_path)
|
| 641 |
|
|
|
|
| 642 |
handwriting_layer = Image.new("RGBA", pil_image.size, (0, 0, 0, 0))
|
| 643 |
|
|
|
|
| 644 |
paste_x = x1 + (box_width - target_width) // 2
|
| 645 |
paste_y = y1 + (box_height - target_height) // 2
|
| 646 |
|
|
|
|
| 647 |
handwriting_layer.paste(resized_handwriting, (paste_x, paste_y), resized_handwriting)
|
| 648 |
|
|
|
|
| 649 |
debug_paste_box = [
|
| 650 |
(paste_x, paste_y),
|
| 651 |
(paste_x + target_width, paste_y + target_height)
|
| 652 |
]
|
| 653 |
debug_draw.rectangle(debug_paste_box, outline=(0, 0, 255, 255), width=1)
|
| 654 |
|
|
|
|
| 655 |
rotation_debug_path = os.path.join(
|
| 656 |
DEBUG_DIR,
|
| 657 |
f"{timestamp}_rotation_paper_{i+1}.png"
|
|
|
|
| 682 |
)
|
| 683 |
rotation_debug.save(rotation_debug_path)
|
| 684 |
|
|
|
|
| 685 |
rotated_layer = handwriting_layer.rotate(
|
| 686 |
-angle,
|
| 687 |
resample=Image.BICUBIC,
|
| 688 |
center=(obj['x'], obj['y'])
|
| 689 |
)
|
| 690 |
|
|
|
|
| 691 |
pil_image = Image.alpha_composite(pil_image, rotated_layer)
|
| 692 |
|
|
|
|
| 693 |
debug_path = os.path.join(DEBUG_DIR, f"{timestamp}_debug_overlay.png")
|
| 694 |
debug_image.save(debug_path)
|
| 695 |
logging.debug(f"Saved debug overlay image to {debug_path}")
|
| 696 |
|
|
|
|
| 697 |
output_path = "/tmp/output_image.png"
|
| 698 |
pil_image.convert("RGB").save(output_path)
|
| 699 |
|
|
|
|
| 700 |
final_debug_path = os.path.join(DEBUG_DIR, f"{timestamp}_final_output.png")
|
| 701 |
pil_image.save(final_debug_path)
|
| 702 |
|
|
|
|
| 703 |
for dir_path in temp_dirs:
|
| 704 |
try:
|
| 705 |
shutil.rmtree(dir_path)
|
| 706 |
except Exception as e:
|
| 707 |
logging.warning(f"Failed to clean up temporary directory {dir_path}: {str(e)}")
|
| 708 |
|
|
|
|
| 709 |
debug_report = {
|
| 710 |
'timestamp': timestamp,
|
| 711 |
'input_image': input_debug_path,
|
|
|
|
| 734 |
'final_output': final_debug_path
|
| 735 |
}
|
| 736 |
|
|
|
|
| 737 |
logging.debug(f"Debug report: {debug_report}")
|
| 738 |
|
|
|
|
| 739 |
return {
|
| 740 |
'output_path': output_path,
|
| 741 |
'debug_report': debug_report
|
| 742 |
}
|
| 743 |
|
| 744 |
except Exception as e:
|
|
|
|
| 745 |
for dir_path in temp_dirs:
|
| 746 |
try:
|
| 747 |
shutil.rmtree(dir_path)
|
|
|
|
| 751 |
logging.error(f"Error: {str(e)}")
|
| 752 |
raise
|
| 753 |
|
| 754 |
+
# -------------------------------------------------------------------------
|
| 755 |
+
# 🎛 Gradio interface wrapper
|
| 756 |
+
# -------------------------------------------------------------------------
|
| 757 |
def gradio_process(image, text, style, bias, color, stroke_width, text_size):
|
| 758 |
global TEXT_SCALE_FACTOR
|
| 759 |
+
TEXT_SCALE_FACTOR = text_size
|
| 760 |
|
| 761 |
if image is None:
|
| 762 |
return None, None, "Please upload an image with paper."
|
|
|
|
| 772 |
output_path = result['output_path']
|
| 773 |
debug_report = result['debug_report']
|
| 774 |
|
|
|
|
| 775 |
debug_msg = f"Processing complete!\n\n"
|
| 776 |
debug_msg += f"Debug information in: {DEBUG_DIR}\n"
|
| 777 |
|
|
|
|
| 778 |
if 'text_dimensions' in debug_report:
|
| 779 |
td = debug_report['text_dimensions']
|
| 780 |
if 'original' in td:
|
|
|
|
| 786 |
if 'trimmed' in td:
|
| 787 |
debug_msg += f"Trimmed size: {td['trimmed']['width']}x{td['trimmed']['height']} px\n"
|
| 788 |
|
|
|
|
| 789 |
if 'paper_dimensions' in debug_report and len(debug_report['paper_dimensions']) > 0:
|
| 790 |
paper = debug_report['paper_dimensions'][0]
|
| 791 |
debug_msg += f"Detected paper: {paper['width']}x{paper['height']} px\n"
|
|
|
|
| 798 |
logging.exception("Processing error")
|
| 799 |
return None, None, f"Error: {str(e)}"
|
| 800 |
|
| 801 |
+
# -------------------------------------------------------------------------
|
| 802 |
+
# 🚀 Gradio App
|
| 803 |
+
# -------------------------------------------------------------------------
|
| 804 |
interface = gr.Interface(
|
| 805 |
fn=gradio_process,
|
| 806 |
inputs=[
|
|
|
|
| 825 |
)
|
| 826 |
)
|
| 827 |
|
|
|
|
| 828 |
if __name__ == "__main__":
|
| 829 |
interface.launch(share=True)
|