ArsVie commited on
Commit
bb9500d
·
verified ·
1 Parent(s): 42b724a

Create MangaTranslator.py

Browse files
Files changed (1) hide show
  1. MangaTranslator.py +790 -0
MangaTranslator.py ADDED
@@ -0,0 +1,790 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ import cv2
4
+ import numpy as np
5
+ import pyphen
6
+ import re
7
+ import torch
8
+ from PIL import Image, ImageDraw, ImageFont
9
+ import transformers.modeling_utils
10
+ import transformers.utils.import_utils
11
+ from ultralytics import YOLO
12
+ from manga_ocr import MangaOcr
13
+ from transformers import AutoModelForCausalLM, AutoTokenizer
14
+ from simple_lama_inpainting import SimpleLama
15
+ import PIL.Image
16
+
17
+ class MangaTranslator:
18
+ def __init__(self, yolo_model_path='comic_yolov8m.pt',
19
+ translation_model="LiquidAI/LFM2-350M-ENJP-MT",
20
+ font_path="font.ttf", custom_translations=None, keep_honorifics=True, debug=True):
21
+
22
+ print("Loading YOLO model...")
23
+ self.yolo_model = YOLO(yolo_model_path)
24
+ self.font_path = font_path
25
+
26
+ print("Loading LaMa Inpainting model...")
27
+ self.lama = SimpleLama()
28
+
29
+ print("Loading MangaOCR model...")
30
+ self.mocr = MangaOcr()
31
+
32
+ # --- LIQUID AI SETUP (Updated) ---
33
+ print(f"Loading Translation Model ({translation_model})...")
34
+ self.device = "cuda" if torch.cuda.is_available() else "cpu"
35
+
36
+ # 1. Load Tokenizer
37
+ self.tokenizer = AutoTokenizer.from_pretrained(
38
+ translation_model,
39
+ trust_remote_code=True # Required for Liquid architectures
40
+ )
41
+
42
+ # 2. Load Model with Trust Remote Code
43
+ self.trans_model = AutoModelForCausalLM.from_pretrained(
44
+ translation_model,
45
+ torch_dtype=torch.float16 if self.device == "cuda" else torch.float32,
46
+ device_map=self.device,
47
+ trust_remote_code=True # Required for Liquid architectures
48
+ )
49
+ self.trans_model.eval()
50
+ # -----------------------
51
+
52
+ self.dic = pyphen.Pyphen(lang='en')
53
+ self.font_cache = {}
54
+ self.custom_translations = custom_translations or {}
55
+ self.keep_honorifics = keep_honorifics
56
+ self.honorifics = ['san', 'chan', 'kun', 'sama', 'senpai', 'sensei', 'dono', 'tan']
57
+
58
+ # For romanization fallback
59
+ try:
60
+ import pykakasi
61
+ self.kakasi = pykakasi.kakasi()
62
+ except ImportError:
63
+ print("Warning: pykakasi not installed. Install with 'pip install pykakasi' for romanization support.")
64
+ self.kakasi = None
65
+
66
+ def _get_font(self, size):
67
+ """Cache fonts to avoid repeated loading"""
68
+ if size not in self.font_cache:
69
+ try:
70
+ self.font_cache[size] = ImageFont.truetype(self.font_path, size)
71
+ except IOError:
72
+ self.font_cache[size] = ImageFont.load_default()
73
+ return self.font_cache[size]
74
+
75
+ def _sort_bubbles(self, bubbles, row_threshold=50):
76
+ bubbles.sort(key=lambda b: b[1])
77
+ sorted_bubbles = []
78
+ if not bubbles:
79
+ return sorted_bubbles
80
+
81
+ current_row = [bubbles[0]]
82
+ for i in range(1, len(bubbles)):
83
+ if abs(bubbles[i][1] - current_row[-1][1]) < row_threshold:
84
+ current_row.append(bubbles[i])
85
+ else:
86
+ current_row.sort(key=lambda b: b[2], reverse=True)
87
+ sorted_bubbles.extend(current_row)
88
+ current_row = [bubbles[i]]
89
+
90
+ current_row.sort(key=lambda b: b[2], reverse=True)
91
+ sorted_bubbles.extend(current_row)
92
+ return sorted_bubbles
93
+
94
+ def _wrap_text_dynamic(self, text, font, max_width):
95
+ words = text.split()
96
+ lines = []
97
+ current_line = []
98
+ current_width = 0
99
+ space_width = font.getlength(" ")
100
+
101
+ for word in words:
102
+ word_width = font.getlength(word)
103
+ potential_width = current_width + word_width + (space_width if current_line else 0)
104
+
105
+ if potential_width <= max_width:
106
+ current_line.append(word)
107
+ current_width = potential_width
108
+ else:
109
+ splits = list(self.dic.iterate(word))
110
+ found_split = False
111
+ for start, end in reversed(splits):
112
+ chunk = start + "-"
113
+ chunk_width = font.getlength(chunk)
114
+ if current_width + chunk_width + (space_width if current_line else 0) <= max_width:
115
+ current_line.append(chunk)
116
+ lines.append(" ".join(current_line))
117
+ current_line = [end]
118
+ current_width = font.getlength(end)
119
+ found_split = True
120
+ break
121
+
122
+ if not found_split:
123
+ if current_line:
124
+ lines.append(" ".join(current_line))
125
+ current_line = [word]
126
+ current_width = word_width
127
+
128
+ if current_line:
129
+ lines.append(" ".join(current_line))
130
+ return "\n".join(lines)
131
+
132
+ def _smart_clean_bubble(self, img, bbox):
133
+ """
134
+ Gaussian blur-based cleaning for transparent effect
135
+ """
136
+ x1, y1, x2, y2 = bbox
137
+
138
+ # Ensure coordinates are within image bounds
139
+ h, w = img.shape[:2]
140
+ x1, y1 = max(0, x1), max(0, y1)
141
+ x2, y2 = min(w, x2), min(h, y2)
142
+
143
+ if x2 <= x1 or y2 <= y1:
144
+ return img
145
+
146
+ # Extract bubble region
147
+ bubble_region = img[y1:y2, x1:x2].copy()
148
+
149
+ if bubble_region.size == 0:
150
+ return img
151
+
152
+ # Apply Gaussian blur for softer look
153
+ blurred = cv2.GaussianBlur(bubble_region, (21, 21), 0)
154
+
155
+ # Brighten the blurred region slightly
156
+ brightened = cv2.addWeighted(blurred, 0.7,
157
+ np.ones_like(blurred) * 255, 0.3, 0)
158
+
159
+ # Place back into image
160
+ img[y1:y2, x1:x2] = brightened
161
+
162
+ return img
163
+
164
+ def _preserve_honorifics(self, original_text, translated_text):
165
+ """
166
+ Detect and preserve Japanese honorifics in romaji form.
167
+ Examples: さん→-san, ちゃん→-chan, 君→-kun, 様→-sama
168
+ """
169
+ if not self.keep_honorifics or not self.kakasi:
170
+ return translated_text
171
+
172
+ # Common honorific patterns in Japanese
173
+ honorific_map = {
174
+ 'さん': '-san',
175
+ 'ちゃん': '-chan',
176
+ 'くん': '-kun',
177
+ '君': '-kun',
178
+ '様': '-sama',
179
+ 'さま': '-sama',
180
+ '先輩': '-senpai',
181
+ 'せんぱい': '-senpai',
182
+ '先生': '-sensei',
183
+ 'せんせい': '-sensei',
184
+ '殿': '-dono',
185
+ 'どの': '-dono',
186
+ 'たん': '-tan',
187
+ }
188
+
189
+ # Find honorifics in original text
190
+ found_honorifics = []
191
+ for jp_hon, rom_hon in honorific_map.items():
192
+ if jp_hon in original_text:
193
+ found_honorifics.append(rom_hon)
194
+
195
+ # If we found honorifics, try to add them back to names in translation
196
+ if found_honorifics:
197
+ # Split into words and check last word for potential name
198
+ words = translated_text.split()
199
+ if len(words) >= 1:
200
+ # Check if translation already has honorific
201
+ last_word = words[-1].lower()
202
+ has_honorific = any(hon.strip('-') in last_word for hon in self.honorifics)
203
+
204
+ if not has_honorific and found_honorifics:
205
+ # Add the first found honorific to what's likely a name
206
+ # Look for capitalized words (likely names)
207
+ for i in range(len(words) - 1, -1, -1):
208
+ if words[i] and words[i][0].isupper():
209
+ # Add honorific to this name
210
+ words[i] = words[i] + found_honorifics[0]
211
+ translated_text = ' '.join(words)
212
+ break
213
+
214
+ return translated_text
215
+
216
+ def _draw_text_with_outline(self, draw, position, text, font,
217
+ text_color="black", outline_color="white",
218
+ outline_width=2, **kwargs):
219
+ """
220
+ Draw text with outline for better readability
221
+ """
222
+ x, y = position
223
+ # Draw outline
224
+ for adj_x in range(-outline_width, outline_width + 1):
225
+ for adj_y in range(-outline_width, outline_width + 1):
226
+ if adj_x != 0 or adj_y != 0:
227
+ draw.multiline_text((x + adj_x, y + adj_y), text,
228
+ fill=outline_color, font=font, **kwargs)
229
+ # Draw main text
230
+ draw.multiline_text(position, text, fill=text_color, font=font, **kwargs)
231
+
232
+ def _calculate_optimal_font_size(self, text, bbox, min_size=12, max_size=36):
233
+ x1, y1, x2, y2 = bbox
234
+ box_width = x2 - x1
235
+ box_height = y2 - y1
236
+
237
+ # --- NEW LOGIC: DETECT VERTICAL BUBBLES ---
238
+ # If height is 1.5x bigger than width, it's a vertical speech bubble.
239
+ is_vertical = box_height > (box_width * 1.5)
240
+
241
+ # If vertical, force text to use only 60% of width (makes a column)
242
+ # If horizontal, use 90% of width (standard)
243
+ target_width_ratio = 0.6 if is_vertical else 0.9
244
+
245
+ # Start with max size and reduce until text fits
246
+ for size in range(max_size, min_size - 1, -1):
247
+ font = self._get_font(size)
248
+
249
+ # Use the calculated target width
250
+ max_line_width = int(box_width * target_width_ratio)
251
+ wrapped = self._wrap_text_dynamic(text, font, max_line_width)
252
+
253
+ # Measure resulting text block
254
+ temp_draw = ImageDraw.Draw(Image.new('RGB', (1, 1)))
255
+ left, top, right, bottom = temp_draw.multiline_textbbox(
256
+ (0, 0), wrapped, font=font, align="center"
257
+ )
258
+ text_width = right - left
259
+ text_height = bottom - top
260
+
261
+ # Check fit (Height is the main constraint)
262
+ if text_height < (box_height - 10):
263
+ # Secondary check: If vertical, ensure we didn't accidentally
264
+ # make it too wide (overflowing the sides)
265
+ if text_width < (box_width - 4):
266
+ return size, wrapped
267
+
268
+ # Fallback: Minimum size
269
+ font = self._get_font(min_size)
270
+ max_line_width = int(box_width * target_width_ratio)
271
+ wrapped = self._wrap_text_dynamic(text, font, max_line_width)
272
+ return min_size, wrapped
273
+
274
+ def _has_japanese_characters(self, text):
275
+ """Check if text contains Japanese characters"""
276
+ japanese_ranges = [
277
+ (0x3040, 0x309F), # Hiragana
278
+ (0x30A0, 0x30FF), # Katakana
279
+ (0x4E00, 0x9FFF), # Kanji
280
+ ]
281
+ for char in text:
282
+ code = ord(char)
283
+ for start, end in japanese_ranges:
284
+ if start <= code <= end:
285
+ return True
286
+ return False
287
+
288
+ def _romanize_japanese(self, text):
289
+ """Convert Japanese text to romaji"""
290
+ if not self.kakasi:
291
+ return text
292
+
293
+ try:
294
+ result = self.kakasi.convert(text)
295
+ return ''.join([item['hepburn'] for item in result])
296
+ except Exception as e:
297
+ print(f" Romanization error: {e}")
298
+ return text
299
+
300
+ def _apply_custom_translations(self, text):
301
+ """Apply custom character name translations"""
302
+ for jp_term, en_term in self.custom_translations.items():
303
+ text = text.replace(jp_term, en_term)
304
+ return text
305
+
306
+ def detect_and_process(self, image_path, output_dir="crops", page_id="", conf_threshold=0.15):
307
+ image = cv2.imread(image_path)
308
+ if image is None: raise ValueError(f"Not found: {image_path}")
309
+
310
+ # 1. Run Prediction
311
+ results = self.yolo_model.predict(source=image, conf=conf_threshold, save=False, verbose=False)
312
+
313
+ # Get the class names dictionary (e.g., {0: 'text', 1: 'bubble'})
314
+ class_names = results[0].names
315
+
316
+ # 2. Extract Boxes AND Classes
317
+ detections = []
318
+ for box in results[0].boxes:
319
+ xyxy = list(map(int, box.xyxy[0].tolist()))
320
+ cls_id = int(box.cls[0])
321
+ label = class_names[cls_id] # e.g., "text" or "bubble" or "face"
322
+
323
+ # Filter: We only care about text/bubbles, not faces/bodies if your model detects them
324
+ if label in ['face', 'body']: continue
325
+
326
+ detections.append({
327
+ "bbox": xyxy,
328
+ "label": label
329
+ })
330
+
331
+ # Sort (top to bottom, right to left for manga)
332
+ # Note: We need a custom sort function since detections is now a dict, not just a list of boxes
333
+ detections = sorted(detections, key=lambda x: (x['bbox'][1], -x['bbox'][0]))
334
+
335
+ if not os.path.exists(output_dir): os.makedirs(output_dir)
336
+
337
+ manga_data = []
338
+ for i, det in enumerate(detections):
339
+ x_min, y_min, x_max, y_max = det['bbox']
340
+
341
+ # ... (Cropping logic stays the same) ...
342
+ crop = image[y_min:y_max, x_min:x_max]
343
+
344
+ # Save crop
345
+ crop_filename = f"bubble_{page_id}_{i+1}.png"
346
+ crop_path = os.path.join(output_dir, crop_filename)
347
+ cv2.imwrite(crop_path, crop)
348
+
349
+ manga_data.append({
350
+ "id": f"{page_id}_{i+1}",
351
+ "page_id": page_id,
352
+ "bbox": [x_min, y_min, x_max, y_max],
353
+ "label": det['label'],
354
+ "crop_path": crop_path,
355
+ "original_text": "",
356
+ "translated_text": ""
357
+ })
358
+
359
+ return image, manga_data
360
+
361
+ def run_ocr(self, manga_data):
362
+ for entry in manga_data:
363
+ crop_path = entry['crop_path']
364
+ japanese_text = self.mocr(crop_path)
365
+
366
+ # Apply custom translations to original text
367
+ japanese_text = self._apply_custom_translations(japanese_text)
368
+
369
+ entry['original_text'] = japanese_text.replace('\n', '')
370
+ return manga_data
371
+
372
+ def _translate_single_bubble(self, text, series_info=None):
373
+ """Translate a single bubble (fallback method)"""
374
+ context_str = ""
375
+ if series_info:
376
+ context_str = f"""
377
+ Context: {series_info.get('title', '')} - {series_info.get('tags', '')}
378
+ """
379
+
380
+ prompt = f"""{context_str}Translate this Japanese manga text to natural English. Return ONLY the English translation, nothing else:
381
+ {text}"""
382
+
383
+ try:
384
+ response = self.llm.invoke(prompt)
385
+ translation = response.content.strip()
386
+
387
+ # Remove common wrapper phrases
388
+ translation = re.sub(r'^(Here\'s the translation:|Translation:|English:)\s*', '', translation, flags=re.IGNORECASE)
389
+ translation = translation.strip('"\'')
390
+
391
+ return translation
392
+ except Exception as e:
393
+ print(f" Translation error: {e}")
394
+ return "[Translation Error]"
395
+
396
+ def translate_batch(self, manga_data, series_info=None):
397
+ """
398
+ Minimalist translation loop for LiquidAI LFM2-350M.
399
+ REMOVED: Context injection (to prevent hallucinations).
400
+ INCLUDED: Fix for Dictionary vs Tensor inputs.
401
+ """
402
+ print(f"Translating {len(manga_data)} bubbles with LiquidAI...")
403
+
404
+ # Strict System Prompt (Required by Model Card)
405
+ system_prompt = "Translate to English."
406
+
407
+ for entry in manga_data:
408
+ text = entry.get('original_text', '').strip()
409
+ if not text: continue
410
+
411
+ # Skip punctuation-only bubbles
412
+ if len(text) < 2 and text in "!?.…":
413
+ entry['translated_text'] = text
414
+ continue
415
+
416
+ # --- NO CONTEXT, JUST TEXT ---
417
+ messages = [
418
+ {"role": "system", "content": system_prompt},
419
+ {"role": "user", "content": text} # Raw text only
420
+ ]
421
+
422
+ # 1. Apply Template
423
+ inputs = self.tokenizer.apply_chat_template(
424
+ messages,
425
+ add_generation_prompt=True,
426
+ return_tensors="pt"
427
+ )
428
+
429
+ # 2. Handle Dict vs Tensor (LiquidAI Quirks)
430
+ if isinstance(inputs, dict) or hasattr(inputs, "keys"):
431
+ inputs = inputs.to(self.device)
432
+ generate_kwargs = inputs
433
+ input_length = inputs["input_ids"].shape[1]
434
+ else:
435
+ inputs = inputs.to(self.device)
436
+ generate_kwargs = {"input_ids": inputs}
437
+ input_length = inputs.shape[1]
438
+
439
+ # 3. Generate
440
+ with torch.no_grad():
441
+ output_ids = self.trans_model.generate(
442
+ **generate_kwargs,
443
+ max_new_tokens=128,
444
+ temperature=0.5,
445
+ top_p=1.0,
446
+ repetition_penalty=1.05,
447
+ do_sample=True
448
+ )
449
+
450
+ # 4. Decode
451
+ translated_text = self.tokenizer.decode(
452
+ output_ids[0][input_length:],
453
+ skip_special_tokens=True
454
+ ).strip()
455
+
456
+ entry['translated_text'] = translated_text
457
+ print(f" JP: {text[:15]}... -> EN: {translated_text}")
458
+
459
+ return manga_data
460
+
461
+
462
+
463
+ def clean_page(self, original_image, page_data, ellipse_padding=8, inpaint_radius=5):
464
+ """
465
+ Strict Hybrid Cleaning:
466
+ - text_bubble -> OpenCV Inpainting inside a shrunk Ellipse mask (Preserves tails)
467
+ - text_free -> LaMa Inpainting on full Rectangle mask (Redraws background)
468
+ """
469
+ final_image = original_image.copy()
470
+ h, w = original_image.shape[:2]
471
+
472
+ # Mask for LaMa (Accumulates all 'text_free' areas)
473
+ lama_mask = np.zeros((h, w), dtype=np.uint8)
474
+ has_lama_work = False
475
+
476
+ for entry in page_data:
477
+ # Skip if no translation (optional, but good for speed)
478
+ if not entry.get('translated_text'): continue
479
+
480
+ bbox = entry['bbox']
481
+ label = entry.get('label', 'text_free')
482
+
483
+ x1, y1, x2, y2 = bbox
484
+
485
+ # Clamp coordinates
486
+ x1, y1 = max(0, x1), max(0, y1)
487
+ x2, y2 = min(w, x2), min(h, y2)
488
+
489
+ # Extract crop for analysis
490
+ crop = final_image[y1:y2, x1:x2]
491
+ if crop.size == 0: continue
492
+
493
+ gray_crop = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
494
+
495
+ # --- STRATEGY 1: SPEECH BUBBLES (OpenCV + Shrunk Ellipse) ---
496
+ if label == 'text_bubble':
497
+ ch, cw = crop.shape[:2]
498
+
499
+ # A. Find the text pixels (dark ink)
500
+ binary_text = cv2.adaptiveThreshold(
501
+ gray_crop, 255, cv2.ADAPTIVE_THRESH_MEAN_C,
502
+ cv2.THRESH_BINARY_INV, 21, 10
503
+ )
504
+
505
+ # B. Create SHRUNK Ellipse Mask
506
+ ellipse_mask = np.zeros((ch, cw), dtype=np.uint8)
507
+ center = (cw // 2, ch // 2)
508
+ # Shrink axes by padding to avoid touching bubble borders
509
+ axes = (max(1, cw // 2 - ellipse_padding), max(1, ch // 2 - ellipse_padding))
510
+ cv2.ellipse(ellipse_mask, center, axes, 0, 0, 360, 255, -1)
511
+
512
+ # C. Combine: Mask ONLY text that is INSIDE the ellipse
513
+ final_mask = cv2.bitwise_and(binary_text, ellipse_mask)
514
+
515
+ # D. Dilate to catch anti-aliasing
516
+ kernel = np.ones((5,5), np.uint8)
517
+ final_mask = cv2.dilate(final_mask, kernel, iterations=1)
518
+
519
+ # E. Run OpenCV Inpainting
520
+ cleaned_crop = cv2.inpaint(crop, final_mask, inpaint_radius, cv2.INPAINT_TELEA)
521
+
522
+ # Paste back
523
+ final_image[y1:y2, x1:x2] = cleaned_crop
524
+
525
+ # --- STRATEGY 2: FREE TEXT (LaMa + Rectangle) ---
526
+ elif label == 'text_free':
527
+ cv2.rectangle(lama_mask, (x1, y1), (x2, y2), 255, -1)
528
+ has_lama_work = True
529
+
530
+ # Run LaMa batch for all free text found
531
+ if has_lama_work:
532
+ # Dilate LaMa mask slightly
533
+ lama_kernel = np.ones((5, 5), np.uint8)
534
+ lama_mask = cv2.dilate(lama_mask, lama_kernel, iterations=1)
535
+
536
+ img_pil = Image.fromarray(cv2.cvtColor(final_image, cv2.COLOR_BGR2RGB))
537
+ mask_pil = Image.fromarray(lama_mask)
538
+
539
+ try:
540
+ # 1. Run Model
541
+ cleaned_pil = self.lama(img_pil, mask_pil)
542
+ cleaned_lama = cv2.cvtColor(np.array(cleaned_pil), cv2.COLOR_RGB2BGR)
543
+
544
+ # 2. Resize fix (LaMa padding issue)
545
+ if cleaned_lama.shape[:2] != (h, w):
546
+ cleaned_lama = cv2.resize(cleaned_lama, (w, h))
547
+
548
+ # 3. Merge LaMa result
549
+ final_image = np.where(lama_mask[:, :, None] == 255, cleaned_lama, final_image)
550
+
551
+ except Exception as e:
552
+ print(f" ⚠ LaMa failed: {e}")
553
+
554
+ return final_image
555
+
556
+ def typeset(self, original_image, manga_data, output_path):
557
+ working_img = self.clean_page(original_image, manga_data)
558
+ # 2. Text Drawing with adaptive sizing and outlines
559
+ img_pil = Image.fromarray(cv2.cvtColor(working_img, cv2.COLOR_BGR2RGB))
560
+ draw = ImageDraw.Draw(img_pil)
561
+
562
+ for entry in manga_data:
563
+ x1, y1, x2, y2 = entry['bbox']
564
+ text = entry.get('translated_text', '')
565
+ if not text: continue
566
+
567
+ # Calculate optimal font size for this bubble
568
+ font_size, wrapped_text = self._calculate_optimal_font_size(
569
+ text, entry['bbox']
570
+ )
571
+
572
+ font = self._get_font(font_size)
573
+
574
+ # Get text dimensions
575
+ left, top, right, bottom = draw.multiline_textbbox(
576
+ (0, 0), wrapped_text, font=font, align="center"
577
+ )
578
+ text_w, text_h = right - left, bottom - top
579
+
580
+ # Center text
581
+ text_x = x1 + ((x2 - x1) - text_w) / 2
582
+ text_y = y1 + ((y2 - y1) - text_h) / 2
583
+
584
+ # Draw with outline for readability
585
+ self._draw_text_with_outline(
586
+ draw, (text_x, text_y), wrapped_text, font,
587
+ text_color="black", outline_color="white",
588
+ outline_width=2, align="center", spacing=2
589
+ )
590
+
591
+ final_img = cv2.cvtColor(np.array(img_pil), cv2.COLOR_RGB2BGR)
592
+ cv2.imwrite(output_path, final_img)
593
+ print(f" Saved: {output_path}")
594
+
595
+ def process_chapter(self, input_folder, output_folder, series_info=None,
596
+ batch_size=4, selected_batches=None):
597
+ """
598
+ Process manga chapter in batches for better context and efficiency
599
+ """
600
+ if not os.path.exists(output_folder):
601
+ os.makedirs(output_folder)
602
+
603
+ valid_ext = ('.png', '.jpg', '.jpeg', '.webp', '.bmp')
604
+ files = [f for f in os.listdir(input_folder) if f.lower().endswith(valid_ext)]
605
+ # Sort numerically (p1, p2, p10 instead of p1, p10, p2)
606
+ files.sort(key=lambda x: int(re.search(r'\d+', x).group()) if re.search(r'\d+', x) else x)
607
+
608
+ total_files = len(files)
609
+ total_batches = (total_files + batch_size - 1) // batch_size
610
+
611
+ # Master list to hold data for the entire chapter
612
+ full_chapter_data = []
613
+
614
+ print(f"Found {total_files} images in {input_folder}")
615
+ print(f"Total batches: {total_batches} (batch size: {batch_size})")
616
+
617
+ if selected_batches:
618
+ print(f"Processing selected batches: {selected_batches}")
619
+ else:
620
+ print(f"Processing all batches\n")
621
+
622
+ # Process in batches
623
+ for batch_start in range(0, total_files, batch_size):
624
+ batch_num = batch_start // batch_size + 1
625
+
626
+ # Skip if not in selected batches
627
+ if selected_batches and batch_num not in selected_batches:
628
+ continue
629
+
630
+ batch_files = files[batch_start:batch_start + batch_size]
631
+ print(f"=== Batch {batch_num}/{total_batches} ({len(batch_files)} pages) ===")
632
+
633
+ # Collect all data for this batch
634
+ batch_data = []
635
+ batch_images = []
636
+
637
+ temp_crop_dir = os.path.join(output_folder, "temp_crops")
638
+
639
+ for idx, filename in enumerate(batch_files):
640
+ page_num = batch_start + idx + 1
641
+ print(f" [{page_num}/{total_files}] Detecting bubbles in {filename}...")
642
+
643
+ input_path = os.path.join(input_folder, filename)
644
+ page_id = f"p{page_num:03d}"
645
+
646
+ try:
647
+ img, data = self.detect_and_process(input_path, output_dir=temp_crop_dir, page_id=page_id)
648
+
649
+ if data:
650
+ print(f" Running OCR on {len(data)} bubbles...")
651
+ data = self.run_ocr(data)
652
+ batch_data.extend(data)
653
+ else:
654
+ print(f" No bubbles detected")
655
+
656
+ batch_images.append((filename, img, page_id))
657
+
658
+ except Exception as e:
659
+ print(f" Error processing {filename}: {e}")
660
+ continue
661
+
662
+ # Translate entire batch at once for context
663
+ if batch_data:
664
+ print(f" Translating {len(batch_data)} bubbles from batch...")
665
+ batch_data = self.translate_batch(batch_data, series_info=series_info)
666
+
667
+ # Add this batch's completed data to the master list
668
+ full_chapter_data.extend(batch_data)
669
+
670
+ # Typeset each page
671
+ print(f" Typesetting pages...")
672
+ for filename, img, page_id in batch_images:
673
+ output_path = os.path.join(output_folder, filename)
674
+
675
+ # Filter data for this specific page
676
+ page_data = [d for d in batch_data if d.get('page_id') == page_id]
677
+
678
+ try:
679
+ self.typeset(img, page_data, output_path)
680
+ except Exception as e:
681
+ print(f" Error typesetting {filename}: {e}")
682
+
683
+ print() # Empty line between batches
684
+
685
+ # --- NEW LOGIC: Save JSON if debug is ON ---
686
+ if self.debug and full_chapter_data:
687
+ json_filename = f"chapter_data.json"
688
+ json_path = os.path.join(output_folder, json_filename)
689
+
690
+ try:
691
+ with open(json_path, 'w', encoding='utf-8') as f:
692
+ json.dump(full_chapter_data, f, ensure_ascii=False, indent=2)
693
+ print(f" [DEBUG] Saved full chapter data to: {json_filename}")
694
+ except Exception as e:
695
+ print(f" [DEBUG] Failed to save JSON: {e}")
696
+
697
+ print(f"\n✓ Chapter processing complete! Output saved to: {output_folder}")
698
+
699
+ def process_single_image(self, image_path, output_path, series_info=None):
700
+ """
701
+ Runs the full pipeline on a SINGLE image file.
702
+ Perfect for demos or testing one page.
703
+ """
704
+ if not os.path.exists(image_path):
705
+ raise FileNotFoundError(f"Image not found: {image_path}")
706
+
707
+ print(f"=== Processing Single Page: {os.path.basename(image_path)} ===")
708
+
709
+ # 1. Setup a temp folder for the bubble crops (required for OCR)
710
+ # We use a fixed folder name for the demo to keep it clean
711
+ temp_crop_dir = "temp_demo_crops"
712
+ if not os.path.exists(temp_crop_dir):
713
+ os.makedirs(temp_crop_dir)
714
+
715
+ # 2. DETECT
716
+ # We use a generic ID 'demo' since we don't have page numbers
717
+ print("1. Detecting Bubbles...")
718
+ original_img, data = self.detect_and_process(
719
+ image_path,
720
+ output_dir=temp_crop_dir,
721
+ page_id="demo"
722
+ )
723
+
724
+ if not data:
725
+ print(" ⚠ No bubbles found! Saving original image...")
726
+ cv2.imwrite(output_path, original_img)
727
+ return
728
+
729
+ # 3. OCR
730
+ print(f"2. Running OCR on {len(data)} bubbles...")
731
+ data = self.run_ocr(data)
732
+
733
+ # 4. TRANSLATE
734
+ print("3. Translating text...")
735
+ # We reuse translate_batch because it handles the logic perfectly,
736
+ # even if the "batch" is just bubbles from one page.
737
+ data = self.translate_batch(data, series_info=series_info)
738
+
739
+ # 5. TYPESET (Clean + Draw)
740
+ print("4. Typesetting (Cleaning & Drawing)...")
741
+ # Ensure output directory exists
742
+ out_dir = os.path.dirname(output_path)
743
+ if out_dir and not os.path.exists(out_dir):
744
+ os.makedirs(out_dir)
745
+
746
+ self.typeset(original_img, data, output_path)
747
+
748
+ print(f"✅ Success! Saved to: {output_path}")
749
+
750
+ # Optional: Return the data if you want to inspect JSON in the demo
751
+ return data
752
+
753
+ if __name__ == "__main__":
754
+ # 1. Define Translation Dictionary (Optional but good for names)
755
+ custom_translations = {
756
+ "ルーグ": "Lugh",
757
+ "トウアハーデ": "Tuatha Dé",
758
+ "ディア": "Dia",
759
+ "タルト": "Tarte",
760
+ }
761
+
762
+ # 2. Initialize the Class
763
+ # Note: We removed 'ollama_model' and added 'translation_model'
764
+ translator = MangaTranslator(
765
+ yolo_model_path='comic-speech-bubble-detector.pt',
766
+ translation_model="LiquidAI/LFM2-350M-ENJP-MT",
767
+ font_path="font.ttf",
768
+ custom_translations=custom_translations,
769
+ debug=True # Keeps the JSON file for debugging
770
+ )
771
+
772
+ # 3. Define Context (Important for tone, even with small models)
773
+
774
+ # 4. Run the Single Page Demo
775
+ # Ensure you have 'raw_images/001.jpg' inside your project folder
776
+ input_file = "chapter_401/001.jpg"
777
+ output_file = "output/001_translated.jpg"
778
+
779
+ if os.path.exists(input_file):
780
+ print(f"🚀 Starting Demo on {input_file}...")
781
+
782
+ translator.process_single_image(
783
+ image_path=input_file,
784
+ output_path=output_file,
785
+ series_info=None
786
+ )
787
+
788
+ print(f"✨ Demo Complete! Check {output_file}")
789
+ else:
790
+ print(f"❌ Error: Could not find {input_file}. Please check your folder structure.")