File size: 27,940 Bytes
92b19da
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
"""

Constrained Recipe-Based Synthetic Handwritten Line Generation



Generates synthetic text lines by concatenating real handwritten word images

with guaranteed text uniqueness, single-writer consistency, and leakage-free

data partitioning.



Usage:

    python synthetic_line_generator.py \

        --unique_words_dir ./data/Unique-Words \

        --person_names_dir ./data/Person-Names \

        --output_dir ./data/Synthetic-Lines \

        --training_writers ./writers/Training.txt \

        --validation_writers ./writers/Validation.txt \

        --testing_writers ./writers/Testing.txt

"""

import os
import glob
import random
import argparse
import re
import numpy as np
from collections import defaultdict
from datetime import datetime
from PIL import Image, TiffImagePlugin

# =============================================================================
# ARGUMENT PARSER
# =============================================================================

def parse_args():
    parser = argparse.ArgumentParser(
        description="Constrained Recipe-Based Synthetic Handwritten Line Generation")

    # Data paths
    parser.add_argument("--unique_words_dir", type=str, required=True,
                        help="Root directory of unique word samples (with Training/Validation/Testing subfolders)")
    parser.add_argument("--person_names_dir", type=str, required=True,
                        help="Root directory of person name samples (with Training/Validation/Testing subfolders)")
    parser.add_argument("--output_dir", type=str, required=True,
                        help="Output directory for generated synthetic lines")

    # Writer files per subset
    parser.add_argument("--training_writers", type=str, default=None,
                        help="Text file listing training writer IDs (one per line)")
    parser.add_argument("--validation_writers", type=str, default=None,
                        help="Text file listing validation writer IDs (one per line)")
    parser.add_argument("--testing_writers", type=str, default=None,
                        help="Text file listing testing writer IDs (one per line)")

    # Canvas and composition parameters
    parser.add_argument("--img_height", type=int, default=155)
    parser.add_argument("--img_width", type=int, default=2470)
    parser.add_argument("--baseline_ratio", type=float, default=0.75)
    parser.add_argument("--text_height_ratio", type=float, default=0.88)
    parser.add_argument("--spacing_min", type=int, default=10)
    parser.add_argument("--spacing_max", type=int, default=30)
    parser.add_argument("--baseline_jitter", type=int, default=1)
    parser.add_argument("--left_margin", type=int, default=8)
    parser.add_argument("--right_margin", type=int, default=8)

    # Grouping parameters
    parser.add_argument("--unique_group_size", type=int, default=20,
                        help="Number of writers sharing the same unique words")
    parser.add_argument("--person_group_size", type=int, default=5,
                        help="Number of writers sharing the same person names")

    # Generation parameters
    parser.add_argument("--max_groups", type=int, default=None,
                        help="Process only first N groups (for testing)")
    parser.add_argument("--seed", type=int, default=42)

    return parser.parse_args()

# =============================================================================
# CONFIGURATION (set from args)
# =============================================================================

class Config:
    """Holds all generation parameters"""
    def __init__(self, args):
        self.img_height = args.img_height
        self.img_width = args.img_width
        self.baseline_ratio = args.baseline_ratio
        self.text_height_ratio = args.text_height_ratio
        self.spacing_range = (args.spacing_min, args.spacing_max)
        self.baseline_jitter = args.baseline_jitter
        self.left_margin = args.left_margin
        self.right_margin = args.right_margin
        self.unique_group_size = args.unique_group_size
        self.person_group_size = args.person_group_size
        self.max_words_for_scaling = 8
        self.source_subsets = ["Training", "Validation", "Testing"]

TiffImagePlugin.WRITE_LIBTIFF = True

# =============================================================================
# LOGGER
# =============================================================================

class Logger:
    def __init__(self, log_path):
        os.makedirs(os.path.dirname(log_path) if os.path.dirname(log_path) else ".", exist_ok=True)
        self.f = open(log_path, "w", encoding="utf-8")
        self.f.write(f"{'=' * 80}\nSYNTHETIC LINE GENERATION LOG\n"
                     f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n{'=' * 80}\n\n")
        self.f.flush()

    def log(self, message, console=True):
        self.f.write(message + "\n")
        self.f.flush()
        if console:
            print(message)

    def section(self, title):
        self.log(f"\n{'=' * 60}\n{title}\n{'=' * 60}")

    def subsection(self, title):
        self.log(f"\n{'-' * 40}\n{title}\n{'-' * 40}")

    def log_line_detail(self, filename, writer_id, recipe_type, subgroup, words_info, text):
        self.log(f"  {filename}.tif", console=False)
        self.log(f"    Writer: DNDK{writer_id:05d}", console=False)
        self.log(f"    Type: {recipe_type}"
                 + (f"  (sub-group {subgroup[0]}-{subgroup[1]})" if subgroup else ""), console=False)
        self.log(f"    Words: {words_info}", console=False)
        self.log(f"    Text: {text}", console=False)

    def close(self):
        self.f.close()

# =============================================================================
# TEXT NORMALIZATION
# =============================================================================

def normalize_label(text):
    if text is None:
        return ""
    text = text.replace("\u00A0", " ").replace("\r", " ").replace("\n", " ")
    return " ".join(text.strip().split())

# =============================================================================
# OTSU THRESHOLD
# =============================================================================

def otsu_threshold(gray_uint8):
    hist = np.bincount(gray_uint8.ravel(), minlength=256).astype(np.float64)
    total = gray_uint8.size
    sum_total = np.dot(np.arange(256), hist)
    sum_b, w_b, max_var, threshold = 0.0, 0.0, 0.0, 127
    for t in range(256):
        w_b += hist[t]
        if w_b == 0:
            continue
        w_f = total - w_b
        if w_f == 0:
            break
        sum_b += t * hist[t]
        m_b = sum_b / w_b
        m_f = (sum_total - sum_b) / w_f
        var_between = w_b * w_f * (m_b - m_f) ** 2
        if var_between > max_var:
            max_var = var_between
            threshold = t
    return threshold

# =============================================================================
# INK EXTRACTION WITH DIACRITICAL PRESERVATION
# =============================================================================

def build_word_cutout_with_baseline(img_pil):
    """Extract ink region with adaptive Otsu threshold (+20 for diacritical preservation)"""
    img_rgb = img_pil.convert("RGB")
    gray = np.array(img_rgb.convert("L"))
    thr = otsu_threshold(gray)

    thr_adjusted = min(thr + 20, 250)
    ink = gray < thr_adjusted
    if ink.mean() < 0.001 or ink.mean() > 0.8:
        ink = gray > max(thr - 20, 5)
    if ink.mean() < 0.001:
        ink = gray < 250

    rows = np.where(ink.any(axis=1))[0]
    cols = np.where(ink.any(axis=0))[0]
    if len(rows) == 0 or len(cols) == 0:
        h, w = gray.shape
        alpha = Image.new("L", (w, h), 0)
        crop = img_rgb.crop((0, 0, w, h)).convert("RGBA")
        crop.putalpha(alpha)
        return crop, int(h * 0.8), (0, 0, w, h)

    top, bottom = rows[0], rows[-1]
    left, right = cols[0], cols[-1]
    bbox = (left, top, right + 1, bottom + 1)
    ink_crop = ink[top:bottom + 1, left:right + 1]
    h, w = ink_crop.shape

    bottoms = np.full(w, np.nan)
    for x in range(w):
        ys = np.where(ink_crop[:, x])[0]
        if ys.size > 0:
            bottoms[x] = ys[-1]
    baseline = int(np.nanmedian(bottoms)) if np.isfinite(bottoms).any() else int(h * 0.8)

    alpha = Image.fromarray((ink_crop.astype(np.uint8)) * 255, mode="L")
    crop = img_rgb.crop(bbox).convert("RGBA")
    crop.putalpha(alpha)
    return crop, baseline, bbox

# =============================================================================
# SCALING HELPERS
# =============================================================================

def scale_word_to_text_height(word_rgba, baseline, target_h):
    w, h = word_rgba.size
    if h <= 0:
        return word_rgba, baseline
    s = target_h / float(h)
    return (word_rgba.resize((max(1, int(round(w * s))), max(1, int(round(h * s)))),
                             Image.LANCZOS), int(round(baseline * s)))

def apply_uniform_scale(word_rgba, baseline, factor):
    w, h = word_rgba.size
    return (word_rgba.resize((max(1, int(round(w * factor))), max(1, int(round(h * factor)))),
                             Image.LANCZOS), int(round(baseline * factor)))

def calculate_scale_for_exact_words(words, cfg):
    target_h = int(round(cfg.img_height * cfg.text_height_ratio))
    usable = cfg.img_width - cfg.left_margin - cfg.right_margin
    total_w = 0
    for wp in words:
        try:
            img = Image.open(wp["path"])
            rgba, bl, _ = build_word_cutout_with_baseline(img)
            rgba_s, _ = scale_word_to_text_height(rgba, bl, target_h)
            total_w += rgba_s.size[0]
        except Exception:
            return 1.0
    total_w += np.mean(cfg.spacing_range) * (len(words) - 1)
    return usable / total_w if total_w > 0 else 1.0

def calculate_standard_scale_factor(word_samples, cfg):
    target_h = int(round(cfg.img_height * cfg.text_height_ratio))
    usable = cfg.img_width - cfg.left_margin - cfg.right_margin
    n = min(50, len(word_samples))
    if n == 0:
        return 1.0
    widths = []
    for wp in random.sample(word_samples, n):
        try:
            img = Image.open(wp["path"])
            rgba, bl, _ = build_word_cutout_with_baseline(img)
            rgba_s, _ = scale_word_to_text_height(rgba, bl, target_h)
            widths.append(rgba_s.size[0])
        except Exception:
            continue
    if not widths:
        return 1.0
    est = np.mean(widths) * cfg.max_words_for_scaling + np.mean(cfg.spacing_range) * (cfg.max_words_for_scaling - 1)
    sf = (usable / est) if est > usable else 1.0
    return sf * 0.95

# =============================================================================
# FILENAME PARSING AND GROUPING
# =============================================================================

def parse_writer_id(filename):
    m = re.search(r"DNDK(\d+)_", filename)
    return int(m.group(1)) if m else None

def parse_word_number(filename):
    m = re.search(r"_(\d+)_(\d+)\.", filename)
    return int(m.group(2)) if m else None

def get_unique_word_group(writer_id, group_size):
    g = (writer_id - 1) // group_size
    start = g * group_size + 1
    return (start, start + group_size - 1)

def get_person_name_subgroup(writer_id, group_size):
    g = (writer_id - 1) // group_size
    start = g * group_size + 1
    return (start, start + group_size - 1)

# =============================================================================
# LINE LENGTH SAMPLING (P(k) distribution)
# =============================================================================

def sample_line_length():
    r = random.random()
    if r < 0.25:
        return 7
    elif r < 0.75:
        return 8
    else:
        return random.choice([4, 5, 6])

# =============================================================================
# WRITER FILE LOADING
# =============================================================================

def load_writer_names_from_file(filepath):
    writers = set()
    if filepath is None or not os.path.isfile(filepath):
        return writers
    with open(filepath, "r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            m = re.search(r"DNDK(\d+)", line)
            if m:
                writers.add(int(m.group(1)))
            else:
                try:
                    writers.add(int(line))
                except ValueError:
                    pass
    return writers

# =============================================================================
# WORD POOL LOADING (cross-subset search)
# =============================================================================

def load_word_pool_all_subsets(root_dir, source_tag, allowed_writers, source_subsets,

                               allowed_exts=(".png", ".jpg", ".jpeg", ".bmp", ".tif", ".tiff")):
    """Search all source subfolders for word samples belonging to allowed writers"""
    if allowed_writers is not None and len(allowed_writers) == 0:
        return {}, 0, {}

    writer_words = defaultdict(dict)
    total_loaded = 0
    subset_counts = {}

    for src_subset in source_subsets:
        subset_dir = os.path.join(root_dir, src_subset)
        if not os.path.exists(subset_dir):
            continue
        loaded = 0
        for ext in allowed_exts:
            for path in glob.glob(os.path.join(subset_dir, f"*{ext}")):
                fn = os.path.basename(path)
                txt_path = os.path.splitext(path)[0] + ".txt"
                if not os.path.isfile(txt_path):
                    continue
                wid = parse_writer_id(fn)
                if wid is None:
                    continue
                if allowed_writers is not None and wid not in allowed_writers:
                    continue
                wn = parse_word_number(fn)
                if wn is None:
                    continue
                if wn in writer_words[wid]:
                    continue
                try:
                    with open(txt_path, "r", encoding="utf-8") as f:
                        lbl = normalize_label(f.read())
                except UnicodeDecodeError:
                    with open(txt_path, "r", encoding="utf-8-sig") as f:
                        lbl = normalize_label(f.read())

                writer_words[wid][wn] = dict(
                    path=path, label=lbl, writer=f"DNDK{wid:05d}",
                    writer_id=wid, word_num=wn, source=source_tag, found_in=src_subset)
                loaded += 1

        subset_counts[src_subset] = loaded
        total_loaded += loaded

    return dict(writer_words), total_loaded, subset_counts

# =============================================================================
# BUILD HIERARCHICAL GROUPS
# =============================================================================

def build_big_groups(unique_data, person_data, cfg):
    all_writers = set(unique_data.keys()) | set(person_data.keys())
    groups = {}
    for wid in all_writers:
        bg = get_unique_word_group(wid, cfg.unique_group_size)
        if bg not in groups:
            groups[bg] = dict(writers=set(), unique_word_nums=set(), subgroups={}, pools={})
        g = groups[bg]
        g["writers"].add(wid)
        g["pools"][wid] = dict(unique=unique_data.get(wid, {}), person_names=person_data.get(wid, {}))
        if wid in unique_data:
            g["unique_word_nums"].update(unique_data[wid].keys())
        if wid in person_data:
            sg = get_person_name_subgroup(wid, cfg.person_group_size)
            if sg not in g["subgroups"]:
                g["subgroups"][sg] = dict(writers=set(), pn_nums=set())
            g["subgroups"][sg]["writers"].add(wid)
            g["subgroups"][sg]["pn_nums"].update(person_data[wid].keys())
    return groups

# =============================================================================
# SAVE HELPERS
# =============================================================================

def save_tiff_with_metadata(image, save_path):
    if image.mode != "RGB":
        image = image.convert("RGB")
    info = TiffImagePlugin.ImageFileDirectory_v2()
    info[317] = 2
    image.save(save_path, format="TIFF", compression="tiff_lzw", dpi=(300, 300), tiffinfo=info)

def save_pair(out_dir, base_name, image, text):
    os.makedirs(out_dir, exist_ok=True)
    save_tiff_with_metadata(image, os.path.join(out_dir, base_name + ".tif"))
    with open(os.path.join(out_dir, base_name + ".txt"), "w", encoding="utf-8") as f:
        f.write(normalize_label(text))

# =============================================================================
# RTL LINE COMPOSITION
# =============================================================================

def compose_line(words, standard_scale, cfg):
    """Compose words right-to-left on a white canvas with baseline alignment"""
    target_text_h = int(round(cfg.img_height * cfg.text_height_ratio))
    target_baseline_y = int(round(cfg.img_height * cfg.baseline_ratio))

    actual_scale = (calculate_scale_for_exact_words(words, cfg)
                    if len(words) == 8 else standard_scale)

    def process_words(scale):
        result = []
        for wp in words:
            img = Image.open(wp["path"])
            rgba, bl, _ = build_word_cutout_with_baseline(img)
            rgba_s, bl_s = scale_word_to_text_height(rgba, bl, target_text_h)
            rgba_f, bl_f = apply_uniform_scale(rgba_s, bl_s, scale)
            result.append(dict(img=rgba_f, baseline=bl_f, label=normalize_label(wp["label"])))
        return result

    processed = process_words(actual_scale)
    word_widths = [p["img"].size[0] for p in processed]
    gaps = [int(random.randint(*cfg.spacing_range) * actual_scale) for _ in range(max(0, len(processed) - 1))]
    content_w = sum(word_widths) + sum(gaps)
    usable = cfg.img_width - cfg.left_margin - cfg.right_margin

    if content_w > usable:
        actual_scale *= (usable / content_w) * 0.92
        processed = process_words(actual_scale)
        word_widths = [p["img"].size[0] for p in processed]
        gaps = [int(random.randint(*cfg.spacing_range) * actual_scale) for _ in range(max(0, len(processed) - 1))]
        content_w = sum(word_widths) + sum(gaps)

    canvas = Image.new("RGB", (cfg.img_width, cfg.img_height), color=(255, 255, 255))
    usable_right = cfg.img_width - cfg.right_margin
    offset_x = max(cfg.left_margin, usable_right - content_w)

    ordered = list(reversed(processed))
    gaps_ordered = list(reversed(gaps)) if gaps else []

    x = offset_x
    for idx, p in enumerate(ordered):
        w, h = p["img"].size
        x = min(x, cfg.img_width - cfg.right_margin - w)
        jitter = random.randint(-cfg.baseline_jitter, cfg.baseline_jitter)
        y = max(0, min(target_baseline_y + jitter - p["baseline"], cfg.img_height - h))
        canvas.paste(p["img"], (x, y), p["img"])
        x += w
        if idx < len(ordered) - 1 and idx < len(gaps_ordered):
            x += gaps_ordered[idx]
        x = min(x, cfg.img_width - cfg.right_margin)

    return canvas, " ".join(p["label"] for p in processed)

# =============================================================================
# PROCESS ONE BIG GROUP
# =============================================================================

def process_big_group(group_range, group_data, out_dir, standard_scale, cfg, logger):
    all_writers = sorted(group_data["writers"])
    if not all_writers:
        return 0

    logger.subsection(f"Big Group {group_range[0]}-{group_range[1]} ({len(all_writers)} writers)")

    # Build tagged word pool per writer
    writer_tagged_pool = {}
    for wid in all_writers:
        pool = {}
        for wn, rec in group_data["pools"][wid]["unique"].items():
            pool[("u", wn)] = rec
        sg = get_person_name_subgroup(wid, cfg.person_group_size)
        for wn, rec in group_data["pools"][wid]["person_names"].items():
            pool[("p", sg[0], wn)] = rec
        writer_tagged_pool[wid] = pool

    unique_tags = sorted([("u", wn) for wn in group_data["unique_word_nums"]])
    sg_pn_tags = {}
    for sg_range, sg_info in sorted(group_data["subgroups"].items()):
        sg_pn_tags[sg_range] = sorted([("p", sg_range[0], wn) for wn in sg_info["pn_nums"]])
    subgroup_list = sorted(group_data["subgroups"].keys())

    # Estimate recipe count
    total_samples = sum(len(p) for p in writer_tagged_pool.values())
    recipe_attempts = max(1, total_samples // 6) * 2

    logger.log(f"  Unique tags: {len(unique_tags)} | Sub-groups: {len(subgroup_list)} | "
               f"Samples: {total_samples} | Attempts: {recipe_attempts}")

    # Generate recipes with signature-based uniqueness
    used_signatures = set()
    recipes = []

    for _ in range(recipe_attempts):
        length = sample_line_length()
        include_pn = (random.random() < 0.40) and bool(subgroup_list)

        if include_pn:
            sg = random.choice(subgroup_list)
            pn_tags = sg_pn_tags.get(sg, [])
            if pn_tags and len(unique_tags) >= 1:
                max_pn = min(3, len(pn_tags), length - 1)
                if max_pn >= 1:
                    n_pn = random.randint(1, max_pn)
                    n_u = length - n_pn
                    if n_u > len(unique_tags):
                        n_u = len(unique_tags)
                        n_pn = length - n_u
                    if n_u >= 1 and 1 <= n_pn <= len(pn_tags):
                        sampled = random.sample(unique_tags, n_u) + random.sample(pn_tags, n_pn)
                        sig = tuple(sorted(sampled))
                        if sig not in used_signatures:
                            recipes.append(dict(tags=sampled, signature=sig, type="mixed", subgroup=sg))
                            used_signatures.add(sig)
                            continue

        if len(unique_tags) >= length:
            sampled = random.sample(unique_tags, length)
            sig = tuple(sorted(sampled))
            if sig not in used_signatures:
                recipes.append(dict(tags=sampled, signature=sig, type="pure", subgroup=None))
                used_signatures.add(sig)

    # Assign recipes: mixed first, then pure
    mixed = [r for r in recipes if r["type"] == "mixed"]
    pure = [r for r in recipes if r["type"] == "pure"]
    random.shuffle(mixed)
    random.shuffle(pure)

    writer_used = defaultdict(set)
    writer_counter = defaultdict(int)
    lines_created = 0

    for recipe in mixed + pure:
        eligible = (sorted(group_data["subgroups"].get(recipe["subgroup"], {}).get("writers", set()))
                    if recipe["type"] == "mixed" else all_writers)

        for wid in eligible:
            pool = writer_tagged_pool.get(wid, {})
            tags = recipe["tags"]
            if all(t in pool for t in tags) and all(t not in writer_used[wid] for t in tags):
                word_records = [pool[t] for t in tags]
                img, text = compose_line(word_records, standard_scale, cfg)

                writer_counter[wid] += 1
                base_name = f"DNDK{wid:05d}_6_{writer_counter[wid]}"
                save_pair(out_dir, base_name, img, text)

                for t in tags:
                    writer_used[wid].add(t)

                lines_created += 1
                if lines_created % 50 == 0:
                    logger.log(f"    ... {lines_created} lines created")
                break

    total_used = sum(len(u) for u in writer_used.values())
    pct = (total_used / total_samples * 100) if total_samples else 0
    logger.log(f"  Result: {lines_created} lines | {total_used}/{total_samples} words ({pct:.1f}%)")

    return lines_created

# =============================================================================
# MAIN GENERATOR
# =============================================================================

def generate(args):
    cfg = Config(args)

    random.seed(args.seed)
    np.random.seed(args.seed)

    writer_files = {
        "Training": args.training_writers,
        "Validation": args.validation_writers,
        "Testing": args.testing_writers,
    }

    os.makedirs(args.output_dir, exist_ok=True)
    logger = Logger(os.path.join(args.output_dir, "generation_log.txt"))

    logger.section("CONFIGURATION")
    logger.log(f"  Canvas: {cfg.img_width} x {cfg.img_height}")
    logger.log(f"  Unique-word group: {cfg.unique_group_size} writers")
    logger.log(f"  Person-name group: {cfg.person_group_size} writers")
    logger.log(f"  Line lengths: 4-8 (50%->8, 25%->7, 25%->4|5|6)")
    logger.log(f"  Person-name mix: ~40% of recipes")
    logger.log(f"  Seed: {args.seed}")

    overall = {}
    for out_subset in ["Training", "Validation", "Testing"]:
        logger.section(f"{out_subset.upper()} SUBSET")

        wf = writer_files.get(out_subset)
        if wf and os.path.isfile(wf):
            allowed_writers = load_writer_names_from_file(wf)
            logger.log(f"  Writers: {len(allowed_writers)} from {wf}")
            if not allowed_writers:
                continue
        else:
            allowed_writers = None
            logger.log(f"  No writer file — using ALL writers")

        unique_data, u_total, u_counts = load_word_pool_all_subsets(
            args.unique_words_dir, "unique", allowed_writers, cfg.source_subsets)
        person_data, p_total, p_counts = load_word_pool_all_subsets(
            args.person_names_dir, "person_name", allowed_writers, cfg.source_subsets)

        logger.log(f"  Unique words: {u_total} from {len(unique_data)} writers")
        logger.log(f"  Person names: {p_total} from {len(person_data)} writers")

        if not unique_data and not person_data:
            continue

        big_groups = build_big_groups(unique_data, person_data, cfg)
        groups_sorted = sorted(big_groups.keys())
        if args.max_groups:
            groups_sorted = groups_sorted[:args.max_groups]

        all_records = []
        for gdata in big_groups.values():
            for pools in gdata["pools"].values():
                all_records.extend(pools["unique"].values())
                all_records.extend(pools["person_names"].values())

        if not all_records:
            continue

        standard_scale = calculate_standard_scale_factor(all_records, cfg)
        subset_out = os.path.join(args.output_dir, out_subset)

        total_lines = 0
        for gi, g_range in enumerate(groups_sorted, 1):
            n = process_big_group(g_range, big_groups[g_range], subset_out, standard_scale, cfg, logger)
            total_lines += n

        overall[out_subset] = total_lines
        logger.log(f"\n  {out_subset} total: {total_lines} lines")

    logger.section("SUMMARY")
    for subset, count in overall.items():
        logger.log(f"  {subset}: {count} lines")
    logger.log(f"  TOTAL: {sum(overall.values())} lines")
    logger.close()

    print(f"\nDone! {sum(overall.values())} lines generated.")

# =============================================================================
# ENTRY POINT
# =============================================================================

if __name__ == "__main__":
    generate(parse_args())