Add DMHY digit-dedup list export

datasets/AnimeName

@@ -1 +1 @@
-Subproject commit 1b87d60c2e232d556ab06a48124985e77af33b53
+Subproject commit 7a09918c1f1190ca0e26b580f932831292b4f01a

tools/export_dmhy_list.py

@@ -0,0 +1,110 @@
+"""Export a digit-skeleton deduplicated DMHY filename list."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+
+from tools.dmhy_dataset import VIDEO_EXTENSIONS, normalize_path_basename
+
+
+DIGIT_RE = re.compile(r"\d+")
+
+
+def strip_video_extension(basename: str) -> tuple[str, str]:
+    stem, ext = os.path.splitext(basename)
+    return stem.strip(), ext.lower()
+
+
+def digit_skeleton(text: str) -> str:
+    """Return a key where only same-position non-digits must match."""
+    return DIGIT_RE.sub("<NUM>", text)
+
+
+def iter_filenames(db_path: Path, min_id: int, max_id: int):
+    conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=30)
+    conn.execute("PRAGMA query_only=ON")
+    try:
+        yield from conn.execute(
+            "SELECT id, filename FROM files WHERE id >= ? AND id <= ? ORDER BY id",
+            (min_id, max_id),
+        )
+    finally:
+        conn.close()
+
+
+def export_list(args: argparse.Namespace) -> None:
+    db_path = Path(args.db)
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=30)
+    conn.execute("PRAGMA query_only=ON")
+    try:
+        db_max_id = conn.execute("SELECT MAX(id) FROM files").fetchone()[0] or 0
+        max_id = min(args.max_id if args.max_id is not None else db_max_id, db_max_id)
+    finally:
+        conn.close()
+
+    seen_stems: set[str] = set()
+    seen_skeletons: set[str] = set()
+    stats = {
+        "scanned_rows": 0,
+        "video_rows": 0,
+        "duplicate_basenames": 0,
+        "duplicate_digit_skeletons": 0,
+        "written_rows": 0,
+    }
+
+    with output_path.open("w", encoding="utf-8", newline="\n") as out:
+        for _, raw_filename in iter_filenames(db_path, args.min_id, max_id):
+            stats["scanned_rows"] += 1
+            basename = normalize_path_basename(raw_filename)
+            stem, ext = strip_video_extension(basename)
+            if ext not in VIDEO_EXTENSIONS:
+                continue
+            stats["video_rows"] += 1
+            if stem in seen_stems:
+                stats["duplicate_basenames"] += 1
+                continue
+            seen_stems.add(stem)
+
+            skeleton = digit_skeleton(stem)
+            if skeleton in seen_skeletons:
+                stats["duplicate_digit_skeletons"] += 1
+                continue
+            seen_skeletons.add(skeleton)
+            out.write(stem + "\n")
+            stats["written_rows"] += 1
+
+    manifest = {
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "source_db": str(db_path),
+        "output": str(output_path),
+        "min_file_id": args.min_id,
+        "last_file_id": max_id,
+        "db_max_file_id_at_export_start": db_max_id,
+        "dedupe_rule": "strip video extension, then replace every contiguous digit run with <NUM>; keep the first row per resulting skeleton",
+        "stats": stats,
+    }
+    manifest_path = output_path.with_suffix(".manifest.json")
+    manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(json.dumps(manifest, ensure_ascii=False, indent=2))
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Export DMHY filename stems deduplicated by digit skeleton")
+    parser.add_argument("--db", default=r"D:\WorkSpace\Python\dmhy-parser\dmhy_anime.db")
+    parser.add_argument("--output", default=r"datasets\AnimeName\dmhy_list.txt")
+    parser.add_argument("--min-id", type=int, default=1)
+    parser.add_argument("--max-id", type=int, default=None)
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    export_list(parse_args())