change client to sqlite

.gitignore

@@ -4,4 +4,5 @@ images
 merge_dev.bat
 __pycache__
 *.png
-*.txt
+*.txt
+db.sqlite

Client/hunt.py

@@ -1,5 +1,5 @@
 import os
-import lmdb
+import sqlite3
 import requests
 import tempfile
 import zipfile
@@ -39,7 +39,51 @@ os.makedirs("images/Stash", exist_ok=True)
 
 images_cache = set(os.listdir("images/Stash"))
 
-db = lmdb.open("db", subdir=True, map_size=1048576 * 2)
+DB_PATH = "db.sqlite"
+
+def open_db(path):
+    conn = sqlite3.connect(path)
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS pixif_cache (
+            post_id TEXT PRIMARY KEY,
+            url TEXT
+        )
+        """
+    )
+    conn.commit()
+    return conn
+
+def chunked(seq, size):
+    for i in range(0, len(seq), size):
+        yield seq[i:i + size]
+
+def fetch_cached_urls(conn, post_ids):
+    post_ids_dict = {post_id: None for post_id in post_ids}
+    if not post_ids:
+        return post_ids_dict
+
+    for chunk in chunked(post_ids, 900):
+        placeholders = ",".join("?" for _ in chunk)
+        query = f"SELECT post_id, COALESCE(url, '') FROM pixif_cache WHERE post_id IN ({placeholders})"
+        for post_id, url in conn.execute(query, chunk):
+            post_ids_dict[post_id] = url
+
+    return post_ids_dict
+
+def upsert_urls(conn, rows):
+    if not rows:
+        return
+    conn.executemany(
+        """
+        INSERT INTO pixif_cache (post_id, url)
+        VALUES (?, ?)
+        ON CONFLICT(post_id) DO UPDATE SET url = excluded.url
+        """,
+        rows,
+    )
+
+conn = open_db(DB_PATH)
 
 valid = [f for f in os.listdir() if f.endswith(".txt")]
 for idx, file in enumerate(valid):
@@ -88,20 +132,28 @@ for index in indexs:
     with open(valid[index], "r") as f:
         post_ids = f.read().split()
 
-    with db.begin(write=True) as txn:
-        post_ids_dict = {post_id: txn.get(post_id.encode()) for post_id in post_ids}
-        filtered = [post_id for post_id in post_ids if post_ids_dict[post_id] is None and f"{post_id}.png" not in images_cache]
-        print(f"Group: {group_name}\nFiltered: {len(filtered)}/{len(post_ids)}")
-
-        if filtered:
-            data = requests.post(f'{endpoint}/pixif', json={"post_ids": filtered, "phpsessid": phpsessid})
-            data = data.json()
-            for post_id, url in data.items():
-                txn.put(post_id.encode(), url.encode())
-                post_ids_dict[post_id] = url
-            no_exif = set(filtered) - set(data.keys())
-            for post_id in no_exif:
-                txn.put(post_id.encode(), b'')
+    post_ids_dict = fetch_cached_urls(conn, post_ids)
+    filtered = [
+        post_id
+        for post_id in post_ids
+        if post_ids_dict[post_id] is None and f"{post_id}.png" not in images_cache
+    ]
+    print(f"Group: {group_name}\nFiltered: {len(filtered)}/{len(post_ids)}")
+
+    if filtered:
+        data = requests.post(f'{endpoint}/pixif', json={"post_ids": filtered, "phpsessid": phpsessid})
+        data = data.json()
+        rows = [(post_id, url) for post_id, url in data.items()]
+        no_exif = set(filtered) - set(data.keys())
+        rows.extend((post_id, "") for post_id in no_exif)
+
+        with conn:
+            upsert_urls(conn, rows)
+
+        for post_id, url in data.items():
+            post_ids_dict[post_id] = url
+        for post_id in no_exif:
+            post_ids_dict[post_id] = ""
 
     to_download = {post_id: decode_if_binary(url) for post_id, url in post_ids_dict.items() if url and f"{post_id}.png" not in images_cache}
     
@@ -122,4 +174,4 @@ for index in indexs:
     if not os.listdir(f'images/{group_name}'):
         os.rmdir(f"images/{group_name}")
 
-db.close()
+conn.close()

Client/migrate_to_sqlite.py

@@ -0,0 +1,88 @@
+import argparse
+import os
+import sqlite3
+
+import lmdb
+
+
+def get_map_size(lmdb_path):
+    data_path = os.path.join(lmdb_path, "data.mdb")
+    try:
+        size = os.path.getsize(data_path)
+    except OSError:
+        return 1048576 * 2
+    return max(size * 2, 1048576 * 2)
+
+
+def open_sqlite(path):
+    conn = sqlite3.connect(path)
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS pixif_cache (
+            post_id TEXT PRIMARY KEY,
+            url TEXT
+        )
+        """
+    )
+    conn.commit()
+    return conn
+
+
+def upsert_rows(conn, rows):
+    if not rows:
+        return
+    conn.executemany(
+        """
+        INSERT INTO pixif_cache (post_id, url)
+        VALUES (?, ?)
+        ON CONFLICT(post_id) DO UPDATE SET url = excluded.url
+        """,
+        rows,
+    )
+
+
+def migrate(lmdb_path, sqlite_path, batch_size):
+    map_size = get_map_size(lmdb_path)
+    lmdb_env = lmdb.open(
+        lmdb_path,
+        subdir=True,
+        readonly=True,
+        lock=False,
+        map_size=map_size,
+    )
+    conn = open_sqlite(sqlite_path)
+
+    rows = []
+    with lmdb_env.begin() as txn:
+        cursor = txn.cursor()
+        for key, value in cursor:
+            post_id = key.decode("utf-8")
+            url = value.decode("utf-8") if value else ""
+            rows.append((post_id, url))
+
+            if len(rows) >= batch_size:
+                with conn:
+                    upsert_rows(conn, rows)
+                rows.clear()
+
+    if rows:
+        with conn:
+            upsert_rows(conn, rows)
+
+    conn.close()
+    lmdb_env.close()
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Migrate LMDB cache to SQLite.")
+    parser.add_argument("--lmdb", default="db", help="Path to LMDB directory.")
+    parser.add_argument("--sqlite", default="db.sqlite", help="Path to SQLite file.")
+    parser.add_argument("--batch-size", type=int, default=1000, help="Rows per batch insert.")
+    args = parser.parse_args()
+
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+    migrate(args.lmdb, args.sqlite, args.batch_size)
+
+
+if __name__ == "__main__":
+    main()