Moved transactions

Client/Extract Pixiv/ai_search.py

@@ -11,7 +11,7 @@ os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
 input_url = input("Enter the URL: ")
 
-pages = 420 // 60
+pages = 120 // 60
 
 params = {
     'raw': input_url,

Client/hunt.py

@@ -27,17 +27,17 @@ for inp in inputs:
         start, end = map(int, inp.split("-"))
         indexs.extend(range(start - 1, end))
 
-with db.begin(write=True) as txn:
-    for index in indexs:
-        group_name = valid[index].rsplit(".", 1)[0]
+for index in indexs:
+    group_name = valid[index].rsplit(".", 1)[0]
 
-        group_cache = os.listdir(f"images/{group_name}")
+    group_cache = os.listdir(f"images/{group_name}")
 
-        with open(valid[index], "r") as f:
-            post_ids = [x for x in f.read().split("\n") if x]
+    with open(valid[index], "r") as f:
+        post_ids = [x for x in f.read().split("\n") if x]
+
+    with db.begin(write=True) as txn:
+        filtered_post_ids = [post_id for post_id in post_ids if txn.get(post_id.encode()) == None]
 
-        filtered_post_ids = [post_id for post_id in post_ids if not txn.get(post_id.encode())]
-        print(filtered_post_ids)
         data = requests.post('https://q6-p.hf.space/pixif', json={"post_ids": filtered_post_ids}).json()
 
         for post_id, url in data.items():
@@ -46,7 +46,7 @@ with db.begin(write=True) as txn:
         missing_post_ids = set(filtered_post_ids) - set(data.keys())
         for post_id in missing_post_ids:
             txn.put(post_id.encode(), b'\x00')
-        
+    
         to_download = {}
         for post_id in set(post_ids):
             if f"{post_id}.png" in images_cache:
@@ -55,14 +55,14 @@ with db.begin(write=True) as txn:
             if url != b'\x00':
                 to_download[post_id] = url.decode()
 
-        images_zip = requests.post('https://q6-p.hf.space/download', json={'posts': data})
+    images_zip = requests.post('https://q6-p.hf.space/download', json={'posts': data})
 
-        with io.BytesIO(images_zip.content) as f:
-            with ZipFile(f) as z:
-                z.extractall("images/Stash")
+    with io.BytesIO(images_zip.content) as f:
+        with ZipFile(f) as z:
+            z.extractall("images/Stash")
 
-        for i, post_id in enumerate(post_ids):
-            if f"{post_id}.png" in images_cache and f"{i}_{post_id}.png" not in group_cache:
-                os.makedirs(f"images/{group_name}", exist_ok=True)
-                os.link(f"images/Stash/{post_id}.png", f"images/{group_name}/{i}_{post_id}.png")
-                continue
+    for i, post_id in enumerate(post_ids):
+        if f"{post_id}.png" in images_cache and f"{i}_{post_id}.png" not in group_cache:
+            os.makedirs(f"images/{group_name}", exist_ok=True)
+            os.link(f"images/Stash/{post_id}.png", f"images/{group_name}/{i}_{post_id}.png")
+            continue

Client/test2.py

@@ -0,0 +1,15 @@
+import os
+import lmdb
+import requests
+import io
+from zipfile import ZipFile
+
+img_base = 'https://i.pximg.net/img-original/'
+
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+os.makedirs("images/Stash", exist_ok=True)
+
+db = lmdb.open("db", subdir=True, map_size=1048576)
+
+with db.begin(write=True) as txn:
+    print(txn.get(b"test") == None)