More efficient filtered

Client/hunt.py

@@ -1,7 +1,6 @@
 import os
 import lmdb
 import requests
-import io
 from concurrent.futures import ThreadPoolExecutor
 
 local = 0
@@ -34,8 +33,6 @@ for inp in inputs:
         start, end = map(int, inp.split("-"))
         indexs.extend(range(start - 1, end))
 
-blacklist = ['\0', None]
-
 def download_image(args):
     post_id, url = args
     full_url = img_base + url
@@ -55,10 +52,9 @@ for index in indexs:
 
     with db.begin(write=True) as txn:
         post_ids_dict = {post_id: txn.get(post_id.encode()) for post_id in post_ids}
-        post_ids_dict = {post_id: url.decode() if url else None for post_id, url in post_ids_dict.items()}
-
-        filtered = [post_id for post_id, url in post_ids_dict.items() if url is None and f"{post_id}.png" not in images_cache]
+        post_ids_dict = {post_id: url.decode() if url != None else None for post_id, url in post_ids_dict.items()}
 
+        filtered = [post_id for post_id, url in post_ids_dict.items() if post_ids_dict[post_id] != '' and f"{post_id}.png" not in images_cache]
         print(f"Group: {group_name}\nFiltered: {len(filtered)}/{len(post_ids_dict)}")
         if filtered:
             data = requests.post(f'{endpoint}/pixif', json={"post_ids": filtered}).json()
@@ -69,10 +65,10 @@ for index in indexs:
 
             no_exif = set(filtered) - set(data.keys())
             for post_id in no_exif:
-                txn.put(post_id.encode(), b'\0')
+                txn.put(post_id.encode(), b'')
 
     to_download = {post_id: url for post_id, url in post_ids_dict.items() 
-                   if url not in blacklist and f"{post_id}.png" not in images_cache}
+                   if url and f"{post_id}.png" not in images_cache}
 
     print(f"Total images to download: {len(to_download)}")
 

Client/show_all.py

@@ -3,7 +3,8 @@ import lmdb
 
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
-db = lmdb.open("db", subdir=True, map_size=524288)
+db = lmdb.open("db", subdir=True, map_size=1048576)
+
 
 items = []
 

Client/t.py

@@ -0,0 +1,12 @@
+import lmdb
+import os
+
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+db = lmdb.open("db", subdir=True, map_size=1048576)
+
+# print all
+with db.begin(write=True) as txn:
+    cursor = txn.cursor()
+    print(txn.get(b'124922612').decode() == '')
+    cursor.close()