Working

Client/hunt.py

@@ -34,56 +34,48 @@ for inp in inputs:
 
 blacklist = ['\0', None]
 
-group_to_post_ids = {}
-post_id_to_groups = {}
-all_post_ids = set()
-
 for index in indexs:
+    images_cache = os.listdir("images/Stash")
     group_name = valid[index].rsplit(".", 1)[0]
     os.makedirs(f"images/{group_name}", exist_ok=True)
     with open(valid[index], "r") as f:
         post_ids = [x for x in f.read().split("\n") if x]
-    group_to_post_ids[group_name] = post_ids
-    for post_id in post_ids:
-        all_post_ids.add(post_id)
-        if post_id not in post_id_to_groups:
-            post_id_to_groups[post_id] = []
-        post_id_to_groups[post_id].append(group_name)
-
-images_cache = os.listdir("images/Stash")
-
-with db.begin(write=True) as txn:
-    post_ids_db = {post_id: txn.get(post_id.encode()) for post_id in all_post_ids}
-    post_ids_db = {post_id: url.decode() if url else None for post_id, url in post_ids_db.items()}
-    filtered = [post_id for post_id, url in post_ids_db.items() if url == None and f"{post_id}.png" not in images_cache]
-    print(f"Filtered: {len(filtered)}/{len(post_ids_db)}")
-    if filtered:
-        data = requests.post(f'{endpoint}/pixif', json={"post_ids": filtered}).json()
-        for post_id, url in data.items():
-            txn.put(post_id.encode(), url.encode())
-            post_ids_db[post_id] = url
-        no_exif = set(filtered) - set(data.keys())
-        for post_id in no_exif:
-            txn.put(post_id.encode(), b'\0')
-
-    to_download = {post_id: url for post_id, url in post_ids_db.items() if url not in blacklist and f"{post_id}.png" not in images_cache}
+
+    with db.begin(write=True) as txn:
+        post_ids = {post_id: txn.get(post_id.encode()) for post_id in post_ids}
+        post_ids = {post_id: url.decode() if url else None for post_id, url in post_ids.items()}
+        filtered = [post_id for post_id, url in post_ids.items() if url == None and f"{post_id}.png" not in images_cache]
+
+        print(f"Group: {group_name}\nFiltered: {len(filtered)}/{len(post_ids)}")
+        if filtered:
+            data = requests.post(f'{endpoint}/pixif', json={"post_ids": filtered}).json()
+
+            for post_id, url in data.items():
+                txn.put(post_id.encode(), url.encode())
+                post_ids[post_id] = url
+
+            no_exif = set(filtered) - set(data.keys())
+
+            for post_id in no_exif:
+                txn.put(post_id.encode(), b'\0')
+
+    to_download = {post_id: url for post_id, url in post_ids.items() if url not in blacklist and f"{post_id}.png" not in images_cache}
     print(f"Downloading {len(to_download)} images...")
     if to_download:
         images_zip = requests.post(f'{endpoint}/download', json={'posts': to_download}, stream=True)
+
         print("Extracting images...")
         with io.BytesIO(images_zip.content) as f:
             with zipfile.ZipFile(f, mode='r') as zipf:
                 zipf.extractall("images/Stash")
+        
 
-images_cache = os.listdir("images/Stash")
-print("Linking images...")
-
-for group_name, post_ids in group_to_post_ids.items():
-    group_folder = f"images/{group_name}"
-    for i, post_id in enumerate(post_ids):
+    images_cache = os.listdir("images/Stash")
+    print("Linking images...")
+    for i, post_id in enumerate(post_ids.keys()):
         if f"{post_id}.png" in images_cache:
-            dest_path = f"{group_folder}/{i}_{post_id}.png"
-            if not os.path.exists(dest_path):
-                os.link(f"images/Stash/{post_id}.png", dest_path)
-    if len(os.listdir(group_folder)) == 0:
-        os.rmdir(group_folder)
+            if not os.path.exists(f"images/{group_name}/{i}_{post_id}.png"):
+                os.link(f"images/Stash/{post_id}.png", f"images/{group_name}/{i}_{post_id}.png")
+
+    if len(os.listdir(f'images/{group_name}')) == 0:
+        os.rmdir(f"images/{group_name}")

build_docker.bat

@@ -0,0 +1 @@
+docker build -t pixif .