Grouping

Client/hunt.py

@@ -34,48 +34,56 @@ for inp in inputs:
 
 blacklist = ['\0', None]
 
+group_to_post_ids = {}
+post_id_to_groups = {}
+all_post_ids = set()
+
 for index in indexs:
-    images_cache = os.listdir("images/Stash")
     group_name = valid[index].rsplit(".", 1)[0]
     os.makedirs(f"images/{group_name}", exist_ok=True)
     with open(valid[index], "r") as f:
         post_ids = [x for x in f.read().split("\n") if x]
-
-    with db.begin(write=True) as txn:
-        post_ids = {post_id: txn.get(post_id.encode()) for post_id in post_ids}
-        post_ids = {post_id: url.decode() if url else None for post_id, url in post_ids.items()}
-        filtered = [post_id for post_id, url in post_ids.items() if url == None and f"{post_id}.png" not in images_cache]
-
-        print(f"Group: {group_name}\nFiltered: {len(filtered)}/{len(post_ids)}")
-        if filtered:
-            data = requests.post(f'{endpoint}/pixif', json={"post_ids": filtered}).json()
-
-            for post_id, url in data.items():
-                txn.put(post_id.encode(), url.encode())
-                post_ids[post_id] = url
-
-            no_exif = set(filtered) - set(data.keys())
-
-            for post_id in no_exif:
-                txn.put(post_id.encode(), b'\0')
-
-    to_download = {post_id: url for post_id, url in post_ids.items() if url not in blacklist and f"{post_id}.png" not in images_cache}
+    group_to_post_ids[group_name] = post_ids
+    for post_id in post_ids:
+        all_post_ids.add(post_id)
+        if post_id not in post_id_to_groups:
+            post_id_to_groups[post_id] = []
+        post_id_to_groups[post_id].append(group_name)
+
+images_cache = os.listdir("images/Stash")
+
+with db.begin(write=True) as txn:
+    post_ids_db = {post_id: txn.get(post_id.encode()) for post_id in all_post_ids}
+    post_ids_db = {post_id: url.decode() if url else None for post_id, url in post_ids_db.items()}
+    filtered = [post_id for post_id, url in post_ids_db.items() if url == None and f"{post_id}.png" not in images_cache]
+    print(f"Filtered: {len(filtered)}/{len(post_ids_db)}")
+    if filtered:
+        data = requests.post(f'{endpoint}/pixif', json={"post_ids": filtered}).json()
+        for post_id, url in data.items():
+            txn.put(post_id.encode(), url.encode())
+            post_ids_db[post_id] = url
+        no_exif = set(filtered) - set(data.keys())
+        for post_id in no_exif:
+            txn.put(post_id.encode(), b'\0')
+
+    to_download = {post_id: url for post_id, url in post_ids_db.items() if url not in blacklist and f"{post_id}.png" not in images_cache}
     print(f"Downloading {len(to_download)} images...")
     if to_download:
         images_zip = requests.post(f'{endpoint}/download', json={'posts': to_download}, stream=True)
-
         print("Extracting images...")
         with io.BytesIO(images_zip.content) as f:
             with zipfile.ZipFile(f, mode='r') as zipf:
                 zipf.extractall("images/Stash")
-        
 
-    images_cache = os.listdir("images/Stash")
-    print("Linking images...")
-    for i, post_id in enumerate(post_ids.keys()):
-        if f"{post_id}.png" in images_cache:
-            if not os.path.exists(f"images/{group_name}/{i}_{post_id}.png"):
-                os.link(f"images/Stash/{post_id}.png", f"images/{group_name}/{i}_{post_id}.png")
+images_cache = os.listdir("images/Stash")
+print("Linking images...")
 
-    if len(os.listdir(f'images/{group_name}')) == 0:
-        os.rmdir(f"images/{group_name}")
+for group_name, post_ids in group_to_post_ids.items():
+    group_folder = f"images/{group_name}"
+    for i, post_id in enumerate(post_ids):
+        if f"{post_id}.png" in images_cache:
+            dest_path = f"{group_folder}/{i}_{post_id}.png"
+            if not os.path.exists(dest_path):
+                os.link(f"images/Stash/{post_id}.png", dest_path)
+    if len(os.listdir(group_folder)) == 0:
+        os.rmdir(group_folder)