X

API/app.py

@@ -194,13 +194,32 @@ async def process_post(post_id, session, semaphore):
             data = await fetch_page(session, f"https://www.pixiv.net/ajax/illust/{post_id}/pages")
             image_urls = [page['urls']['original'] for page in data['body'] if 'png' in page['urls']['original']]
             
-            tasks = [get_exif(image_url, session) for image_url in image_urls]
-            exif_data_list = await asyncio.gather(*tasks)
+            initial_chunks = [
+                (0, 1),
+                (1, 6),
+                (6, 10),
+                (10, 21),
+                (21, 31),
+                (31, 41),
+            ]
             
+            chunks = initial_chunks[:]
+            start = 41
+            while start < len(image_urls):
+                end = min(start + 10, len(image_urls))
+                chunks.append((start, end))
+                start = end
+
+            exif_data_list = []
+            for s, e in chunks:
+                chunk_tasks = [get_exif(image_urls[i], session) for i in range(s, e)]
+                exif_data_list.extend(await asyncio.gather(*chunk_tasks))
+
             for image_url, metadata in zip(image_urls, exif_data_list):
                 exif_type = determine_exif_type(metadata)
                 if exif_type not in ['photoshop', 'celsys', None]:
                     return post_id, image_url
+
             return post_id, None
         except:
             return post_id, None

Client/Extract Pixiv/user.py

@@ -11,8 +11,10 @@ else:
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
 inp = input("Enter User IDs (separated by spaces or commas): ")
-while inp != "":
-    inp += input()
+inps='1'
+while inps != "":
+    inps = input('')
+    inp += inps
 user_ids = re.findall(r"\d+", inp)
 
 user_ids = [int(uid) for uid in user_ids]

Client/hunt.py

@@ -18,7 +18,7 @@ os.makedirs("images/Stash", exist_ok=True)
 
 images_cache = os.listdir("images/Stash")
 
-db = lmdb.open("db", subdir=True, map_size=524288)
+db = lmdb.open("db", subdir=True, map_size=1048576)
 valid = [f for f in os.listdir() if f.endswith(".txt")]
 
 for idx, file in enumerate(valid):