Hyper Filter

API/app.py

@@ -189,33 +189,49 @@ def parse_png_metadata(data):
         index += chunk_len + 4
     return None
 
-async def process_post(post_id, session, semaphore):
+async def process_image(image_dict, session, semaphore):
     async with semaphore:
-        try:
-            data = await fetch_page(session, f"https://www.pixiv.net/ajax/illust/{post_id}/pages")
-            image_urls = [page['urls']['original'] for page in data['body'] if 'png' in page['urls']['original']][:20]
-            for image_url in image_urls:
-                metadata = await get_exif(image_url, session)
-                exif_type = determine_exif_type(metadata)
-                if exif_type not in ['photoshop', 'celsys', None]:
-                    return post_id, image_url
-            return post_id, None
-        except Exception as e:
-            return post_id, None
+        image_url = image_dict['image_url']
+        post_id = image_dict['post_id']
+        metadata = await get_exif(image_url, session)
+        exif_type = determine_exif_type(metadata)
+        if exif_type not in ['photoshop', 'celsys', None]:
+            return {'post_id': post_id, 'image_url': image_url}
+    return None
+
 
 @app.post("/pixif")
 async def pixif(
     items: pixifModel
 ):
     post_ids = items.post_ids
-    semaphore = asyncio.Semaphore(100)
+    semaphore = asyncio.Semaphore(200)
 
     async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
-        tasks = [process_post(post_id, session, semaphore) for post_id in post_ids]
-        results = await asyncio.gather(*tasks)
+        pages_tasks = [fetch_page(session, f"https://www.pixiv.net/ajax/illust/{post_id}/pages") for post_id in post_ids]
+        pages_results = await asyncio.gather(*pages_tasks)
+    
+        image_list = []
+        for post_id, data in zip(post_ids, pages_results):
+            image_urls = [page['urls']['original'] for page in data['body'] if 'png' in page['urls']['original']][:20]
+            for image_url in image_urls:
+                image_list.append({'post_id': post_id, 'image_url': image_url})
+
+        image_list.sort(key=lambda x: int(x['post_id']))
+
+        batch_size = 20
+        for i in range(0, len(image_list), batch_size):
+            batch = image_list[i:i+batch_size]
+            tasks = [process_image(image_dict, session, semaphore) for image_dict in batch]
+            results = await asyncio.gather(*tasks)
+            for result in results:
+                if result:
+                    post_id = result['post_id']
+                    image_url = result['image_url']
+                    image_exifs = {post_id: image_url.replace('https://i.pximg.net/img-original/', '', 1)}
+                    return image_exifs
+        return {}
 
-    image_exifs = {post_id: image_url.replace('https://i.pximg.net/img-original/', '', 1) for post_id, image_url in results if image_url}
-    return image_exifs
 
 async def generate_zip(posts, session):
     zip_buffer = io.BytesIO()

Client/hunt.py

@@ -9,7 +9,7 @@ img_base = 'https://i.pximg.net/img-original/img/'
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 os.makedirs("images/Stash", exist_ok=True)
 
-db = lmdb.open("db", subdir=True, map_size=1048576)
+db = lmdb.open("db", subdir=True, map_size=524288)
 valid = [f for f in os.listdir() if f.endswith(".txt")]
 
 for idx, file in enumerate(valid):

Client/t2.py

@@ -4,7 +4,7 @@ import lmdb
 
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
-db = lmdb.open("db", subdir=True, map_size=1048576)
+db = lmdb.open("db", subdir=True, map_size=524288)
 
 # view all in db
 with db.begin() as txn: