X

.gitignore

@@ -1,3 +1,4 @@
 .env
 *.txt
-db
+db
+images

API/app.py

@@ -244,6 +244,7 @@ async def download(
             zipf.write(f"Stash/{post_id}.png", post_id + ".png")
             
     return FileResponse(path=f"Stash/{timezip}.zip", media_type="application/zip", filename=f"{timezip}.zip")
+
 @app.get("/")
 async def read_root():
     return {"message": "Hello, World!"}

Client/clear_db.py

@@ -0,0 +1,10 @@
+import os
+import shutil
+
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+shutil.rmtree("images", ignore_errors=True)
+shutil.rmtree("db", ignore_errors=True)
+for file in os.listdir():
+    if file.endswith(".txt"):
+        os.remove(file)

Client/hunt.py

@@ -1,6 +1,8 @@
 import os
 import lmdb
 import requests
+import io
+from zipfile import ZipFile
 
 img_base = 'https://i.pximg.net/img-original/'
 
@@ -29,6 +31,9 @@ with db.begin(write=True) as txn:
     for index in indexs:
         group_name = valid[index].rsplit(".", 1)[0]
 
+        os.makedirs(f"images/{group_name}", exist_ok=True)
+        group_cache = os.listdir(f"images/{group_name}")
+
         with open(valid[index], "r") as f:
             post_ids = [x for x in f.read().split("\n") if x]
 
@@ -42,7 +47,22 @@ with db.begin(write=True) as txn:
         for post_id in missing_post_ids:
             txn.put(post_id.encode(), b'\x00')
         
-        data.update({post_id: txn.get(post_id.encode()).decode() for post_id in set(post_ids) - missing_post_ids if txn.get(post_id.encode()) != b'\x00'})
-        
-        print(data)
-        images_zip = requests.post('https://q6-p.hf.space/download', json={'posts': data}).json()
+        to_download = {}
+        for post_id in set(post_ids):
+            if f"{post_id}.png" in images_cache:
+                continue
+            url = txn.get(post_id.encode())
+            if url != b'\x00':
+                to_download[post_id] = url.decode()
+
+        images_zip = requests.post('https://q6-p.hf.space/download', json={'posts': data})
+
+        with io.BytesIO(images_zip.content) as f:
+            with ZipFile(f) as z:
+                z.extractall("images/Stash")
+
+        for i, post_id in enumerate(post_ids):
+            print(post_id)
+            if f"{post_id}.png" in images_cache and f"{i}_{post_id}.png" not in group_cache:
+                os.link(f"images/Stash/{post_id}.png", f"images/{group_name}/{i}_{post_id}.png")
+                continue