tqdm
Browse files- Client/hunt.py +4 -2
Client/hunt.py
CHANGED
|
@@ -2,6 +2,7 @@ import os
|
|
| 2 |
import lmdb
|
| 3 |
import requests
|
| 4 |
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
| 5 |
|
| 6 |
local = 0
|
| 7 |
if local:
|
|
@@ -54,7 +55,7 @@ for index in indexs:
|
|
| 54 |
post_ids_dict = {post_id: txn.get(post_id.encode()) for post_id in post_ids}
|
| 55 |
post_ids_dict = {post_id: url.decode() if url != None else None for post_id, url in post_ids_dict.items()}
|
| 56 |
|
| 57 |
-
filtered = [post_id for post_id, url in post_ids_dict.items() if
|
| 58 |
print(f"Group: {group_name}\nFiltered: {len(filtered)}/{len(post_ids_dict)}")
|
| 59 |
if filtered:
|
| 60 |
data = requests.post(f'{endpoint}/pixif', json={"post_ids": filtered}).json()
|
|
@@ -78,7 +79,8 @@ for index in indexs:
|
|
| 78 |
if to_download_items:
|
| 79 |
print("Starting download...")
|
| 80 |
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
| 81 |
-
|
|
|
|
| 82 |
|
| 83 |
images_cache = os.listdir("images/Stash")
|
| 84 |
|
|
|
|
| 2 |
import lmdb
|
| 3 |
import requests
|
| 4 |
from concurrent.futures import ThreadPoolExecutor
|
| 5 |
+
from tqdm import tqdm
|
| 6 |
|
| 7 |
local = 0
|
| 8 |
if local:
|
|
|
|
| 55 |
post_ids_dict = {post_id: txn.get(post_id.encode()) for post_id in post_ids}
|
| 56 |
post_ids_dict = {post_id: url.decode() if url != None else None for post_id, url in post_ids_dict.items()}
|
| 57 |
|
| 58 |
+
filtered = [post_id for post_id, url in post_ids_dict.items() if url == None and f"{post_id}.png" not in images_cache]
|
| 59 |
print(f"Group: {group_name}\nFiltered: {len(filtered)}/{len(post_ids_dict)}")
|
| 60 |
if filtered:
|
| 61 |
data = requests.post(f'{endpoint}/pixif', json={"post_ids": filtered}).json()
|
|
|
|
| 79 |
if to_download_items:
|
| 80 |
print("Starting download...")
|
| 81 |
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
| 82 |
+
for result in tqdm(executor.map(download_image, to_download_items), total=len(to_download_items), desc="Downloading"):
|
| 83 |
+
pass
|
| 84 |
|
| 85 |
images_cache = os.listdir("images/Stash")
|
| 86 |
|