q6 commited on
Commit
300c794
·
1 Parent(s): 3c5fc21

hunt more dynamic

Browse files
Files changed (1) hide show
  1. Client/hunt.py +4 -1
Client/hunt.py CHANGED
@@ -16,6 +16,8 @@ img_base = 'https://i.pximg.net/img-original/img/'
16
  os.chdir(os.path.dirname(os.path.abspath(__file__)))
17
  os.makedirs("images/Stash", exist_ok=True)
18
 
 
 
19
  db = lmdb.open("db", subdir=True, map_size=524288)
20
  valid = [f for f in os.listdir() if f.endswith(".txt")]
21
 
@@ -60,6 +62,7 @@ for index in indexs:
60
  txn.put(post_id.encode(), b'\0')
61
 
62
  to_download = {post_id: url for post_id, url in post_ids.items() if url not in blacklist and f"{post_id}.png" not in images_cache}
 
63
  print(f"Downloading {len(to_download)} images...")
64
  if to_download:
65
  images_zstd = requests.post(f'{endpoint}/download', json={'posts': to_download}, stream=True)
@@ -79,7 +82,7 @@ for index in indexs:
79
  for i, post_id in enumerate(post_ids.keys()):
80
  if f"{post_id}.png" in images_cache:
81
  if not os.path.exists(f"images/{group_name}/{i}_{post_id}.png"):
82
- os.link(f"images/Stash/{post_id}.png", f"images/{group_name}/{i}_{post_id}.png")
83
 
84
  if len(os.listdir(f'images/{group_name}')) == 0:
85
  os.rmdir(f"images/{group_name}")
 
16
  os.chdir(os.path.dirname(os.path.abspath(__file__)))
17
  os.makedirs("images/Stash", exist_ok=True)
18
 
19
+ images_cache = os.listdir("images/Stash")
20
+
21
  db = lmdb.open("db", subdir=True, map_size=524288)
22
  valid = [f for f in os.listdir() if f.endswith(".txt")]
23
 
 
62
  txn.put(post_id.encode(), b'\0')
63
 
64
  to_download = {post_id: url for post_id, url in post_ids.items() if url not in blacklist and f"{post_id}.png" not in images_cache}
65
+ post_ids.update(to_download)
66
  print(f"Downloading {len(to_download)} images...")
67
  if to_download:
68
  images_zstd = requests.post(f'{endpoint}/download', json={'posts': to_download}, stream=True)
 
82
  for i, post_id in enumerate(post_ids.keys()):
83
  if f"{post_id}.png" in images_cache:
84
  if not os.path.exists(f"images/{group_name}/{i}_{post_id}.png"):
85
+ os.link(f"images/Stash/{post_id}.png", f"images/{group_name}/{i}_{post_id}_{post_ids[post_id].split('p')[1].split('.')[0]}.png")
86
 
87
  if len(os.listdir(f'images/{group_name}')) == 0:
88
  os.rmdir(f"images/{group_name}")