File size: 7,061 Bytes
1c6923c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 | import argparse
import os
import sys
import json
from concurrent.futures import ThreadPoolExecutor
import filetype
import requests
from tqdm.contrib.concurrent import thread_map
from tqdm import tqdm
def get_download_path():
if os.name == 'nt':
import winreg
sub_key = r'SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders'
downloads_guid = '{374DE290-123F-4565-9164-39C4925E467B}'
with winreg.OpenKey(winreg.HKEY_CURRENT_USER, sub_key) as key:
location = winreg.QueryValueEx(key, downloads_guid)[0]
return location
else:
return os.path.join(os.path.expanduser('~'), 'Downloads')
def get_default_dir(search_key: str):
search_key = search_key.replace(" ", ".")
default_dir = get_download_path()
default_dir = os.path.join(default_dir, "google-image-scraper")
default_dir = os.path.join(default_dir, search_key)
return default_dir
def check_pos_int(val: int):
val = int(val)
if val > 0:
return val
else:
raise ValueError
def get_arguments(argv=sys.argv):
parser = argparse.ArgumentParser(description="Scrape google for images")
parser.add_argument("keyword",
help="the phrase used to find images",
type=str,
nargs=1)
parser.add_argument("-c", "--count",
help="How many images to try to scrape",
type=check_pos_int,
nargs="?",
default=10)
parser.add_argument("-d", "--directory",
help="where to save scraped images",
type=str,
nargs="?")
parser.add_argument("-t", "--threads",
help="How many threads to spawn and download from",
type=check_pos_int,
nargs="?",
default=1)
args = parser.parse_args(argv[1:])
if args.directory is None:
print(args.keyword[0])
args.directory = get_default_dir(args.keyword[0])
return args
def sanitize_query(query: str):
return query.replace(' ', '+')
def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
def add_filetype(file_path: str):
try:
img_type = filetype.guess(file_path).mime.split('/')[-1]
except:
img_type = 'jpg'
new_path = '.'.join((os.path.splitext(file_path)[0], img_type))
try:
os.rename(file_path, new_path)
return 0
except Exception as err:
eprint("Couldn't rename file at path {}".format(file_path))
eprint(err)
return 1
DEBUG = False
# search url and browser headers for spoofing
headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/110.0'}
search_url = "https://www.google.com/search?q={}&tbm=isch&async=_id:islrg_c,_fmt:json&asearch=ichunklite&ijn={}"
def get_image_urls(query: str, page: int):
all_images = {}
try:
response = requests.get(search_url.format(query, page), headers=headers)
if DEBUG and response.status_code != 200:
eprint("ERROR: Status code {} for page {}."
.format(response.status_code, page))
if response.status_code == 200:
json_text = response.content.decode('utf8').removeprefix(")]}'")
json_data = json.loads(json_text)
try:
results = json_data['ichunklite']['results']
for result in results:
id = result['image_docid']
image_url = result['viewer_metadata']['original_image']['url']
all_images[id] = image_url
except Exception as err:
if DEBUG:
eprint(f"ERROR: Issue parsing json file for page {page}")
eprint(err)
else:
if DEBUG:
eprint("ERROR: Status code {} for page {}."
.format(response.status_code, page))
except Exception as err:
if DEBUG:
eprint("ERROR: There was an issue requesting the json file")
eprint(err)
return all_images
def download_image(img_url: str, file_path: str):
try:
response = requests.get(img_url, headers=headers)
except Exception as err:
if DEBUG:
eprint(f"ERROR: Can't request image url: {img_url}")
eprint(err)
return 1
if response.status_code == 200:
response_content_type = response.headers.get("content-type", '').lower()
if "image" in response_content_type:
with open(file_path, 'wb') as img:
img.write(response.content)
add_filetype(file_path)
return 0
else:
if DEBUG:
eprint("ERROR: Bad content-type\n{}"
.format(response_content_type))
return 1
else:
if DEBUG:
eprint("ERROR: Status code {} for url {}."
.format(response.status_code, url))
return 1
def get_manifest(search_key: str, image_cnt: int):
err_cnt = 0
err_limit = 5
img_manifest = {}
manifest_len = image_cnt
results_page = 0
search_key = sanitize_query(search_key)
while len(img_manifest.items()) < image_cnt:
try:
img_manifest.update(get_image_urls(search_key, results_page))
results_page += 1
except:
print(f"err_cnt: {err_cnt}")
err_cnt += 1
if err_cnt > err_limit:
eprint("Couldn't request all images")
manifest_len = len(img_manifest.items())
break
print("Found {} of {} image sources".format(manifest_len, image_cnt))
return list(img_manifest.items())[0:manifest_len]
def scrape_images(search_key, image_cnt, directory, threads):
if DEBUG:
print("savedir: {}".format(directory))
if not os.path.exists(directory):
os.makedirs(directory)
id_url_manifest = get_manifest(search_key, image_cnt)
with ThreadPoolExecutor(max_workers=threads) as pool:
with tqdm(total=len(id_url_manifest)) as progress:
futures = []
for result in id_url_manifest:
id: str = result[0]
url: str = result[1]
save_path = os.path.join(directory, id)
future = pool.submit(download_image, url, save_path)
future.add_done_callback(lambda p: progress.update(1))
futures.append(future)
for future in futures:
result = future.result()
return 0
if __name__ == "__main__":
args = get_arguments(sys.argv)
scrape_images(args.keyword[0], args.count, args.directory, args.threads) |