File size: 7,061 Bytes
1c6923c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
import argparse
import os
import sys
import json
from concurrent.futures import ThreadPoolExecutor
import filetype
import requests
from tqdm.contrib.concurrent import thread_map
from tqdm import tqdm

def get_download_path():
    if os.name == 'nt':
        import winreg
        sub_key = r'SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders'
        downloads_guid = '{374DE290-123F-4565-9164-39C4925E467B}'
        with winreg.OpenKey(winreg.HKEY_CURRENT_USER, sub_key) as key:
            location = winreg.QueryValueEx(key, downloads_guid)[0]
        return location
    else:
        return os.path.join(os.path.expanduser('~'), 'Downloads')


def get_default_dir(search_key: str):
    search_key = search_key.replace(" ", ".")
    default_dir = get_download_path()
    default_dir = os.path.join(default_dir, "google-image-scraper")
    default_dir = os.path.join(default_dir, search_key)
    return default_dir

def check_pos_int(val: int):
    val = int(val)
    if val > 0:
        return val
    else:
        raise ValueError

def get_arguments(argv=sys.argv):
    parser = argparse.ArgumentParser(description="Scrape google for images")
    parser.add_argument("keyword", 
                        help="the phrase used to find images",
                        type=str,
                        nargs=1)
    parser.add_argument("-c", "--count",
                        help="How many images to try to scrape",
                        type=check_pos_int,
                        nargs="?",
                        default=10)
    parser.add_argument("-d", "--directory",
                        help="where to save scraped images",
                        type=str,
                        nargs="?")
    parser.add_argument("-t", "--threads",
                        help="How many threads to spawn and download from",
                        type=check_pos_int,
                        nargs="?",
                        default=1)
    args = parser.parse_args(argv[1:])
    if args.directory is None:
        print(args.keyword[0])
        args.directory = get_default_dir(args.keyword[0])
    return args

def sanitize_query(query: str):
    return query.replace(' ', '+')

def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

def add_filetype(file_path: str):
    try:
        img_type = filetype.guess(file_path).mime.split('/')[-1]
    except:
        img_type = 'jpg'
    new_path = '.'.join((os.path.splitext(file_path)[0], img_type))
    try:
        os.rename(file_path, new_path)
        return 0
    except Exception as err:
        eprint("Couldn't rename file at path {}".format(file_path))
        eprint(err)
        return 1

DEBUG = False

# search url and browser headers for spoofing
headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/110.0'}
search_url = "https://www.google.com/search?q={}&tbm=isch&async=_id:islrg_c,_fmt:json&asearch=ichunklite&ijn={}"

def get_image_urls(query: str, page: int):
    all_images = {}
    try:
        response = requests.get(search_url.format(query, page), headers=headers)
        if DEBUG and response.status_code != 200:
            eprint("ERROR: Status code {} for page {}."
                   .format(response.status_code, page))
            
        if response.status_code == 200:
            json_text = response.content.decode('utf8').removeprefix(")]}'")
            json_data = json.loads(json_text)
            try:
                results = json_data['ichunklite']['results']
                for result in results:
                    id = result['image_docid']
                    image_url = result['viewer_metadata']['original_image']['url']
                    all_images[id] = image_url
            except Exception as err:
                if DEBUG:
                    eprint(f"ERROR: Issue parsing json file for page {page}")
                    eprint(err)
        else:
            if DEBUG:
                eprint("ERROR: Status code {} for page {}."
                       .format(response.status_code, page))
            
    except Exception as err:
        if DEBUG:
            eprint("ERROR: There was an issue requesting the json file")
            eprint(err)
    return all_images

def download_image(img_url: str, file_path: str):
    try:
        response = requests.get(img_url, headers=headers)
    except Exception as err:
        if DEBUG:
            eprint(f"ERROR: Can't request image url: {img_url}")
            eprint(err)
        return 1

    if response.status_code == 200:
        response_content_type = response.headers.get("content-type", '').lower()
        if "image" in response_content_type:
            with open(file_path, 'wb') as img:
                img.write(response.content)
            add_filetype(file_path)
            return 0
        else:
            if DEBUG:
                eprint("ERROR: Bad content-type\n{}"
                       .format(response_content_type))
            return 1
    else:
        if DEBUG:
            eprint("ERROR: Status code {} for url {}."
                   .format(response.status_code, url))
        return 1

def get_manifest(search_key: str, image_cnt: int):
    err_cnt = 0
    err_limit = 5

    img_manifest = {}
    manifest_len = image_cnt

    results_page = 0
    search_key = sanitize_query(search_key)
    while len(img_manifest.items()) < image_cnt:
        try:
            img_manifest.update(get_image_urls(search_key, results_page))
            results_page += 1
        except:
            print(f"err_cnt: {err_cnt}")
            err_cnt += 1
        if err_cnt > err_limit:
            eprint("Couldn't request all images")
            manifest_len = len(img_manifest.items())
            break

    print("Found {} of {} image sources".format(manifest_len, image_cnt))
    return list(img_manifest.items())[0:manifest_len]

def scrape_images(search_key, image_cnt, directory, threads):
    if DEBUG:
        print("savedir: {}".format(directory))
    if not os.path.exists(directory):
        os.makedirs(directory)

    id_url_manifest = get_manifest(search_key, image_cnt)
    with ThreadPoolExecutor(max_workers=threads) as pool:
        with tqdm(total=len(id_url_manifest)) as progress:
            futures = [] 
            for result in id_url_manifest:
                id: str = result[0]
                url: str = result[1]
                save_path = os.path.join(directory, id)
                future = pool.submit(download_image, url, save_path)
                future.add_done_callback(lambda p: progress.update(1))
                futures.append(future)
            for future in futures:
                result = future.result()
    return 0

if __name__ == "__main__":
    args = get_arguments(sys.argv)
    scrape_images(args.keyword[0], args.count, args.directory, args.threads)