utils.py

from gevent import monkey
def stub(*args, **kwargs):  # pylint: disable=unused-argument
    pass
monkey.patch_all = stub
import grequests
import requests

import torch
device = "cuda" if torch.cuda.is_available() else "cpu"



def filter_invalid_urls(urls, photo_ids):
    rs = (grequests.get(u) for u in urls)
    results = grequests.map(rs)

    valid_image_ids = []
    valid_image_urls = []
    for i, res in enumerate(results):
        if res and res.status_code == 200:
            u = f"https://unsplash.com/photos/{photo_ids[i]}/download?w=320"
            valid_image_urls.append(u)
            valid_image_ids.append(photo_ids[i])

    return dict(
        image_ids=valid_image_ids,
        image_urls=valid_image_urls
    )