Spaces:
Sleeping
Sleeping
File size: 477 Bytes
61e520f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 |
# utils.py
import time, urllib.parse, logging, random
from dateutil import parser as dateparser
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
def canonicalize_url(url, base=None):
if base:
return urllib.parse.urljoin(base, url)
return url
def parse_date(s):
try:
return dateparser.parse(s)
except Exception:
return None
def backoff_sleep(attempt):
time.sleep(min(2**attempt + random.random(), 10))
|