File size: 1,068 Bytes
531597e 2695dfd 531597e 2695dfd 531597e 2695dfd 531597e 2695dfd 531597e 2695dfd 531597e 2695dfd 531597e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | import logging
import shelve
from pathlib import Path
import pandas as pd
logging.basicConfig(level=logging.INFO)
def cached_download_csv(data_path: Path, url: str, **kwargs) -> pd.DataFrame:
shelf_filename = data_path.joinpath("shelf.dat").as_posix()
try:
s = shelve.open(shelf_filename, writeback=True, flag="c")
except Exception as exn:
logging.exception(
"Unexpected exception, while trying to access the shelf"
)
return pd.read_csv(url, **kwargs)
with s as shelf:
logging.info(f"Opened or created the shelf file '{shelf_filename}'")
maybe_cached = shelf.get(url)
if maybe_cached is None:
logging.info(f"Downloawding URL '{url}'")
df = pd.read_csv(url, **kwargs)
shelf[url] = df
return df
else:
logging.info(f"Re-using cached URL '{url}'")
assert isinstance(maybe_cached, pd.DataFrame)
return maybe_cached
# The context manager 'shelf' ensure set value as written back or return
|