| import logging | |
| import shelve | |
| from pathlib import Path | |
| import pandas as pd | |
| logging.basicConfig(level=logging.INFO) | |
| def cached_download_csv(data_path: Path, url: str, **kwargs) -> pd.DataFrame: | |
| shelf_filename = data_path.joinpath("shelf.dat").as_posix() | |
| try: | |
| s = shelve.open(shelf_filename, writeback=True, flag="c") | |
| except Exception as exn: | |
| logging.exception( | |
| "Unexpected exception, while trying to access the shelf" | |
| ) | |
| return pd.read_csv(url, **kwargs) | |
| with s as shelf: | |
| logging.info(f"Opened or created the shelf file '{shelf_filename}'") | |
| maybe_cached = shelf.get(url) | |
| if maybe_cached is None: | |
| logging.info(f"Downloawding URL '{url}'") | |
| df = pd.read_csv(url, **kwargs) | |
| shelf[url] = df | |
| return df | |
| else: | |
| logging.info(f"Re-using cached URL '{url}'") | |
| assert isinstance(maybe_cached, pd.DataFrame) | |
| return maybe_cached | |
| # The context manager 'shelf' ensure set value as written back or return | |