refactor: extract cached_download()

src/app.py

@@ -1,9 +1,10 @@
 import numpy as np
 import pandas as pd
 import streamlit as st
-import shelve
 import os
-import pathlib
+from pathlib import Path
+
+from athai.data_utils import cached_download_csv
 
 st.title("Uber pickups in NYC")
 
@@ -13,32 +14,11 @@ DATA_URL = (
     "streamlit-demo-data/uber-raw-data-sep14.csv.gz"
 )
 
-DATA_PATH = pathlib.Path(os.environ.get("APP_DATA"))
-
-def cached_download_csv(url : str, **kwargs) -> pd.DataFrame:
-    shelf_filename = DATA_PATH.joinpath("shelf.dat").as_posix()
-    try:
-        s = shelve.open(shelf_filename, writeback=True, flag='c')
-    except Exception as exn:
-        st.warning("Unexpected exception, by-passing persistent cache :\n" + exn.with_traceback())
-        return pd.read_csv(url, **kwargs)
-    with s as shelf:
-        st.info(f"Opened or created the shelf file '{shelf_filename}'")
-        maybe_cached = shelf.get(url)
-        if maybe_cached is None:
-            st.info(f"Downloawding URL '{url}'")
-            df = pd.read_csv(url, **kwargs)
-            shelf[url] = df
-            return df
-        else :  
-            st.info(f"Re-using cached URL '{url}'")
-            assert isinstance(maybe_cached, pd.DataFrame)
-            return maybe_cached
-    # The context manager 'shelf' ensure set value as written back or return 
+DATA_PATH = Path(os.environ.get("APP_DATA"))
 
 @st.cache_resource
 def load_data(nrows):
-    data = cached_download_csv(DATA_URL, nrows=nrows)
+    data = cached_download_csv(DATA_PATH, DATA_URL,nrows=nrows)
     def lowercase(x):
         return str(x).lower()
 

src/athai/data_utils.py

@@ -0,0 +1,30 @@
+from pathlib import Path
+import pandas as pd
+import logging
+
+import shelve
+
+logging.basicConfig(level=logging.INFO)
+
+def cached_download_csv(data_path : Path, url : str, **kwargs) -> pd.DataFrame:
+    shelf_filename = data_path.joinpath("shelf.dat").as_posix()
+    try:
+        s = shelve.open(shelf_filename, writeback=True, flag='c')
+    except Exception as exn:
+        logging.warning("Unexpected exception, by-passing persistent cache :\n" + str())
+        logging.exception("Unexpected exception, while trying to access the shelf")
+        return pd.read_csv(url, **kwargs)
+    with s as shelf:
+        logging.info(f"Opened or created the shelf file '{shelf_filename}'")
+        maybe_cached = shelf.get(url)
+        if maybe_cached is None:
+            logging.info(f"Downloawding URL '{url}'")
+            df = pd.read_csv(url, **kwargs)
+            shelf[url] = df
+            return df
+        else :  
+            logging.info(f"Re-using cached URL '{url}'")
+            assert isinstance(maybe_cached, pd.DataFrame)
+            return maybe_cached
+    # The context manager 'shelf' ensure set value as written back or return 
+