Anthony Truchet commited on
Commit ·
531597e
1
Parent(s): f92ad8c
refactor: run pre-commit hooks for formating
Browse files- .gitignore +1 -1
- doc/_autosummary/athai.rst +1 -0
- packages.txt +1 -1
- scripts/build-run-local-docker.sh +4 -4
- src/apps/streamlit_demo.py +8 -5
- src/athai/data_utils.py +14 -9
.gitignore
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
|
| 2 |
|
| 3 |
.envrc
|
| 4 |
.idea
|
|
|
|
| 1 |
+
doc/_autosummary
|
| 2 |
|
| 3 |
.envrc
|
| 4 |
.idea
|
doc/_autosummary/athai.rst
CHANGED
|
@@ -27,5 +27,6 @@
|
|
| 27 |
:toctree:
|
| 28 |
:recursive:
|
| 29 |
|
|
|
|
| 30 |
athai.hello
|
| 31 |
|
|
|
|
| 27 |
:toctree:
|
| 28 |
:recursive:
|
| 29 |
|
| 30 |
+
athai.data_utils
|
| 31 |
athai.hello
|
| 32 |
|
packages.txt
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
nano
|
| 2 |
vim
|
| 3 |
-
tree
|
|
|
|
| 1 |
nano
|
| 2 |
vim
|
| 3 |
+
tree
|
scripts/build-run-local-docker.sh
CHANGED
|
@@ -12,8 +12,8 @@ echo "Running from ROOT_DIRECTORY: ${ROOT_DIRECTORY}"
|
|
| 12 |
# Export the dependencies managed by Poetry in the requirements.txt format for pip
|
| 13 |
poetry export -f requirements.txt -o requirements.txt --without-hashes --only=main
|
| 14 |
|
| 15 |
-
# Re-build the docker image.
|
| 16 |
-
docker build --label tests -t $CONTAINER_NAME .
|
| 17 |
|
| 18 |
# During development ONLY use a `bind mount` to enable
|
| 19 |
# editing the code without having to rebuild the container.
|
|
@@ -23,6 +23,6 @@ docker run --rm -it \
|
|
| 23 |
--mount type=bind,source=${ROOT_DIRECTORY}/src/,target=/app/,readonly \
|
| 24 |
--mount type=bind,source=${ROOT_DIRECTORY}/.streamlit,target=/user/.streamlit,readonly \
|
| 25 |
$CONTAINER_NAME:latest \
|
| 26 |
-
$@
|
| 27 |
|
| 28 |
-
popd
|
|
|
|
| 12 |
# Export the dependencies managed by Poetry in the requirements.txt format for pip
|
| 13 |
poetry export -f requirements.txt -o requirements.txt --without-hashes --only=main
|
| 14 |
|
| 15 |
+
# Re-build the docker image.
|
| 16 |
+
docker build --label tests -t $CONTAINER_NAME .
|
| 17 |
|
| 18 |
# During development ONLY use a `bind mount` to enable
|
| 19 |
# editing the code without having to rebuild the container.
|
|
|
|
| 23 |
--mount type=bind,source=${ROOT_DIRECTORY}/src/,target=/app/,readonly \
|
| 24 |
--mount type=bind,source=${ROOT_DIRECTORY}/.streamlit,target=/user/.streamlit,readonly \
|
| 25 |
$CONTAINER_NAME:latest \
|
| 26 |
+
$@
|
| 27 |
|
| 28 |
+
popd
|
src/apps/streamlit_demo.py
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import numpy as np
|
| 2 |
import pandas as pd
|
| 3 |
import streamlit as st
|
| 4 |
-
import os
|
| 5 |
-
from pathlib import Path
|
| 6 |
|
| 7 |
from athai.data_utils import cached_download_csv
|
| 8 |
|
|
|
|
| 9 |
st.title("Uber pickups in NYC")
|
| 10 |
|
| 11 |
DATE_COLUMN = "date/time"
|
|
@@ -16,10 +18,11 @@ DATA_URL = (
|
|
| 16 |
|
| 17 |
DATA_PATH = Path(os.environ.get("APP_DATA"))
|
| 18 |
|
|
|
|
| 19 |
@st.cache_resource
|
| 20 |
def load_data(nrows):
|
| 21 |
-
data = cached_download_csv(DATA_PATH, DATA_URL,
|
| 22 |
-
|
| 23 |
def lowercase(x):
|
| 24 |
return str(x).lower()
|
| 25 |
|
|
@@ -56,4 +59,4 @@ if uploaded_file is not None:
|
|
| 56 |
st.write(len(bytes_data), "bytes")
|
| 57 |
|
| 58 |
|
| 59 |
-
st.markdown("")
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
import numpy as np
|
| 5 |
import pandas as pd
|
| 6 |
import streamlit as st
|
|
|
|
|
|
|
| 7 |
|
| 8 |
from athai.data_utils import cached_download_csv
|
| 9 |
|
| 10 |
+
|
| 11 |
st.title("Uber pickups in NYC")
|
| 12 |
|
| 13 |
DATE_COLUMN = "date/time"
|
|
|
|
| 18 |
|
| 19 |
DATA_PATH = Path(os.environ.get("APP_DATA"))
|
| 20 |
|
| 21 |
+
|
| 22 |
@st.cache_resource
|
| 23 |
def load_data(nrows):
|
| 24 |
+
data = cached_download_csv(DATA_PATH, DATA_URL, nrows=nrows)
|
| 25 |
+
|
| 26 |
def lowercase(x):
|
| 27 |
return str(x).lower()
|
| 28 |
|
|
|
|
| 59 |
st.write(len(bytes_data), "bytes")
|
| 60 |
|
| 61 |
|
| 62 |
+
st.markdown("")
|
src/athai/data_utils.py
CHANGED
|
@@ -1,18 +1,24 @@
|
|
|
|
|
|
|
|
| 1 |
from pathlib import Path
|
|
|
|
| 2 |
import pandas as pd
|
| 3 |
-
import logging
|
| 4 |
|
| 5 |
-
import shelve
|
| 6 |
|
| 7 |
logging.basicConfig(level=logging.INFO)
|
| 8 |
|
| 9 |
-
|
|
|
|
| 10 |
shelf_filename = data_path.joinpath("shelf.dat").as_posix()
|
| 11 |
try:
|
| 12 |
-
s = shelve.open(shelf_filename, writeback=True, flag=
|
| 13 |
except Exception as exn:
|
| 14 |
-
logging.warning(
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
return pd.read_csv(url, **kwargs)
|
| 17 |
with s as shelf:
|
| 18 |
logging.info(f"Opened or created the shelf file '{shelf_filename}'")
|
|
@@ -22,9 +28,8 @@ def cached_download_csv(data_path : Path, url : str, **kwargs) -> pd.DataFrame:
|
|
| 22 |
df = pd.read_csv(url, **kwargs)
|
| 23 |
shelf[url] = df
|
| 24 |
return df
|
| 25 |
-
else
|
| 26 |
logging.info(f"Re-using cached URL '{url}'")
|
| 27 |
assert isinstance(maybe_cached, pd.DataFrame)
|
| 28 |
return maybe_cached
|
| 29 |
-
# The context manager 'shelf' ensure set value as written back or return
|
| 30 |
-
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import shelve
|
| 3 |
from pathlib import Path
|
| 4 |
+
|
| 5 |
import pandas as pd
|
|
|
|
| 6 |
|
|
|
|
| 7 |
|
| 8 |
logging.basicConfig(level=logging.INFO)
|
| 9 |
|
| 10 |
+
|
| 11 |
+
def cached_download_csv(data_path: Path, url: str, **kwargs) -> pd.DataFrame:
|
| 12 |
shelf_filename = data_path.joinpath("shelf.dat").as_posix()
|
| 13 |
try:
|
| 14 |
+
s = shelve.open(shelf_filename, writeback=True, flag="c")
|
| 15 |
except Exception as exn:
|
| 16 |
+
logging.warning(
|
| 17 |
+
"Unexpected exception, by-passing persistent cache :\n" + str(exn)
|
| 18 |
+
)
|
| 19 |
+
logging.exception(
|
| 20 |
+
"Unexpected exception, while trying to access the shelf"
|
| 21 |
+
)
|
| 22 |
return pd.read_csv(url, **kwargs)
|
| 23 |
with s as shelf:
|
| 24 |
logging.info(f"Opened or created the shelf file '{shelf_filename}'")
|
|
|
|
| 28 |
df = pd.read_csv(url, **kwargs)
|
| 29 |
shelf[url] = df
|
| 30 |
return df
|
| 31 |
+
else:
|
| 32 |
logging.info(f"Re-using cached URL '{url}'")
|
| 33 |
assert isinstance(maybe_cached, pd.DataFrame)
|
| 34 |
return maybe_cached
|
| 35 |
+
# The context manager 'shelf' ensure set value as written back or return
|
|
|