{ "cells": [ { "cell_type": "code", "execution_count": 9, "id": "b107913b-6509-4e81-910c-d85c6b2d21a0", "metadata": {}, "outputs": [], "source": [ "import os\n", "from pathlib import Path\n", "\n", "import numpy as np\n", "import pandas as pd\n", "\n", "import shelve" ] }, { "cell_type": "code", "execution_count": 3, "id": "101058a0-be75-469e-888e-ca30085b3728", "metadata": {}, "outputs": [], "source": [ "from athai.data_utils import cached_download_csv" ] }, { "cell_type": "code", "execution_count": 4, "id": "ff4bfe45-7c3a-4f2a-8bb1-1c0949e59439", "metadata": {}, "outputs": [], "source": [ "DATE_COLUMN = \"date/time\"\n", "DATA_URL = (\n", " \"https://s3-us-west-2.amazonaws.com/\"\n", " \"streamlit-demo-data/uber-raw-data-sep14.csv.gz\"\n", ")\n", "\n", "DATA_PATH = Path(os.environ.get(\"APP_DATA\"))\n" ] }, { "cell_type": "code", "execution_count": 5, "id": "740320f0-ac83-4a7b-8087-f930e9e43da4", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:Opened or created the shelf file '/data/app/shelf.dat'\n", "INFO:root:Re-using cached URL 'https://s3-us-west-2.amazonaws.com/streamlit-demo-data/uber-raw-data-sep14.csv.gz'\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Date/TimeLatLonBase
09/1/2014 0:01:0040.2201-74.0021B02512
19/1/2014 0:01:0040.7500-74.0027B02512
29/1/2014 0:03:0040.7559-73.9864B02512
39/1/2014 0:06:0040.7450-73.9889B02512
49/1/2014 0:11:0040.8145-73.9444B02512
...............
99959/8/2014 18:15:0040.7194-74.0000B02512
99969/8/2014 18:15:0040.7426-74.0079B02512
99979/8/2014 18:16:0040.7358-73.9758B02512
99989/8/2014 18:16:0040.7385-73.9952B02512
99999/8/2014 18:16:0040.7279-73.9961B02512
\n", "

10000 rows × 4 columns

\n", "
" ], "text/plain": [ " Date/Time Lat Lon Base\n", "0 9/1/2014 0:01:00 40.2201 -74.0021 B02512\n", "1 9/1/2014 0:01:00 40.7500 -74.0027 B02512\n", "2 9/1/2014 0:03:00 40.7559 -73.9864 B02512\n", "3 9/1/2014 0:06:00 40.7450 -73.9889 B02512\n", "4 9/1/2014 0:11:00 40.8145 -73.9444 B02512\n", "... ... ... ... ...\n", "9995 9/8/2014 18:15:00 40.7194 -74.0000 B02512\n", "9996 9/8/2014 18:15:00 40.7426 -74.0079 B02512\n", "9997 9/8/2014 18:16:00 40.7358 -73.9758 B02512\n", "9998 9/8/2014 18:16:00 40.7385 -73.9952 B02512\n", "9999 9/8/2014 18:16:00 40.7279 -73.9961 B02512\n", "\n", "[10000 rows x 4 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "nrows=10\n", "data = cached_download_csv(DATA_PATH, DATA_URL, nrows=nrows)\n", "data" ] }, { "cell_type": "code", "execution_count": 7, "id": "35a636c3-a1cd-40df-8bd2-9f008046a99f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'/data/app/shelf.dat'" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "shelf_filename = DATA_PATH.joinpath(\"shelf.dat\").as_posix()\n", "shelf_filename" ] }, { "cell_type": "code", "execution_count": 10, "id": "a2dfa021-231c-4fcb-9928-66e69ef802d5", "metadata": {}, "outputs": [], "source": [ "shelf = shelve.open(shelf_filename, writeback=True, flag=\"c\")" ] }, { "cell_type": "code", "execution_count": 12, "id": "072a0f31-7e76-4187-9206-2ed108645709", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['https://s3-us-west-2.amazonaws.com/streamlit-demo-data/uber-raw-data-sep14.csv.gz']" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "list(shelf.keys())" ] }, { "cell_type": "code", "execution_count": 13, "id": "44347fa2-132e-4f3a-9229-05acb5173cfc", "metadata": {}, "outputs": [], "source": [ "stuff = [ 1, \"two\", {3, 4, 5} ]" ] }, { "cell_type": "code", "execution_count": 14, "id": "c9599633-9edf-427a-a8af-9cba912d90cc", "metadata": {}, "outputs": [], "source": [ "shelf[\"whatever\"] = stuff" ] }, { "cell_type": "code", "execution_count": 15, "id": "a7be5ccd-4191-4c8f-a1dd-bce1b5885945", "metadata": {}, "outputs": [], "source": [ "shelf.sync()" ] }, { "cell_type": "code", "execution_count": 16, "id": "5f2979f3-3228-4d98-8e82-9c2e700c8312", "metadata": {}, "outputs": [], "source": [ "shelf.close()" ] }, { "cell_type": "code", "execution_count": null, "id": "7d4750ff-d7b1-4bea-98ac-027b6f8e10f0", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 5 }