whoa!

app.py

@@ -1,4 +1,5 @@
 import streamlit as st
+import leafmap.maplibregl as leafmap
 import pandas as pd
 import pydeck as pdk
 import numpy as np
@@ -7,13 +8,16 @@ from matplotlib import cm
 from PIL import Image
 import ibis
 from ibis import _
+from huggingface_hub import HfApi, login
 
-con = ibis.duckdb.connect(extensions=["spatial"])
+
+con = ibis.duckdb.connect(extensions=["spatial", "httpfs"])
 con.raw_sql('''
 INSTALL h3 FROM community;
 LOAD h3;
 ''')
 
+
 @ibis.udf.scalar.builtin
 def h3_latlng_to_cell(lat: float, lng: float, zoom: int) -> int:
     ...
@@ -32,17 +36,25 @@ st.set_page_config(page_title="GBIF Observations Explorer", page_icon=image, lay
 st.header("GBIF Observations Explorer", divider="rainbow")
 # st.set_page_config(page_title="H3 in Streamlit", layout="wide")
 
-h3_parquet = "https://data.source.coop/cboettig/gbif/gbif_us_h3.parquet"
-
-
-con = ibis.duckdb.connect(extensions=["spatial", "h3"])
+# h3_parquet = "https://data.source.coop/cboettig/gbif/gbif_us_h3.parquet"
+# h3_parquet = "/home/rstudio/source.coop/cboettig/gbif/gbif_us_h3.parquet"
+# h3_parquet = "hf://datasets/boettiger-lab/gbif/usa_h3/*/*.parquet"
+# h3_parquet = "hf://datasets/boettiger-lab/gbif/gbif_ca_h3.parquet"
+h3_parquet = "https://data.source.coop/cboettig/gbif/gbif_ca.geoparquet"
+#h3_parquet = "https://huggingface.co/datasets/boettiger-lab/gbif/resolve/main/gbif_ca.geoparquet"
+#h3_parquet = "gbif_ca.geoparquet"
 gbif_h3 = con.read_parquet(h3_parquet)
 
+# +
+# ibis doesn't know that duckdb can access hf://
+#con.raw_sql(f"CREATE OR REPLACE VIEW gbif AS SELECT * FROM read_parquet('{h3_parquet}')")
+#gbif_h3 = con.table("gbif")
+# -
 # ------ Visualisation 1 ---------
 
 
 # @st.cache_resource(ttl="2d")
-def filter_gbif(_df, species="Canis lupus", bbox = [-130., 0., -90., 40.]):
+def filter_gbif(_df, species="Canis lupus", bbox = [-130., 30., -90., 60.]):
      return (_df
             .filter(_.decimallongitude >= bbox[0],
                     _.decimallongitude < bbox[2],
@@ -59,7 +71,7 @@ def get_h3point_df(_df, resolution: float) -> pd.DataFrame:
             .rename(hex = column)
             .group_by(_.hex)
             .agg(n = _.count())
-            .mutate(wkt =  h3_cell_to_boundary_wkt(_.hex))
+       #     .mutate(wkt =  h3_cell_to_boundary_wkt(_.hex))
             .mutate(v = _.n.log())
             .mutate(normalized_values = _.v / _.v.max())
             .to_pandas()
@@ -67,31 +79,74 @@ def get_h3point_df(_df, resolution: float) -> pd.DataFrame:
     rgb = cm.viridis(df.normalized_values)
     rgb_array = np.round( rgb * 255 ).astype(int).clip(0,255).tolist()
     df['rgb'] = rgb_array
-    #df['viridis_hex'] = colors.to_hex(rgb)  
+    #df['viridis_hex'] = colors.to_hex(rgb)  # not robust?
     df['viridis_hex'] = [f"#{int(c[0] * 255):02x}{int(c[1] * 255):02x}{int(c[2] * 255):02x}" for c in rgb]    
     return df
 
+# +
+# #%%time
+#df = filter_gbif(gbif_h3, species = "Canis lupus")
+#df = get_h3point_df(df, 6)
+#df.head(10)
+
+# +
 
-def get_coverage_layer(df: pd.DataFrame, v_scale = 0) -> pdk.Layer:
-    return pdk.Layer(
-        "H3HexagonLayer",
-        df,
-        get_hexagon="hex",
-        filled=True,
-        auto_highlight=True,
-        get_fill_color="rgb",
-        get_elevation="normalized_values",
-        elevation_scale=5000 * 10 ** v_scale,
-        elevation_range=[0,1],
-        pickable=True,
-        extruded=True,
-        line_width_min_pixels=1,
-    )
 
+def host_df(df, filename = "live.json", repo_id="boettiger-lab/gbif"):
+    df.to_json(".static/"+filename, orient='records', indent=2)
+    from huggingface_hub import HfApi, login
+    api = HfApi()
+    info = api.upload_file(
+        path_or_fileobj=".static/"+filename,
+        path_in_repo="live/" + filename,
+        repo_id=repo_id,
+        repo_type="dataset",
+    )
+    commit_hash = info.oid
+    return  f"https://huggingface.co/datasets/{repo_id}/resolve/{commit_hash}/live/{filename}"
+    
+    #from minio import Minio
+    #key = st.secrets["MINIO_KEY"]
+    #secret = st.secrets["MINIO_SECRET"]
+    #client = Minio("minio.carlboettiger.info", key, secret)
+    #client.fput_object("public-biodiversity", name, name)
+    return info
+    
+def hex_layer(m, df: pd.DataFrame, v_scale = 1):
+    url = host_df(df)
+    
+    deck_grid_layer = {
+        "@@type": "H3HexagonLayer",
+        "id": "my-layer",
+        "data": url,
+        "getHexagon": "@@=hex",
+        "getFillColor": "@@=rgb",
+        "getElevation": "@@=normalized_values",
+        "elevationScale": 5000 * 10 ** v_scale,
+        "elevationRange": [0,1],
+    }
+    return m.add_deck_layers([deck_grid_layer])
+
+
+# +
+# #%%time
+def local_test():
+    bbox = [-120, 37, -118, 39]
+    df = filter_gbif(gbif_h3, species = "Canis lupus", bbox = bbox)
+    df = get_h3point_df(df, 6)
+    m = leafmap.Map(style="positron",  center=(-121.4, 37.74), zoom=7,)
+    hex_layer(m, df)
+    return m
+    
+
+#local_test()
+# -
+
+m = leafmap.Map(style="positron",  center=(-121.4, 37.74), zoom=7,)
 
 col1, col2, col3 = st.columns(3)
 with col1:
-    h3_resolut_1 = st.slider("H3 resolution", min_value=0, max_value=12, value=5)
+    h3_resolut_1 = st.slider("H3 resolution", min_value=2, max_value=11, value=5)
     v_scale = st.slider("vertical scale", min_value=-1, max_value=3, value=1)
 
 with col2:
@@ -99,13 +154,13 @@ with col2:
                                  "Select longitude range",
                                  min_value=-130.0,
                                  max_value=-65.0,
-                                 value=(-120.0, -115.0),  # Default selected range
+                                 value=(-128.0, -115.0),  # Default selected range
                                  step=0.1)
     min_lat, max_lat = st.slider(
                                  "Select latitude range",
-                                 min_value=0.0,
-                                 max_value=50.0,
-                                 value=(36., 39.0),  # Default selected range
+                                 min_value=20.0,
+                                 max_value=70.0,
+                                 value=(30.0, 42.0),  # Default selected range
                                  step=0.1)
 
 with col3:
@@ -115,22 +170,12 @@ bbox = [min_lng, min_lat, max_lng, max_lat]
 center_lat = (bbox[1] + bbox[3]) / 2
 center_lng = (bbox[0] + bbox[2]) / 2
 
+# +
 df = filter_gbif(gbif_h3, species, bbox)
 df = get_h3point_df(df, h3_resolut_1)
-layer_coverage_1 = get_coverage_layer(df, v_scale)
-visible_layers_coverage_1 = [layer_coverage_1]
-
-st.pydeck_chart(
-    pdk.Deck(map_provider='carto', 
-        map_style='light',
-        initial_view_state=pdk.ViewState(
-            latitude=center_lat, longitude=center_lng, zoom=h3_resolut_1, height=600
-        ),
-        tooltip={"html": "<b>Count:</b> {n}", "style": {"color": "white"}},
-        layers=visible_layers_coverage_1,
-    )
-)
 
+hex_layer(m, df)
 
+m.to_streamlit()
 
 

requirements.txt

@@ -9,4 +9,4 @@ referencing==0.35.1
 rasterio==1.3.10
 shapely==2.0.4
 shiny==0.10.2
-requests
+huggingface_hub