Restructure repo. Show map is functional. Calculation is not yet functional.

app.py

@@ -10,10 +10,10 @@ import pandas as pd
 import plotly.graph_objects as go
 import yaml
 import numpy as np
-from google.oauth2 import service_account
 
 
-from utils.js import get_window_url_params
+from utils.gradio import get_window_url_params
+from utils.duckdb_queries import list_projects_by_author, get_project_geometry
 
 # Logging
 logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO)
@@ -47,25 +47,32 @@ class IndexGenerator:
         self,
         centroid,
         roi_radius,
-        year,
         indices_file,
         project_name="",
         map=None,
     ):
+
+
+        # Authenticate to GEE & DuckDB
+        self._authenticate_ee(GEE_SERVICE_ACCOUNT)
+        self.con = self._get_duckdb_conn()
+
+
+        # Set instance variables
         self.indices = self._load_indices(indices_file)
         self.centroid = centroid
         self.roi = ee.Geometry.Point(*centroid).buffer(roi_radius)
-        self.year = year
-        self.start_date = str(datetime.date(self.year, 1, 1))
-        self.end_date = str(datetime.date(self.year, 12, 31))
-        self.daterange = [self.start_date, self.end_date]
-        self.project_name = project_name
+        # self.start_date = str(datetime.date(self.year, 1, 1))
+        # self.end_date = str(datetime.date(self.year, 12, 31))
+        # self.daterange = [self.start_date, self.end_date]
+        # self.project_name = project_name
         self.map = map
         if self.map is not None:
             self.show = True
         else:
             self.show = False
 
+ 
     def _cloudfree(self, gee_path):
         """
         Internal method to generate a cloud-free composite.
@@ -184,208 +191,195 @@ class IndexGenerator:
         df = pd.DataFrame(data)
         return df
 
+    @staticmethod
+    def _get_duckdb_conn():
+        logging.info("Configuring DuckDB connection...")
+        # use `climatebase` db
+        if not os.getenv("motherduck_token"):
+            raise Exception(
+                "No motherduck token found. Please set the `motherduck_token` environment variable."
+            )
+        else:
+            con = duckdb.connect("md:climatebase")
+            con.sql("USE climatebase;")
 
-def set_up_duckdb():
-    logging.info("set up duckdb")
-    # use `climatebase` db
-    if not os.getenv("motherduck_token"):
-        raise Exception(
-            "No motherduck token found. Please set the `motherduck_token` environment variable."
-        )
-    else:
-        con = duckdb.connect("md:climatebase")
-        con.sql("USE climatebase;")
-
-    # load extensions
-    con.sql("""INSTALL spatial; LOAD spatial;""")
-
-    return con
+        # load extensions
+        con.sql("""INSTALL spatial; LOAD spatial;""")
+        logging.info("Configured DuckDB connection.")
 
+        return con
 
-def authenticate_ee(ee_service_account):
-    """
-    Huggingface Spaces does not support secret files, therefore authenticate with an environment variable containing the JSON.
-    """
-    logging.info("authenticate_ee")
-    credentials = ee.ServiceAccountCredentials(
-        ee_service_account, key_data=os.environ["ee_service_account"]
-    )
-    ee.Initialize(credentials)
-
-
-def load_indices(indices_file):
-    # Read index configurations
-    with open(indices_file, "r") as stream:
-        try:
-            return yaml.safe_load(stream)
-        except yaml.YAMLError as e:
-            logging.error(e)
-            return None
-
-
-def create_dataframe(years, project_name):
-    dfs = []
-    logging.info(years)
-    indices = load_indices(INDICES_FILE)
-    for year in years:
-        logging.info(year)
-        ig = IndexGenerator(
-            centroid=LOCATION,
-            roi_radius=ROI_RADIUS,
-            year=year,
-            indices_file=INDICES_FILE,
-            project_name=project_name,
+    @staticmethod
+    def _authenticate_ee(ee_service_account):
+        """
+        Huggingface Spaces does not support secret files, therefore authenticate with an environment variable containing the JSON.
+        """
+        logging.info("Authenticating to Google Earth Engine...")
+        credentials = ee.ServiceAccountCredentials(
+            ee_service_account, key_data=os.environ["ee_service_account"]
         )
-        df = ig.generate_composite_index_df(list(indices.keys()))
-        dfs.append(df)
-    return pd.concat(dfs)
-
-
-# h/t: https://community.plotly.com/t/dynamic-zoom-for-mapbox/32658/12
-def get_plotting_zoom_level_and_center_coordinates_from_lonlat_tuples(
-    longitudes=None, latitudes=None
-):
-    """Function documentation:\n
-    Basic framework adopted from Krichardson under the following thread:
-    https://community.plotly.com/t/dynamic-zoom-for-mapbox/32658/7
-
-    # NOTE:
-    # THIS IS A TEMPORARY SOLUTION UNTIL THE DASH TEAM IMPLEMENTS DYNAMIC ZOOM
-    # in their plotly-functions associated with mapbox, such as go.Densitymapbox() etc.
+        ee.Initialize(credentials)
+        logging.info("Authenticated to Google Earth Engine.")
+
+    def _create_dataframe(self, years, project_name):
+        dfs = []
+        logging.info(years)
+        indices = self._load_indices(INDICES_FILE)
+        for year in years:
+            logging.info(year)
+            ig = IndexGenerator(
+                centroid=LOCATION,
+                roi_radius=ROI_RADIUS,
+                year=year,
+                indices_file=INDICES_FILE,
+                project_name=project_name,
+            )
+            df = ig.generate_composite_index_df(list(indices.keys()))
+            dfs.append(df)
+        return pd.concat(dfs)
+
+    # h/t: https://community.plotly.com/t/dynamic-zoom-for-mapbox/32658/12
+    def _latlon_to_config(
+        self,
+        longitudes=None, 
+        latitudes=None
+    ):
+        """Function documentation:\n
+        Basic framework adopted from Krichardson under the following thread:
+        https://community.plotly.com/t/dynamic-zoom-for-mapbox/32658/7
 
-    Returns the appropriate zoom-level for these plotly-mapbox-graphics along with
-    the center coordinate tuple of all provided coordinate tuples.
-    """
+        # NOTE:
+        # THIS IS A TEMPORARY SOLUTION UNTIL THE DASH TEAM IMPLEMENTS DYNAMIC ZOOM
+        # in their plotly-functions associated with mapbox, such as go.Densitymapbox() etc.
 
-    # Check whether both latitudes and longitudes have been passed,
-    # or if the list lenghts don't match
-    if (latitudes is None or longitudes is None) or (len(latitudes) != len(longitudes)):
-        # Otherwise, return the default values of 0 zoom and the coordinate origin as center point
-        return 0, (0, 0)
-
-    # Get the boundary-box
-    b_box = {}
-    b_box["height"] = latitudes.max() - latitudes.min()
-    b_box["width"] = longitudes.max() - longitudes.min()
-    b_box["center"] = (np.mean(longitudes), np.mean(latitudes))
-
-    # get the area of the bounding box in order to calculate a zoom-level
-    area = b_box["height"] * b_box["width"]
-
-    # * 1D-linear interpolation with numpy:
-    # - Pass the area as the only x-value and not as a list, in order to return a scalar as well
-    # - The x-points "xp" should be in parts in comparable order of magnitude of the given area
-    # - The zpom-levels are adapted to the areas, i.e. start with the smallest area possible of 0
-    # which leads to the highest possible zoom value 20, and so forth decreasing with increasing areas
-    # as these variables are antiproportional
-    zoom = np.interp(
-        x=area,
-        xp=[0, 5**-10, 4**-10, 3**-10, 2**-10, 1**-10, 1**-5],
-        fp=[20, 15, 14, 13, 12, 7, 5],
-    )
+        Returns the appropriate zoom-level for these plotly-mapbox-graphics along with
+        the center coordinate tuple of all provided coordinate tuples.
+        """
 
-    # Finally, return the zoom level and the associated boundary-box center coordinates
-    return zoom, b_box["center"]
-
-
-def show_project_map(project_name):
-    prepared_statement = con.execute(
-        "SELECT geometry FROM project WHERE name = ? LIMIT 1", [project_name]
-    ).fetchall()
-    features = json.loads(prepared_statement[0][0].replace("'", '"'))["features"]
-    geometry = features[0]["geometry"]
-    longitudes = np.array(geometry["coordinates"])[0, :, 0]
-    latitudes = np.array(geometry["coordinates"])[0, :, 1]
-    (
-        zoom,
-        bbox_center,
-    ) = get_plotting_zoom_level_and_center_coordinates_from_lonlat_tuples(
-        longitudes, latitudes
-    )
-    fig = go.Figure(
-        go.Scattermapbox(
-            mode="markers",
-            lon=[bbox_center[0]],
-            lat=[bbox_center[1]],
-            marker={"size": 20, "color": ["cyan"]},
+        # Check whether both latitudes and longitudes have been passed,
+        # or if the list lenghts don't match
+        if (latitudes is None or longitudes is None) or (
+            len(latitudes) != len(longitudes)
+        ):
+            # Otherwise, return the default values of 0 zoom and the coordinate origin as center point
+            return 0, (0, 0)
+
+        # Get the boundary-box
+        b_box = {}
+        b_box["height"] = latitudes.max() - latitudes.min()
+        b_box["width"] = longitudes.max() - longitudes.min()
+        b_box["center"] = (np.mean(longitudes), np.mean(latitudes))
+
+        # get the area of the bounding box in order to calculate a zoom-level
+        area = b_box["height"] * b_box["width"]
+
+        # * 1D-linear interpolation with numpy:
+        # - Pass the area as the only x-value and not as a list, in order to return a scalar as well
+        # - The x-points "xp" should be in parts in comparable order of magnitude of the given area
+        # - The zpom-levels are adapted to the areas, i.e. start with the smallest area possible of 0
+        # which leads to the highest possible zoom value 20, and so forth decreasing with increasing areas
+        # as these variables are antiproportional
+        zoom = np.interp(
+            x=area,
+            xp=[0, 5**-10, 4**-10, 3**-10, 2**-10, 1**-10, 1**-5],
+            fp=[20, 15, 14, 13, 12, 7, 5],
         )
-    )
-
-    fig.update_layout(
-        mapbox={
-            "style": "stamen-terrain",
-            "center": {"lon": bbox_center[0], "lat": bbox_center[1]},
-            "zoom": zoom,
-            "layers": [
-                {
-                    "source": {
-                        "type": "FeatureCollection",
-                        "features": [{"type": "Feature", "geometry": geometry}],
-                    },
-                    "type": "fill",
-                    "below": "traces",
-                    "color": "royalblue",
-                }
-            ],
-        },
-        margin={"l": 0, "r": 0, "b": 0, "t": 0},
-    )
-
-    return fig
-
 
-# minMax.getInfo()
-def calculate_biodiversity_score(start_year, end_year, project_name):
-    years = []
-    for year in range(start_year, end_year):
-        row_exists = con.execute(
-            "SELECT COUNT(1) FROM bioindicator WHERE (year = ? AND project_name = ?)",
-            [year, project_name],
-        ).fetchall()[0][0]
-        if not row_exists:
-            years.append(year)
-
-    if len(years) > 0:
-        df = create_dataframe(years, project_name)
+        # Finally, return the zoom level and the associated boundary-box center coordinates
+        return zoom, b_box["center"]
+
+    def show_project_map(self, project_name):
+        breakpoint()
+        prepared_statement = get_project_geometry(project_name)
+            # self.con.execute("SELECT geometry FROM project WHERE name = ? LIMIT 1", [project_name]).fetchall()
+        features = json.loads(prepared_statement[0][0].replace("'", '"'))["features"]
+        geometry = features[0]["geometry"]
+        longitudes = np.array(geometry["coordinates"])[0, :, 0]
+        latitudes = np.array(geometry["coordinates"])[0, :, 1]
+        zoom, bbox_center = self._latlon_to_config(longitudes, latitudes)
+        fig = go.Figure(
+            go.Scattermapbox(
+                mode="markers",
+                lon=[bbox_center[0]],
+                lat=[bbox_center[1]],
+                marker={"size": 20, "color": ["cyan"]},
+            )
+        )
 
-        # Write score table to `_temptable`
-        con.sql(
-            "CREATE OR REPLACE TABLE _temptable AS SELECT *, (value * area) AS score FROM (SELECT year, project_name, AVG(value) AS value, area  FROM df GROUP BY year, project_name, area ORDER BY project_name)"
+        fig.update_layout(
+            mapbox={
+                "style": "stamen-terrain",
+                "center": {"lon": bbox_center[0], "lat": bbox_center[1]},
+                "zoom": zoom,
+                "layers": [
+                    {
+                        "source": {
+                            "type": "FeatureCollection",
+                            "features": [{"type": "Feature", "geometry": geometry}],
+                        },
+                        "type": "fill",
+                        "below": "traces",
+                        "color": "royalblue",
+                    }
+                ],
+            },
+            margin={"l": 0, "r": 0, "b": 0, "t": 0},
         )
 
-        # Create `bioindicator` table IF NOT EXISTS.
-        con.sql(
+        return fig
+
+    def calculate_biodiversity_score(self, start_year, end_year, project_name):
+        years = []
+        for year in range(start_year, end_year):
+            row_exists = con.execute(
+                "SELECT COUNT(1) FROM bioindicator WHERE (year = ? AND project_name = ?)",
+                [year, project_name],
+            ).fetchall()[0][0]
+            if not row_exists:
+                years.append(year)
+
+        if len(years) > 0:
+            df = self._create_dataframe(years, project_name)
+
+            # Write score table to `_temptable`
+            self.con.sql(
+                "CREATE OR REPLACE TABLE _temptable AS SELECT *, (value * area) AS score FROM (SELECT year, project_name, AVG(value) AS value, area  FROM df GROUP BY year, project_name, area ORDER BY project_name)"
+            )
+
+            # Create `bioindicator` table IF NOT EXISTS.
+            self.con.sql(
+                """
+                USE climatebase;
+                CREATE TABLE IF NOT EXISTS bioindicator (year BIGINT, project_name VARCHAR(255), value DOUBLE, area DOUBLE, score DOUBLE, CONSTRAINT unique_year_project_name UNIQUE (year, project_name));
             """
-            USE climatebase;
-            CREATE TABLE IF NOT EXISTS bioindicator (year BIGINT, project_name VARCHAR(255), value DOUBLE, area DOUBLE, score DOUBLE, CONSTRAINT unique_year_project_name UNIQUE (year, project_name));
-        """
-        )
-        # UPSERT project record
-        con.sql(
+            )
+            # UPSERT project record
+            self.con.sql(
+                """
+                INSERT INTO bioindicator FROM _temptable
+                ON CONFLICT (year, project_name) DO UPDATE SET value = excluded.value;
             """
-            INSERT INTO bioindicator FROM _temptable
-            ON CONFLICT (year, project_name) DO UPDATE SET value = excluded.value;
-        """
-        )
-        logging.info("upsert records into motherduck")
-    scores = con.execute(
-        "SELECT * FROM bioindicator WHERE (year >= ? AND year <= ? AND project_name = ?)",
-        [start_year, end_year, project_name],
-    ).df()
-    return scores
+            )
+            logging.info("upsert records into motherduck")
+        scores = self.con.execute(
+            "SELECT * FROM bioindicator WHERE (year >= ? AND year <= ? AND project_name = ?)",
+            [start_year, end_year, project_name],
+        ).df()
+        return scores
+
+
+# Instantiate outside gradio app to avoid re-initializing GEE, which is slow
+indexgenerator = IndexGenerator(
+    centroid=LOCATION,
+    roi_radius=ROI_RADIUS,
+    indices_file=INDICES_FILE,
+)
 
+with gr.Blocks() as demo:
+    print("start gradio app")
 
-def motherduck_list_projects(author_id):
-    return con.execute(
-        "SELECT DISTINCT name FROM project WHERE authorId = ? AND geometry != 'null'",
-        [author_id],
-    ).df()
 
 
-with gr.Blocks() as demo:
-    # Environment setup
-    authenticate_ee(GEE_SERVICE_ACCOUNT)
-    con = set_up_duckdb()
     with gr.Column():
         m1 = gr.Plot()
         with gr.Row():
@@ -402,19 +396,19 @@ with gr.Blocks() as demo:
             label="Biodiversity scores by year",
         )
     calc_btn.click(
-        calculate_biodiversity_score,
+        indexgenerator.calculate_biodiversity_score,
         inputs=[start_year, end_year, project_name],
         outputs=results_df,
     )
     view_btn.click(
-        fn=show_project_map,
+        fn=indexgenerator.show_project_map,
         inputs=[project_name],
         outputs=[m1],
     )
 
     def update_project_dropdown_list(url_params):
         username = url_params.get("username", "default")
-        projects = motherduck_list_projects(author_id=username)
+        projects = list_projects_by_author(author_id=username)
         # to-do: filter projects based on user
         return gr.Dropdown.update(choices=projects["name"].tolist())
 

utils/duckdb_queries.py

@@ -0,0 +1,30 @@
+import os
+import duckdb
+
+import logging
+
+
+# Configure DuckDB connection
+logging.info("Configuring DuckDB connection...")
+
+if not os.getenv("motherduck_token"):
+    raise Exception(
+        "No motherduck token found. Please set the `motherduck_token` environment variable."
+    )
+else:
+    con = duckdb.connect("md:climatebase")
+    con.sql("USE climatebase;")
+
+# load extensions
+con.sql("""INSTALL spatial; LOAD spatial;""")
+logging.info("Configured DuckDB connection.")
+
+
+def list_projects_by_author(author_id):
+    return con.execute(
+        "SELECT DISTINCT name FROM project WHERE authorId = ? AND geometry != 'null'",
+        [author_id],
+    ).df()
+
+def get_project_geometry(project_name):
+    return con.execute("SELECT geometry FROM project WHERE name = ? LIMIT 1", [project_name]).fetchall()