Implement all changes

app/pages/0_🌍_AOIs.py

@@ -2,7 +2,7 @@ import folium
 import streamlit as st
 from folium.plugins import Draw
 from src.config_parameters import params
-from src.gfm import create_aoi, delete_aoi, retrieve_all_aois
+from src.gfm import get_cached_aois, get_cached_gfm_handler
 from src.utils import (
     add_about,
     get_aoi_id_from_selector_preview,
@@ -44,7 +44,8 @@ radio_selection = st.radio(
 # call to render Folium map in Streamlit
 folium_map = folium.Map([39, 0], zoom_start=8)
 
-aois = retrieve_all_aois()
+gfm = get_cached_gfm_handler()
+aois = get_cached_aois()
 
 # See Areas will show all areas collected from GFM.
 # Collecting AOIs is done on first page load and when switching from a different radio selection back to See Areas
@@ -140,8 +141,8 @@ elif radio_selection == "Delete Area":
             confirm_delete = st.text_input("Enter area name")
             if st.button("Confirm"):
                 if confirm_delete == aoi_name:
-                    delete_aoi(selected_area_id)
-                    retrieve_all_aois.clear()
+                    gfm.delete_aoi(selected_area_id)
+                    get_cached_aois.clear()
                     st.toast("Area successfully deleted")
                     st.rerun()
                 else:
@@ -172,7 +173,7 @@ if save_area:
         print("starting to post new area name to gfm api")
         coordinates = selected_area_geojson["geometry"]["coordinates"]
 
-        create_aoi(new_area_name, coordinates)
+        gfm.create_aoi(new_area_name, coordinates)
         st.toast("Area successfully created")
 
 st.session_state["prev_page"] = "aois"

app/pages/1_💧_Flood_extent_analysis.py

@@ -1,18 +1,15 @@
 from datetime import date, timedelta
 
 import folium
+import pandas as pd
 import streamlit as st
 from src import hf_utils
 from src.config_parameters import params
-from src.gfm import (
-    download_flood_product,
-    get_area_products,
-    retrieve_all_aois,
-)
+from src.gfm import get_cached_aois, get_cached_gfm_handler
 from src.utils import (
     add_about,
     get_aoi_id_from_selector_preview,
-    get_existing_flood_geojson,
+    get_existing_geojson,
     set_tool_page_style,
     toggle_menu_button,
 )
@@ -40,12 +37,13 @@ set_tool_page_style()
 # Create two rows: top and bottom panel
 row1 = st.container()
 row2 = st.container()
-# Crate two columns in the top panel: input map and paramters
-col1, col2, col3, col4, col5 = row1.columns([1, 1, 1, 1, 1])
+# Create two columns in the top panel: input map and paramters
+col1, col2, col3, col4 = row1.columns([1, 1, 1, 2])
 col2_1, col2_2 = row2.columns([3, 2])
 
-# Retrieve AOIs to fill AOI selector
-aois = retrieve_all_aois()
+# Retrieve GFM Handler and AOIs to fill AOI selector
+gfm = get_cached_gfm_handler()
+aois = get_cached_aois()
 
 
 if "all_products" not in st.session_state:
@@ -88,71 +86,200 @@ with col4:
     it will not trigger any product downloads.
     """,
     )
-    show_available_products = st.button("Show available products")
+    show_available_products = st.button("Show GFM products")
 
 # If button above is triggered, get products from GFM
 # Then save all products to the session state and rerun the app to display them
 if show_available_products:
-    products = get_area_products(selected_area_id, start_date, end_date)
+    products = gfm.get_area_products(selected_area_id, start_date, end_date)
     st.session_state["all_products"] = products
     st.rerun()
 
 # Contains the product checkboxes if they exist after pushing the "Show available products" button
 with col2_2:
+    row_checkboxes = st.container()
+    row_buttons = st.container()
+
+with row_checkboxes:
     checkboxes = list()
     # Products are checked against the index to check whether they are already downloaded
     index_df = hf_utils.get_geojson_index_df()
     if st.session_state["all_products"]:
+        # Get unique product time groups
+        unique_time_groups = set()
         for product in st.session_state["all_products"]:
-            suffix = ""
-            if product["product_id"] in index_df["product"].values:
-                suffix = " - Already Downloaded"
-            checkbox = st.checkbox(product["product_time"] + suffix)
-            checkboxes.append(checkbox)
+            unique_time_groups.add(product["product_time_group"])
+
+        # Create dataframe for the table
+        product_data = []
+        for time_group in sorted(unique_time_groups):
+            # Check if any product in this group is already downloaded
+            products_in_group = [
+                p
+                for p in st.session_state["all_products"]
+                if p["product_time_group"] == time_group
+            ]
+
+            available_status = ""
+            for product in products_in_group:
+                if product["product_id"] in index_df["product"].values:
+                    available_status = "Available in Floodmap"
+                    flood_geojson_path = index_df.loc[
+                        index_df["product"] == product["product_id"],
+                        "flood_geojson_path",
+                    ].values[0]
+                    dataset_link = f"https://huggingface.co/datasets/rodekruis/flood-mapping/resolve/main/{flood_geojson_path}"
+
+            product_data.append(
+                {
+                    "Check": False,
+                    "Product time": time_group,
+                    "Available": dataset_link,
+                }
+            )
+
+        product_groups_df = pd.DataFrame(product_data)
+
+        # Create the data editor with checkbox column
+        product_groups_st_df = st.data_editor(
+            product_groups_df,
+            column_config={
+                "Check": st.column_config.CheckboxColumn(
+                    "Select",
+                    help="Select products to process",
+                    default=False,
+                ),
+                "Product time": st.column_config.TextColumn(
+                    "Product Time Group", disabled=True
+                ),
+                "Available": st.column_config.LinkColumn(
+                    "Available in dataset", display_text="Dataset link"
+                ),
+            },
+            hide_index=True,
+            disabled=["Product time", "Available"],
+        )
+
+        # Convert checkbox states to list for compatibility with existing code
+        checkboxes = product_groups_st_df["Check"].tolist()
+
+with row_buttons:
+    below_checkbox_col1, below_checkbox_col2 = row_buttons.columns([1, 1])
 
 # Contains the "Download Products" button
-with col5:
+with below_checkbox_col1:
     st.text(
         "Button info",
         help=""
         """
-    Will download the selected products (click "Show available products"
-    first if there are none). Products that show that they have already
-    been downloaded can be left checked, they will be skipped.
+    Will download the selected products from GFM to the Floodmap app
+    (click "Show available products" first if there are none).
+    Products that show that they have already been downloaded can be left checked,
+    they will be skipped.
     """,
     )
-    download_products = st.button("Download products")
+    download_products = st.button("Download to Floodmap")
 
     # If the button is clicked download all checked products that have not been downloaded yet
     if download_products:
         index_df = hf_utils.get_geojson_index_df()
-        for i, checkbox in enumerate(checkboxes):
-            if checkbox:
-                product_to_download = st.session_state["all_products"][i]
+        # Get selected time groups from the table
+        selected_time_groups = product_groups_st_df[product_groups_st_df["Check"]][
+            "Product time"
+        ].tolist()
+
+        # For each selected time group
+        for time_group in selected_time_groups:
+            # Get all products for this time group
+            products_in_group = [
+                p
+                for p in st.session_state["all_products"]
+                if p["product_time_group"] == time_group
+            ]
+
+            # Download each product in the group that hasn't been downloaded yet
+            for product_to_download in products_in_group:
                 if product_to_download["product_id"] not in index_df["product"].values:
                     with st.spinner(
                         f"Getting GFM files for {product_to_download['product_time']}, this may take a couple of minutes"
                     ):
-                        download_flood_product(selected_area_id, product_to_download)
+                        gfm.download_flood_product(
+                            selected_area_id, product_to_download
+                        )
         st.rerun()
 
 # For all the selected products add them to the map if they are available
 feature_groups = []
+selected_geojsons = []
 if st.session_state["all_products"]:
     index_df = hf_utils.get_geojson_index_df()
+    # Get unique time groups
+    unique_time_groups = sorted(
+        set(p["product_time_group"] for p in st.session_state["all_products"])
+    )
+
+    # For each checkbox (which corresponds to a time group)
     for i, checkbox in enumerate(checkboxes):
         if checkbox:
-            product_id = st.session_state["all_products"][i]["product_id"]
-            if product_id in index_df["product"].values:
-                feat_group_flood = get_existing_flood_geojson(product_id)
-                feature_groups.append(feat_group_flood)
+            time_group = unique_time_groups[i]
+            # Get all products for this time group
+            products_in_group = [
+                p
+                for p in st.session_state["all_products"]
+                if p["product_time_group"] == time_group
+            ]
+
+            # Create a feature group for this time group
+            flood_featuregroup = folium.FeatureGroup(name=time_group)
+            footprint_featuregroup = folium.FeatureGroup(name="Sentinel footprint")
+            group_has_features = False
+
+            # Add all available products from this group to the feature group
+            for product in products_in_group:
+                if product["product_id"] in index_df["product"].values:
+                    # Get the raw geojsons for further usage in the app
+                    flood_geojson = get_existing_geojson(product["product_id"], "flood")
+                    selected_geojsons.append(flood_geojson)
+                    # Convert geojsons to folium features to display on the map
+                    flood_folium_geojson = folium.GeoJson(
+                        flood_geojson,
+                        style_function=lambda x: {
+                            "fillColor": "#ff0000",
+                            "color": "#ff0000",
+                            "fillOpacity": 0.2,
+                        },
+                    )
+                    flood_featuregroup.add_child(flood_folium_geojson)
+
+                    footprint_geojson = get_existing_geojson(
+                        product["product_id"], "footprint"
+                    )
+                    footprint_folium_geojson = folium.GeoJson(
+                        footprint_geojson,
+                        style_function=lambda x: {
+                            "fillColor": "yellow",
+                            "color": "yellow",
+                            "fillOpacity": 0.2,
+                            "weight": 0,
+                        },
+                    )
+                    footprint_featuregroup.add_child(footprint_folium_geojson)
+                    group_has_features = True
+
+            # Only add the feature group if it contains any features
+            if group_has_features:
+                feature_groups.append(flood_featuregroup)
+                feature_groups.append(footprint_featuregroup)
 
 # Contains the map
 with col2_1:
     if selected_area_id:
         # display the bounding box
         bounding_box = aois[selected_area_id]["bbox"]
-        geojson_selected_area = folium.GeoJson(bounding_box)
+        geojson_selected_area = folium.GeoJson(
+            bounding_box,
+            style_function=lambda x: {"fillOpacity": 0.2, "weight": 1},
+        )
         feat_group_selected_area = folium.FeatureGroup(name="selected_area")
         feat_group_selected_area.add_child(geojson_selected_area)
         feature_groups.append(feat_group_selected_area)
@@ -162,11 +289,35 @@ with col2_1:
     folium_map.fit_bounds(feat_group_selected_area.get_bounds())
 
     m = st_folium(
-        folium_map,
-        width=800,
-        height=450,
-        feature_group_to_add=feature_groups,
+        folium_map, width=800, height=450, feature_group_to_add=feature_groups
     )
 
+    if flood_featuregroup:
+        flood_part_of_legend = """
+        <div style="display: flex; align-items: center;">
+            <div style="width: 20px; height: 20px; background:  rgba(255, 0, 0, .2); border: 1px solid red;"></div>
+            <div style="margin-left: 5px;">Floods</div>
+        </div>
+        <div style="display: flex; align-items: center;">
+            <div style="width: 20px; height: 20px; background:  rgba(255, 255, 0, .2); border: 1px solid yellow;"></div>
+            <div style="margin-left: 5px;">Sentinel Footprint</div>
+        </div>
+        """
+    else:
+        flood_part_of_legend = ""
+    st.markdown(
+        f"""
+        <div style="display: flex; align-items: center; gap: 20px;">
+            <div style="display: flex; align-items: center;">
+                <div style="width: 20px; height: 20px; background:  rgba(51, 136, 255, .2); border: 1px solid #3388ff;"></div>
+                <div style="margin-left: 5px;">AOI</div>
+            </div>
+            {flood_part_of_legend}
+        </div>
+        """,
+        unsafe_allow_html=True,
+    )
+
+
 # Keep track of which page we're currently on for page switch events
 st.session_state["prev_page"] = "flood_extent"

app/src/gfm.py

@@ -1,7 +1,7 @@
 import io
 import os
 import zipfile
-from pathlib import Path
+from datetime import datetime, timedelta
 
 import pandas as pd
 import requests
@@ -32,145 +32,182 @@ def get_gfm_user_and_token():
     return user_id, access_token
 
 
-# Gets all products for an AOI in a daterange
-def get_area_products(area_id, from_date, to_date):
-    user_id, access_token = get_gfm_user_and_token()
-    header = {"Authorization": f"bearer {access_token}"}
-
-    base_url = "https://api.gfm.eodc.eu/v1"
-
-    params = {
-        "time": "range",
-        "from": f"{from_date}T00:00:00",
-        "to": f"{to_date}T00:00:00",
-    }
-    prod_url = f"{base_url}/aoi/{area_id}/products"
-    response = requests.get(prod_url, headers=header, params=params)
-    products = response.json()["products"]
-    print(f"Found {len(products)} products for {area_id}")
-
-    return products
-
+class GFMHandler:
+    def __init__(self):
+        self.base_url = "https://api.gfm.eodc.eu/v1"
+        self.user_id, self.access_token = self._get_gfm_user_and_token()
+        self.header = {"Authorization": f"bearer {self.access_token}"}
+
+    def _get_gfm_user_and_token(self):
+        username = os.environ["gfm_username"]
+        password = os.environ["gfm_password"]
+
+        # Get token, setup header
+        token_url = f"{self.base_url}/auth/login"
+        payload = {"email": username, "password": password}
+
+        response = requests.post(token_url, json=payload)
+        user_id = response.json()["client_id"]
+        access_token = response.json()["access_token"]
+        print("retrieved user id and access token")
+
+        return user_id, access_token
+
+    def _refresh_token(self):
+        """Refresh the access token and update the authorization header"""
+        self.user_id, self.access_token = self._get_gfm_user_and_token()
+        self.header = {"Authorization": f"bearer {self.access_token}"}
+        print("Refreshed access token")
+
+    def _make_request(self, method, url, **kwargs):
+        """Make an API request with automatic token refresh on authentication failure"""
+        try:
+            response = requests.request(method, url, headers=self.header, **kwargs)
+            if response.status_code == 401:  # Unauthorized
+                print("Token expired, refreshing...")
+                self._refresh_token()
+                # Retry the request with new token
+                response = requests.request(method, url, headers=self.header, **kwargs)
+            response.raise_for_status()
+            return response
+        except requests.exceptions.RequestException as e:
+            print(f"Request failed: {e}")
+            raise
+
+    def get_area_products(self, area_id, from_date, to_date):
+        params = {
+            "time": "range",
+            "from": f"{from_date}T00:00:00",
+            "to": f"{to_date}T00:00:00",
+        }
+        prod_url = f"{self.base_url}/aoi/{area_id}/products"
+        response = self._make_request("GET", prod_url, params=params)
+        products = response.json()["products"]
+        print(f"Found {len(products)} products for {area_id}")
+
+        # Sort products by timestamp
+        products.sort(key=lambda x: x["product_time"])
+
+        # Group products that are within 1 minute of each other
+        if products:
+            current_group_time = products[0]["product_time"]
+            products[0]["product_time_group"] = current_group_time
+
+            for i in range(1, len(products)):
+                product_time = datetime.fromisoformat(
+                    products[i]["product_time"].replace("Z", "+00:00")
+                )
+                current_time = datetime.fromisoformat(
+                    current_group_time.replace("Z", "+00:00")
+                )
+                time_diff = product_time - current_time
+
+                # If more than 1 minute apart, start a new group
+                if time_diff > timedelta(minutes=1):
+                    current_group_time = products[i]["product_time"]
+
+                products[i]["product_time_group"] = current_group_time
+
+        return products
+
+    def download_flood_product(self, area_id, product):
+        product_id = product["product_id"]
+        product_time = product["product_time"]
+
+        print(f"Downloading product: {product_id}")
+
+        download_url = f"{self.base_url}/download/product/{product_id}"
+        response = self._make_request("GET", download_url)
+        download_link = response.json()["download_link"]
+
+        # Download and unzip file
+        r = requests.get(download_link)
+        buffer = io.BytesIO(r.content)
+        hf_api = hf_utils.get_hf_api()
+
+        data = {
+            "aoi_id": area_id,
+            "datetime": product_time,
+            "product": product_id,
+        }
 
-# Will download a product by product_id, saves the flood geojson and updates the index for caching
-def download_flood_product(area_id, product, output_file_path=None):
-    user_id, access_token = get_gfm_user_and_token()
-    header = {"Authorization": f"bearer {access_token}"}
+        with zipfile.ZipFile(buffer, "r") as z:
+            namelist = z.namelist()
 
-    base_url = "https://api.gfm.eodc.eu/v1"
+            for file_type in ["flood", "footprint"]:
+                filename = next(
+                    name
+                    for name in namelist
+                    if file_type in name.lower() and name.endswith(".geojson")
+                )
 
-    if not output_file_path:
-        base_file_path = "./output"
-
-    product_id = product["product_id"]
-    product_time = product["product_time"]
-
-    output_file_path = f"{base_file_path}"
-    Path(output_file_path).mkdir(parents=True, exist_ok=True)
-
-    print(f"Downloading product: {product_id}")
-
-    download_url = f"{base_url}/download/product/{product_id}"
-    response = requests.get(download_url, headers=header)
-    download_link = response.json()["download_link"]
-
-    # Download and unzip file
-    r = requests.get(download_link)
-    buffer = io.BytesIO(r.content)
-    hf_api = hf_utils.get_hf_api()
-
-    with zipfile.ZipFile(buffer, "r") as z:
-        namelist = z.namelist()
-        for name in namelist:
-            if "FLOOD" in name and ".geojson" in name:
-                flood_filename = name
-                path_in_repo = f"flood-geojson/{flood_filename}"
-                break
-        with z.open(flood_filename) as f:
-            hf_api.upload_file(
-                path_or_fileobj=f,
-                path_in_repo=path_in_repo,
-                repo_id="rodekruis/flood-mapping",
-                repo_type="dataset",
-            )
-
-    df = pd.DataFrame(
-        {
-            "aoi_id": [area_id],
-            "datetime": [product_time],
-            "product": [product_id],
-            "path_in_repo": [path_in_repo],
-        }
-    )
+                path_in_repo = f"flood-geojson/{filename}"
 
-    index_df = hf_utils.get_geojson_index_df()
-    index_df = pd.concat([index_df, df], ignore_index=True)
-    hf_utils.update_geojson_index_df(index_df)
+                with z.open(filename) as f:
+                    hf_api.upload_file(
+                        path_or_fileobj=f,
+                        path_in_repo=path_in_repo,
+                        repo_id="rodekruis/flood-mapping",
+                        repo_type="dataset",
+                    )
 
-    print(f"Product {product_id} downloaded succesfully")
+                data[f"{file_type}_geojson_path"] = path_in_repo
 
+        df = pd.DataFrame([data])
 
-# Gets all AOIs and transforms them to a dict with some features that are useful in the app, like the bbox
-@st.cache_resource
-def retrieve_all_aois():
-    print("Retrieving all AOIs from GFM API")
-    user_id, access_token = get_gfm_user_and_token()
-    header = {"Authorization": f"bearer {access_token}"}
+        index_df = hf_utils.get_geojson_index_df()
+        index_df = pd.concat([index_df, df], ignore_index=True)
+        hf_utils.update_geojson_index_df(index_df)
 
-    base_url = "https://api.gfm.eodc.eu/v1"
+        print(f"Product {product_id} downloaded succesfully")
 
-    aoi_url = f"{base_url}/aoi/user/{user_id}"
-    response = requests.get(aoi_url, headers=header)
+    def retrieve_all_aois(self):
+        print("Retrieving all AOIs from GFM API")
+        aoi_url = f"{self.base_url}/aoi/user/{self.user_id}"
+        response = self._make_request("GET", aoi_url)
 
-    aois = response.json()["aois"]
+        aois = response.json()["aois"]
 
-    aois = {
-        aoi["aoi_id"]: {
-            "name": aoi["aoi_name"],
-            "bbox": aoi["geoJSON"],
-            "name_id_preview": f"{aoi['aoi_name']} - {aoi['aoi_id'][:6]}...",
+        aois = {
+            aoi["aoi_id"]: {
+                "name": aoi["aoi_name"],
+                "bbox": aoi["geoJSON"],
+                "name_id_preview": f"{aoi['aoi_name']} - {aoi['aoi_id'][:6]}...",
+            }
+            for aoi in aois
         }
-        for aoi in aois
-    }
-
-    return aois
 
+        return aois
 
-# Creates AOI on GFM using API
-def create_aoi(new_area_name, coordinates):
-    user_id, access_token = get_gfm_user_and_token()
-    header = {"Authorization": f"bearer {access_token}"}
-
-    base_url = "https://api.gfm.eodc.eu/v1"
+    def create_aoi(self, new_area_name, coordinates):
+        print("Creating new area of impact")
+        create_aoi_url = f"{self.base_url}/aoi/create"
 
-    # Create area of impact
-    print("Creating new area of impact")
-    create_aoi_url = f"{base_url}/aoi/create"
+        payload = {
+            "aoi_name": new_area_name,
+            "description": new_area_name,
+            "user_id": self.user_id,
+            "geoJSON": {"type": "Polygon", "coordinates": coordinates},
+        }
 
-    payload = {
-        "aoi_name": new_area_name,
-        "description": new_area_name,
-        "user_id": user_id,
-        "geoJSON": {"type": "Polygon", "coordinates": coordinates},
-    }
+        self._make_request("POST", create_aoi_url, json=payload)
+        print("Posted new AOI")
 
-    r = requests.post(url=create_aoi_url, json=payload, headers=header)
-    r.raise_for_status()
-    print("Posted new AOI")
+    def delete_aoi(self, aoi_id):
+        print(f"Deleting area of impact {aoi_id}")
+        delete_aoi_url = f"{self.base_url}/aoi/delete/id/{aoi_id}"
+        print(delete_aoi_url)
 
+        self._make_request("DELETE", delete_aoi_url)
+        print("AOI deleted")
 
-# Deletes AOI on GFM using API
-def delete_aoi(aoi_id):
-    user_id, access_token = get_gfm_user_and_token()
-    header = {"Authorization": f"bearer {access_token}"}
 
-    base_url = "https://api.gfm.eodc.eu/v1"
+@st.cache_resource
+def get_cached_gfm_handler():
+    return GFMHandler()
 
-    # Create area of impact
-    print(f"Deleting area of impact {aoi_id}")
-    delete_aoi_url = f"{base_url}/aoi/delete/id/{aoi_id}"
-    print(delete_aoi_url)
 
-    r = requests.delete(url=delete_aoi_url, headers=header)
-    r.raise_for_status()
-    print("AOI deleted")
+@st.cache_resource
+def get_cached_aois():
+    gfm = get_cached_gfm_handler()
+    return gfm.retrieve_all_aois()

app/src/hf_utils.py

@@ -6,7 +6,7 @@ from huggingface_hub import HfApi
 
 
 @st.cache_resource
-def get_hf_api():
+def get_hf_api() -> HfApi:
     return HfApi()
 
 

app/src/utils.py

@@ -2,8 +2,8 @@
 
 import json
 import os
+from typing import Literal
 
-import folium
 import streamlit as st
 
 from src import hf_utils
@@ -157,15 +157,14 @@ def add_about():
     )
 
 
-def get_existing_flood_geojson(product_id):
+def get_existing_geojson(product_id, file_type: Literal["flood", "footprint"]):
     """
     Getting a saved GFM flood geojson in an output folder of GFM files. Merge in one feature group if multiple.
     """
     index_df = hf_utils.get_geojson_index_df()
-    path_in_repo = index_df[index_df["product"] == product_id].path_in_repo.values[0]
-
-    # Combine multiple flood files into a FeatureGroup
-    flood_geojson_group = folium.FeatureGroup(name=product_id)
+    path_in_repo = index_df[index_df["product"] == product_id][
+        f"{file_type}_geojson_path"
+    ].values[0]
 
     hf_api = hf_utils.get_hf_api()
     subfolder, filename = path_in_repo.split("/")
@@ -178,7 +177,5 @@ def get_existing_flood_geojson(product_id):
 
     with open(geojson_path, "r") as f:
         geojson_data = json.load(f)
-        flood_layer = folium.GeoJson(geojson_data)
-        flood_geojson_group.add_child(flood_layer)
 
-    return flood_geojson_group
+    return geojson_data