Added GeoTIFF support and utilities page

pyproject.toml

@@ -18,7 +18,8 @@ dependencies = [
     "scipy", 
     "pandas",
     "networkx",
-    "openpyxl"
+    "openpyxl",
+    "rasterio"
 ]
 
 [tool.hatch.version]

requirements.txt

@@ -6,4 +6,5 @@ lorem_text
 matplotlib
 psycopg2-binary 
 scipy
-pandas
+pandas
+rasterio

tomorrowcities/content/articles/data_formats.md

@@ -49,5 +49,8 @@ where
 * **residents (integer)** stores the number of individual live in the building.
 * **expStr (string)** 
 
+### Intensity Measures
+Whether it is flood, debris or earthquake, every hazard map should contain at least two properties: a point geometry and intensity measure denotes by 'im'. The data can be provided via GeoTIFF or GeoJSON format. TIFF files should contain CRS 
+information so that the engine could map the coordinates to a common CRS to conduct calculations.
 
 

tomorrowcities/content/articles/welcome.md

@@ -15,6 +15,13 @@ category:
 TCDSE is a web application designed to conduct computational tasks to generate information needed for decision mechanisms in designing future cities. The web application, which will be referred as TCDSE for short, contains a computational engine capable of executing several hazard scenarios on different exposure datasets and infrastructures. 
 
 ## What is New?
+* basemap is changed to ESri.WorldImagery to see the landscapes especially rivers.
+* utilities page is added.
+* Excel to GeoJSON converted is added to utilities page.
+* GeoTIFF support is added. 
+* When an intensity layer is added via GeoTIFF format, only the non-zero intensity measure are retained by the engine.
+* In map visualization of intensity layer, only the largest 500k points are displayed to render the map faster.
+* rasterio.transform.xy function is replaced with a faster local implementation.
 * Info box is added next to the map to see overall information and building/landuse details
 when clicked.
 * Implementation Capacity Score is added. If medium or low is selected, then building-level metrics is increased by 25% and 50%, respectively. If high is selected, there is no change in the metrics.

tomorrowcities/pages/__init__.py

@@ -5,7 +5,7 @@ import dataclasses
 
 from ..data import articles
 
-route_order = ["/", "docs","engine","settings","account"]
+route_order = ["/", "docs","engine","utilities","settings","account"]
 
 def check_auth(route, children):
     # This can be replaced by a custom function that checks if the user is
@@ -13,7 +13,7 @@ def check_auth(route, children):
 
     # routes that are public or only for admin
     # the rest only requires login
-    public_paths = ["/","docs","engine","account"]
+    public_paths = ["/","docs","engine","utilities","account"]
     admin_paths = ["settings"]
 
 

tomorrowcities/pages/engine.py

@@ -10,6 +10,9 @@ from typing import Tuple, Optional
 import ipyleaflet
 from ipyleaflet import AwesomeIcon, Marker
 import numpy as np
+import rasterio 
+from rasterio.warp import calculate_default_transform, reproject, Resampling
+import io
 import logging, sys
 #logging.basicConfig(stream=sys.stderr, level=logging.INFO)
 
@@ -96,9 +99,7 @@ layers = solara.reactive({
             'force_render': solara.reactive(False),
             'visible': solara.reactive(False),
             'extra_cols': {'ds': 0, 'is_damaged': False, 'is_operational': True},
-            'cols_required': set(['geometry', 'fltytype', 'strctype', 'utilfcltyc', 'indpnode', 'guid', 
-                         'node_id', 'x_coord', 'y_coord', 'pwr_plant', 'serv_area', 'n_bldgs', 
-                         'income', 'eq_vuln']),
+            'cols_required': set(['geometry', 'node_id', 'pwr_plant', 'n_bldgs', 'eq_vuln']),
             'cols': set(['geometry', 'fltytype', 'strctype', 'utilfcltyc', 'indpnode', 'guid', 
                          'node_id', 'x_coord', 'y_coord', 'pwr_plant', 'serv_area', 'n_bldgs', 
                          'income', 'eq_vuln'])},
@@ -110,8 +111,7 @@ layers = solara.reactive({
             'force_render': solara.reactive(False),
             'visible': solara.reactive(False),
             'extra_cols': {},
-            'cols_required': set(['from_node', 'direction', 'pipetype', 'edge_id', 'guid', 'capacity', 
-                         'geometry', 'to_node', 'length']),
+            'cols_required': set(['geometry','from_node','to_node', 'edge_id']),
             'cols': set(['from_node', 'direction', 'pipetype', 'edge_id', 'guid', 'capacity', 
                          'geometry', 'to_node', 'length'])},
         'power fragility': {
@@ -261,7 +261,9 @@ def create_map_layer(df, name):
         return existing_map_layer
     
     if name == "intensity":
-        locs = np.array([df.geometry.y.to_list(), df.geometry.x.to_list(), df.im.to_list()]).transpose().tolist()
+        # Take the largest 500_000 values to display
+        df_limited = df.sort_values(by='im',ascending=False).head(500_000)
+        locs = np.array([df_limited.geometry.y.to_list(), df_limited.geometry.x.to_list(), df_limited.im.to_list()]).transpose().tolist()
         map_layer = ipyleaflet.Heatmap(locations=locs, radius = 10) 
     elif name == "landuse":
         map_layer = ipyleaflet.GeoJSON(data = json.loads(df.to_json()),
@@ -282,7 +284,7 @@ def create_map_layer(df, name):
             x = node.geometry.x
             y = node.geometry.y
             marker_color = 'blue' if node['is_operational'] else 'red'
-            icon_name = 'fa-industry' if node['pwr_plant'] else 'bolt'
+            icon_name = 'fa-industry' if node['pwr_plant'] == 1 else 'bolt'
             icon_color = 'black'
             marker = Marker(icon=AwesomeIcon(
                         name=icon_name,
@@ -302,6 +304,52 @@ def create_map_layer(df, name):
     layers.value['layers'][name]['force_render'].set(False)
     return map_layer
 
+def fast_transform_xy(T,x,y):
+    TI = rasterio.transform.IDENTITY.translation(0.5, 0.5)
+    TI_mat = np.array([[TI[0],TI[1],TI[2]],[TI[3],TI[4],TI[5]]])
+    T_mat = np.array([[T[0],T[1],T[2]],[T[3],T[4],T[5]]])
+    n = len(x)
+    first_input = np.ones((3,n))
+    first_input[0,:] = x
+    first_input[1,:] = y
+    first_pass = np.dot(TI_mat, first_input)
+    second_inp = np.concatenate([first_pass[[1]],first_pass[[0]],first_input[[2]]])    
+    second_pass = np.dot(T_mat, second_inp)
+    return second_pass[0], second_pass[1]
+
+def read_tiff(file_bytes):
+    byte_io = io.BytesIO(file_bytes)
+    with rasterio.open(byte_io) as src:
+        ims = src.read()
+    
+    current_crs = src.crs
+    target_crs = 'EPSG:4326'
+    transform, width, height = calculate_default_transform(current_crs, target_crs, src.width, src.height, *src.bounds)
+    ims_transformed = np.zeros((height, width))
+
+    print('start reproject ..........')
+    reproject(
+        source=ims[0],
+        destination=ims_transformed,
+        src_transform=src.transform,
+        src_crs=current_crs,
+        dst_transform=transform,
+        dst_crs=target_crs,
+        resampling=Resampling.nearest)
+
+
+    lon_pos, lat_pos = np.meshgrid(range(width),range(height))
+    print('start transform ..........')
+    #lon, lat = rasterio.transform.xy(transform,lat_pos.flatten(),lon_pos.flatten())
+    lon, lat = fast_transform_xy(transform,lat_pos.flatten(),lon_pos.flatten())
+    print('start dataframe ..........')
+    gdf = gpd.GeoDataFrame(ims_transformed.flatten(), 
+            geometry = gpd.points_from_xy(lon, lat, crs="EPSG:4326"))
+    gdf = gdf.rename(columns={0:'im'})
+    # return only the non-zero intensity measures
+    return gdf[gdf['im'] > 0]
+    #return gdf.sort_values(by='im',ascending=False).head(10000)
+
 @solara.component
 def MetricWidget(name, description, value, max_value, render_count):
     value, set_value = solara.use_state_or_update(value)
@@ -342,6 +390,8 @@ def import_data(fileinfo: solara.components.file_drop.FileInfo):
     extension = fileinfo['name'].split('.')[-1]
     if extension == 'xlsx':
         df = pd.read_excel(data)
+    elif extension in ['tiff','tif']:
+        df = read_tiff(data) 
     else:
         json_string = data.decode('utf-8')
         json_data = json.loads(json_string)
@@ -491,7 +541,7 @@ def MapViewer():
     zoom, set_zoom = solara.use_state(default_zoom)
     #center, set_center = solara.use_state(default_center)
 
-    base_map = ipyleaflet.basemaps["Stamen"]["Watercolor"]
+    base_map = ipyleaflet.basemaps["Esri"]["WorldImagery"]
     base_layer = ipyleaflet.TileLayer.element(url=base_map.build_url())
     map_layers = [base_layer]
 

tomorrowcities/pages/utilities.py

@@ -0,0 +1,174 @@
+import solara
+import time
+import random
+import json
+import pandas as pd
+import os
+os.environ['USE_PYGEOS'] = '0'
+import geopandas as gpd
+from typing import Tuple, Optional
+import ipyleaflet
+from ipyleaflet import AwesomeIcon, Marker
+import numpy as np
+import logging, sys
+#logging.basicConfig(stream=sys.stderr, level=logging.INFO)
+
+from ..backend.engine import compute, compute_power_infra, calculate_metrics
+
+app_data = solara.reactive({'df': solara.reactive(None), 
+                            'gdf': solara.reactive(None),
+                            'selected_columns': solara.reactive([]),
+                            'lat': solara.reactive(None),
+                            'lon': solara.reactive(None),
+                            'im': solara.reactive(None)})
+
+def import_data(fileinfo: solara.components.file_drop.FileInfo):
+    data = fileinfo['data']
+    extension = fileinfo['name'].split('.')[-1]
+    if extension == 'xlsx':
+        df = pd.read_excel(data)
+    elif extension == 'csv':
+        df = pd.read_csv(data)
+    else:
+        json_string = data.decode('utf-8')
+        json_data = json.loads(json_string)
+        if "features" in json_data.keys():
+            df = gpd.GeoDataFrame.from_features(json_data['features'])
+        else:
+            df = pd.read_json(json_string)
+
+    df.columns = df.columns.str.lower()
+
+    return df
+
+
+@solara.component
+def FileDropZone():
+    total_progress, set_total_progress = solara.use_state(-1)
+    fileinfo, set_fileinfo = solara.use_state(None)
+    result, set_result = solara.use_state(solara.Result(True))
+
+    def load():
+        print('loading')
+        if fileinfo is not None:
+            print('processing file')
+            df = import_data(fileinfo)
+            print('importing done')
+            if df is not None:
+                app_data.value['df'].set(df)
+            else:
+                return False
+        return True
+        
+    def progress(x):
+        set_total_progress(x)
+
+    def on_file_deneme(f):
+        set_fileinfo(f)
+    
+    result = solara.use_thread(load, dependencies=[fileinfo])
+
+    solara.Markdown("Step1: Drag and drop your Excel file to the below area.")
+    solara.FileDrop(on_total_progress=progress,
+                    on_file=on_file_deneme, 
+                    lazy=False)
+    if total_progress > -1 and total_progress < 100:
+        solara.Text(f"Uploading {total_progress}%")
+        solara.ProgressLinear(value=total_progress)
+    else:
+        if result.state == solara.ResultState.FINISHED:
+            if result.value:
+                solara.Text("Spacer", style={'visibility':'hidden'})
+            else:
+                solara.Text("Unrecognized file")
+            solara.ProgressLinear(value=False)
+        elif result.state == solara.ResultState.INITIAL:
+            solara.Text("Spacer", style={'visibility':'hidden'})
+            solara.ProgressLinear(value=False)
+        elif result.state == solara.ResultState.ERROR:
+            solara.Text(f'{result.error}')
+            solara.ProgressLinear(value=False)
+        else:
+            solara.Text("Reading the contents")
+            solara.ProgressLinear(value=True)
+
+@solara.component
+def DataframeDisplayer():
+    if app_data.value['df'].value is not None:
+        solara.DataFrame(app_data.value['df'].value)
+
+@solara.component
+def FieldSelector():
+    solara.Markdown("Step 2: Select the field names")
+    if app_data.value['df'].value is not None:
+        solara.Text("lat")
+        solara.ToggleButtonsSingle(value=app_data.value['lat'].value, values=list(app_data.value['df'].value.columns), on_value=app_data.value['lat'].set)
+        solara.Text("lon")
+        solara.ToggleButtonsSingle(value=app_data.value['lon'].value, values=list(app_data.value['df'].value.columns),on_value=app_data.value['lon'].set)
+        solara.Text("im")
+        solara.ToggleButtonsSingle(value=app_data.value['im'].value, values=list(app_data.value['df'].value.columns),on_value=app_data.value['im'].set)
+
+@solara.component
+def Downloader():
+    error_msg, set_error_msg = solara.use_state(None)
+    success_msg, set_success_msg = solara.use_state(None)
+    def generate():
+        set_error_msg(None)
+        set_success_msg(None)
+        if app_data.value['df'].value is not None:
+            if app_data.value['lat'].value is not None:
+                if app_data.value['lon'].value is not None:
+                    if app_data.value['im'].value is not None:
+                        try:
+                            lat = app_data.value['lat'].value
+                            lon = app_data.value['lon'].value
+                            im = app_data.value['im'].value
+                            df_conv = app_data.value['df'].value[[lat,lon,im]]
+                            df_conv = df_conv.rename(columns={lat:'lat',lon:'lon',im:'im'})
+                            gdf = gpd.GeoDataFrame(df_conv[['im']], 
+                                geometry = gpd.points_from_xy(df_conv[lon], df_conv[lat], crs="EPSG:4326"))
+                            app_data.value['gdf'].set(gdf)
+                            set_success_msg("Your file is ready")
+                            set_error_msg(None)
+                        except Exception as e:
+                            app_data.value['gdf'].set(None)
+                            set_error_msg(repr(e))
+                            set_success_msg(None)
+
+    solara.Markdown("Step 3: Clict to generate GeoJSON")                    
+    solara.Button("Generate GeoJSON", icon_name="mdi-cloud-download-outline", color="primary", on_click=generate)
+    if error_msg is not None:
+        solara.Error(error_msg)
+    if success_msg is not None:
+        solara.Success(success_msg)
+    if app_data.value['gdf'].value is not None:
+        file_object = app_data.value['gdf'].value .to_json()
+        with solara.FileDownload(file_object, "intensity_blabla.geojson", mime_type="application/geo+json"):
+            solara.Button("Click to Downlaod genereated GeoJSON", icon_name="mdi-cloud-download-outline", color="primary")
+@solara.component
+def Utilities():
+    with solara.Columns([30,80]):
+        FileDropZone()
+        FieldSelector()
+    Downloader()
+    DataframeDisplayer()
+    
+
+@solara.component
+def Page(name: Optional[str] = None, page: int = 0, page_size=100):
+    css = """
+    .v-input {
+        height: 10px;
+    }
+
+    .v-btn-toggle:not(.v-btn-toggle--dense) .v-btn.v-btn.v-size--default {
+        height: 24px;
+        min-height: 0;
+        min-width: 24px;
+    }
+
+    """
+    solara.Style(value=css)
+    solara.Title("TCDSE » Engine")
+
+    Utilities()