Completed landslide + building + metric average

tomorrowcities/backend/engine.py

@@ -290,7 +290,8 @@ def compute_power_infra(buildings, household, nodes,edges,intensity,fragility,ha
 
 def compute(gdf_landuse, gdf_buildings, df_household, df_individual,gdf_intensity, df_hazard, hazard_type, policies=[]):
 
-    np.random.seed(seed=0)
+    if hazard_type != "landslide":
+        np.random.seed(seed=0)
 
     column_names = {'zoneID':'zoneid','bldID':'bldid','nHouse':'nhouse',
                     'specialFac':'specialfac','expStr':'expstr','repValue':'repvalue',
@@ -344,6 +345,15 @@ def compute(gdf_landuse, gdf_buildings, df_household, df_individual,gdf_intensit
 
     gdf_building_intensity['rnd'] = np.random.random((len(gdf_building_intensity),1))
 
+    if hazard_type == "landslide":
+        gdf_building_collapse_prob = gdf_building_intensity.merge(df_hazard, 
+                                        on=['expstr','susceptibility'], how='left')
+        gdf_building_collapse_prob['ds'] = DS_NO
+        collapsed_idx = (gdf_building_collapse_prob['rnd'] < gdf_building_collapse_prob['collapse_probability']) 
+        gdf_building_collapse_prob.loc[collapsed_idx, 'ds'] = DS_COLLAPSED
+        bld_hazard = gdf_building_collapse_prob[['bldid','ds']]
+        return bld_hazard
+
     # TODO: Check if the logic makes sense
     if hazard_type == HAZARD_FLOOD:
         away_from_flood = gdf_building_intensity['distance'] > threshold_flood_distance

tomorrowcities/content/articles/contribution.md

@@ -0,0 +1,58 @@
+---
+author: huseyin.kaya
+title: Contributing
+description: A step-by-step guideline for contributors
+image: https://github.com/TomorrowsCities/tomorrowcities/blob/main/tomorrowcities/content/images/data.png?raw=true
+thumbnail: https://github.com/TomorrowsCities/tomorrowcities/blob/main/tomorrowcities/content/images/data.png?raw=true
+alt: "Data Formats"
+createdAt: 2023-10-10
+duration: 6 min read
+category:
+  - general
+---
+
+[TOC]
+
+## Adding a new layer
+
+On top of the **engine.py**, there is a reactive variable caller **layers**
+to contain all information about the layers. You can add a new layer to that
+variable. Here we provide an example.
+
+```python
+layers = solara.reactive({
+    # ... 
+    'layers' : {
+        'landslide susceptibility': {
+            'render_order': 50,
+            'map_info_tooltip': 'Number of zones in the susceptibility map',
+            'data': solara.reactive(None),
+            'map_layer': solara.reactive(None),
+            'force_render': solara.reactive(False),
+            'visible': solara.reactive(False),
+            'pre_processing': identity_preprocess,
+            'extra_cols': {},
+            'attributes_required': [set(['id','susceptibility','geometry'])],
+            'attributes': [set(['id','susceptibility','geometry'])]},
+    #...
+```
+
+The above step is the minimum requirement of importing data and visualizing 
+on the map. If you need extra stuff for visualization, then edit **create_map_layer** function.
+
+## Adding a new Hazard
+Add your hazard to application state:
+
+```python
+app_state = solara.reactive({
+    #...
+    'hazard_list': ["earthquake","flood","landslide"],
+    #...
+```
+
+Update **is_ready_to_run** to add the required layers for your hazard.
+
+Update **execute_engine** to include the relation with your hazard and infrastructure type.
+For each instrastructure type, you'll see dedicated functions such as **execute_road**,
+or **execute_power**, etc. Inside each one, make sure you load the necessary data
+before calling the backend.

tomorrowcities/content/articles/landslide.md

@@ -0,0 +1,73 @@
+---
+author: huseyin.kaya
+title: Land Slide
+description: Land Slide damage assessment via Monte-Carlo Approach
+image: https://raw.githubusercontent.com/TomorrowsCities/tomorrowcities/main/tomorrowcities/content/images/landslide.jpg?raw=true
+thumbnail: https://raw.githubusercontent.com/TomorrowsCities/tomorrowcities/main/tomorrowcities/content/images/landslide.jpg?raw=true
+alt: "Landslide analysis"
+createdAt: 2023-10-10
+duration: 6 min read
+category:
+  - general
+---
+
+[TOC]
+
+## Data Formats
+ 
+### Susceptibility Map
+Since it contains geo-spatial information, the format of this file should be GeoJSON.
+
+|id|susceptibility|geometry|
+|--|--------------|--------|
+|0 |low           |MULTIPOLYGON (((82.66785 27.80147, 82.66906 27... |
+|1 |medium        |MULTIPOLYGON (((82.60206 27.85078, 82.60202 27... |
+|2 |low           |MULTIPOLYGON (((82.68074 27.79833, 82.68025 27... |
+|3 |high          |MULTIPOLYGON (((82.62131 27.86873, 82.62120 27... |
+
+### Fragility
+Collapse probability depends on three factors:
+
+* **expstr**: taxonomy of the structure
+* **susceptibility**: which susceptibility zone the structure is in
+* **trigger level**: amount of rainfall which is categorized into minor, moderate and severe.
+
+Since it is a tabular data with no geo-spatial context, the data can be provided in 
+Microsoft Excel or JSON format.
+
+|expstr|susceptibility|minor|moderate|severe|
+|------|--------------|-----|--------|------|
+|Adb+HC+10s+Edu| 	low 	|0.01 	|0.02 	|0.03|
+|Adb+HC+1s+Res |	low 	|0.03 	|0.04 	|0.05|
+|Adb+HC+1s+ResCom|	low |0.06 	|0.08 	|0.09| 
+
+### Trigger Level
+Trigger level selection becomes active when the user selects "landuse" hazard type. 
+There are three pre-defined levels: minor, moderate, and severe. The fragility data 
+should contain a column for each category.
+
+## Algorithm
+The algorithm starts with geo-spatial merge of susceptibility map and the building layer. 
+For this purpose, **sjoin_nearest** function of GeoPandas is used. As a result of 
+merging process, the buildings have susceptibility attribute. 
+
+The merged tabular data is left joined with fragility data. In this merge, instead of using all minor, moderate
+and severe columns, only the column matching to the trigger level is used. For instance, if the trigger level
+is moderate, expstr, susceptibility,  and moderate columns are used. As a result of this merge, the buildings
+now have collapse probabilities.
+
+The next step is Monte-Carlo simulation. For each building, a random number between zero and one is picked.
+If it is less than the building's collapse probability, then the damage state of the building is set to DS_COLLAPSED (4)
+otherwise to DS_NO (0).
+
+Then by using household, landuse and individual layers, the seven impact metrics are calculated.
+
+Above process is repeated N times and the impact metrics are averaged. In the metric widgets 
+these averaged impact metrics are displayed. 
+
+In the map, and the layer displayer, however, the damage states of the buildings and other infrastructures are displayed 
+according to the last Monte-Carlo random realization. However, the attributes metric1,...,metric7 denote the average 
+over N random trials. 
+
+
+

tomorrowcities/content/articles/metrics.md

@@ -66,3 +66,7 @@ An individual is assumed to be displaced when any of the following condition hol
 * the individual's workplace, school, or associated hospital is damaged
 * the individual can not reach to workplace, school, or associated hospital via transportation network
 * the individual's workplace, school, or associated hospital has no electricity
+
+## Average Impact Metrics in Monte-Carlo Simulations
+When the hazard scenario is run with different random realizations such as landslide calculations,
+then the impact metrics are calculated by taking average of all simulations. 

tomorrowcities/pages/engine.py

@@ -21,7 +21,7 @@ import logging, sys
 #logging.basicConfig(stream=sys.stderr, level=logging.INFO)
 import pickle
 import datetime
-from .settings import storage
+from .settings import storage, landslide_max_trials
 from ..backend.engine import compute, compute_power_infra, compute_road_infra, calculate_metrics
 from ..backend.utils import building_preprocess, identity_preprocess
 from .utilities import S3FileBrowser, extension_list, extension_list_w_dots
@@ -30,11 +30,36 @@ from .docs import data_import_help
 layers = solara.reactive({
     'infra': solara.reactive(["building"]),
     'hazard': solara.reactive("flood"),
+    'hazard_list': ["earthquake","flood","landslide"],
     'datetime_analysis': datetime.datetime.utcnow(),
     'road_water_height_threshold': solara.reactive(0.3),
+    'landslide_trigger_level': solara.reactive('moderate'),
+    'landslide_trigger_level_list': ['minor','moderate','severe'],
     'dialog_message_to_be_shown': solara.reactive(None),
     'version': '0.2.3',
     'layers' : {
+        'landslide fragility': {
+            'render_order': 0,
+            'map_info_tooltip': 'Number of landslide fragility records',
+            'data': solara.reactive(None),
+            'map_layer': solara.reactive(None),
+            'force_render': solara.reactive(False),
+            'visible': solara.reactive(False),
+            'pre_processing': identity_preprocess,
+            'extra_cols': {},
+            'attributes_required': [set(['expstr','susceptibility','minor','moderate','severe'])],
+            'attributes': [set(['expstr','susceptibility','minor','moderate','severe'])]},
+        'landslide susceptibility': {
+            'render_order': 21,
+            'map_info_tooltip': 'Number of zones in the landslide susceptibility map',
+            'data': solara.reactive(None),
+            'map_layer': solara.reactive(None),
+            'force_render': solara.reactive(False),
+            'visible': solara.reactive(False),
+            'pre_processing': identity_preprocess,
+            'extra_cols': {},
+            'attributes_required': [set(['id','susceptibility','geometry'])],
+            'attributes': [set(['id','susceptibility','geometry'])]},
         'building': {
             'render_order': 50,
             'map_info_tooltip': 'Number of buildings',
@@ -318,6 +343,17 @@ def load_app_state():
         loaded_state = pickle.load(fileObj)
         load_from_state(loaded_state)
 
+def generic_layer_colors(feature):
+    return None
+
+def susceptibility_colors(feature):
+    susceptibility = feature['properties']['susceptibility']
+    s_to_color = {'low': 'green', 'medium':'orange','high': 'red'}
+    return {'fillColor':  s_to_color[susceptibility], 'color':  s_to_color[susceptibility]}
+
+def generic_layer_click_handler(event=None, feature=None, id=None, properties=None):
+    layers.value['map_info_detail'].set(properties)
+    layers.value['map_info_button'].set("detail")  
 
 def building_colors(feature):
     ds_to_color = {0: 'lavender', 1:'violet',2:'fuchsia',3:'indigo',4:'darkslateblue',5:'black'}
@@ -328,7 +364,7 @@ def building_colors(feature):
     occupancy = feature['properties']['occupancy']
     normal_color = 'green' if occupancy == 'Hea' else 'blue'
     #return {'fillColor': 'black', 'color': 'red' if hospital_access == False else normal_color}
-    return {'fillColor': 'black', 'color': 'green' if has_power else 'red'}
+    return {'fillColor': 'black', 'color': 'green' if ds == 0 else 'red'}
 
 def building_click_handler(event=None, feature=None, id=None, properties=None):
     layers.value['map_info_detail'].set(properties)
@@ -502,9 +538,18 @@ def create_map_layer(df, name):
         map_layer= ipyleaflet.MarkerCluster(markers=markers,
                                                    disable_clustering_at_zoom=5)
         
-        
+    elif name == 'landslide susceptibility':
+        map_layer = ipyleaflet.GeoJSON(data = json.loads(df.to_json()),
+            style={'opacity': 1, 'dashArray': '9', 'fillOpacity': 0.5, 'weight': 1},
+            hover_style={'color': 'white', 'dashArray': '0', 'fillOpacity': 0.5},
+            style_callback=susceptibility_colors)
+        map_layer.on_click(generic_layer_click_handler)     
     else:
-        map_layer = ipyleaflet.GeoData(geo_dataframe = df)
+        map_layer = ipyleaflet.GeoJSON(data = json.loads(df.to_json()),
+            style={'opacity': 1, 'dashArray': '9', 'fillOpacity': 0.5, 'weight': 1},
+            hover_style={'color': 'white', 'dashArray': '0', 'fillOpacity': 0.5},
+            style_callback=generic_layer_colors)
+        map_layer.on_click(generic_layer_click_handler)
     layers.value['layers'][name]['map_layer'].set(map_layer)
     layers.value['layers'][name]['force_render'].set(False)
     return map_layer
@@ -950,7 +995,7 @@ def MapViewer():
 @solara.component
 def ExecutePanel(): 
 
-
+    progress_message, set_progress_message = solara.use_state("")
     execute_counter, set_execute_counter = solara.use_state(0)
     execute_btn_disabled, set_execute_btn_disabled = solara.use_state(False)
     execute_error = solara.reactive("")
@@ -977,6 +1022,13 @@ def ExecutePanel():
                 missing += list(set(["road edges","road nodes","intensity"]) - existing_layers)
             if "building" in infra:
                 missing += list(set(["landuse","building","household","individual","intensity","vulnerability"]) - existing_layers)
+        elif hazard == "landslide":
+            if "power" in  infra:
+                missing += list(set(["power edges","power nodes","landslide fragility","landslide susceptibility"]) - existing_layers)
+            if "road" in  infra:
+                missing += list(set(["road edges","road nodes","landslide fragility","landslide susceptibility"]) - existing_layers)
+            if "building" in infra:
+                missing += list(set(["landuse","building","household","individual","landslide fragility","landslide susceptibility"]) - existing_layers)
  
         if infra == []:
             missing += ['You should select at least one of power, road or building']
@@ -1071,14 +1123,28 @@ def ExecutePanel():
             buildings_freqincome = buildings[['bldid']].merge(freqincome,on='bldid',how='left')
             buildings['freqincome'] = buildings_freqincome['freqincome']
             print('policies',policies)
-            df_bld_hazard = compute(
-                landuse,
-                buildings, 
-                household, 
-                individual,
-                intensity,
-                fragility if layers.value['hazard'].value == "earthquake" else vulnerability, 
-                layers.value['hazard'].value,policies=policies)
+            if layers.value['hazard'].value == 'landslide':
+                fragility = layers.value['layers']['landslide fragility']['data'].value
+                intensity = layers.value['layers']['landslide susceptibility']['data'].value
+                trigger_level= layers.value['landslide_trigger_level'].value
+                df_bld_hazard = compute(
+                    landuse,
+                    buildings,
+                    household,
+                    individual,
+                    intensity,
+                    fragility[['expstr','susceptibility',trigger_level]].rename(columns={trigger_level:'collapse_probability'}),
+                    layers.value['hazard'].value,
+                    policies=policies)
+            else:
+                df_bld_hazard = compute(
+                    landuse,
+                    buildings,
+                    household,
+                    individual,
+                    intensity,
+                    fragility if layers.value['hazard'].value == "earthquake" else vulnerability,
+                    layers.value['hazard'].value, policies=policies)
             buildings['ds'] = list(df_bld_hazard['ds'])
 
             return buildings
@@ -1111,25 +1177,40 @@ def ExecutePanel():
             is_ready, missing = is_ready_to_run(layers.value['infra'].value, layers.value['hazard'].value)
             if not is_ready:
                 raise Exception(f'Missing {missing}')
-            
-            if 'power' in layers.value['infra'].value:
-                nodes, buildings, household = execute_power()
-                layers.value['layers']['power nodes']['data'].set(nodes)
+            max_trials = landslide_max_trials.value  if layers.value['hazard'].value == "landslide" else 1
+            for trial in range(1,max_trials+1):
+                set_progress_message(f'Monte-Carlo trial {trial}/{max_trials}...')
+                if 'power' in layers.value['infra'].value:
+                    nodes, buildings, household = execute_power()
+                    layers.value['layers']['power nodes']['data'].set(nodes)
+                    layers.value['layers']['building']['data'].set(buildings)
+                    layers.value['layers']['household']['data'].set(household)
+                if 'road' in layers.value['infra'].value:
+                    edges, buildings, household, individual = execute_road()
+                    layers.value['layers']['road edges']['data'].set(edges)
+                    layers.value['layers']['building']['data'].set(buildings)
+                    layers.value['layers']['household']['data'].set(household)
+                    layers.value['layers']['individual']['data'].set(individual)
+                if 'building' in layers.value['infra'].value:
+                    buildings = execute_building()
+                    layers.value['layers']['building']['data'].set(buildings)
+
+                buildings = execute_metric()
                 layers.value['layers']['building']['data'].set(buildings)
-                layers.value['layers']['household']['data'].set(household)
-            if 'road' in layers.value['infra'].value:
-                edges, buildings, household, individual = execute_road()
-                layers.value['layers']['road edges']['data'].set(edges)
-                layers.value['layers']['building']['data'].set(buildings)
-                layers.value['layers']['household']['data'].set(household)
-                layers.value['layers']['individual']['data'].set(individual)
-            if 'building' in layers.value['infra'].value:
-                buildings = execute_building()
-                layers.value['layers']['building']['data'].set(buildings)
-
-            buildings = execute_metric()
-            layers.value['layers']['building']['data'].set(buildings)
 
+                # Taking average without storing intermediate values
+                # averaga_n = [(n-1)*average_(n-1) + value_n]/2
+                if trial == 1:
+                    avg_metrics = buildings[layers.value['metrics'].keys()]
+                else:
+                    avg_metrics = ((trial-1) * avg_metrics + buildings[layers.value['metrics'].keys()])/trial
+                for metric in layers.value['metrics'].keys():
+                    print('average', metric, avg_metrics[metric].sum())
+
+            for metric in layers.value['metrics'].keys():
+                buildings[metric] = list(avg_metrics[metric])
+                layers.value['metrics'][metric]['value'] = avg_metrics[metric].sum()
+            set_progress_message('')
             # trigger render event
             layers.value['render_count'].set(layers.value['render_count'].value + 1)
             if 'power' in layers.value['infra'].value:
@@ -1154,7 +1235,13 @@ def ExecutePanel():
             solara.ToggleButtonsMultiple(value=layers.value['infra'].value, on_value=layers.value['infra'].set, values=["building","power","road"])
         solara.Markdown("#### Hazard")
         with solara.Row(justify="left"):
-            solara.ToggleButtonsSingle(value=layers.value['hazard'].value, on_value=layers.value['hazard'].set, values=["earthquake","flood"])
+            solara.ToggleButtonsSingle(value=layers.value['hazard'].value, on_value=layers.value['hazard'].set, values=layers.value['hazard_list'])
+        if layers.value['hazard'].value == 'landslide':
+            solara.Markdown("#### Landslide trigger level")
+            with solara.Row(justify="left"):
+                solara.ToggleButtonsSingle(value=layers.value['landslide_trigger_level'].value,
+                    on_value=layers.value['landslide_trigger_level'].set,
+                    values=layers.value['landslide_trigger_level_list'])
         with solara.Tooltip("Building-level metrics will be increased by 25% and 50% for medium and low"):
             solara.Markdown("#### Implementation Capacity Score")
         with solara.Row(justify="left"):
@@ -1180,8 +1267,10 @@ def ExecutePanel():
 
     if result.state in [solara.ResultState.RUNNING, solara.ResultState.WAITING]:
         set_execute_btn_disabled(True)
+        solara.Text(progress_message)
         solara.ProgressLinear(value=True)
     else:
+        solara.Text("Spacer", style={"visibility": "hidden"})
         set_execute_btn_disabled(False)
         solara.ProgressLinear(value=False)
 

tomorrowcities/pages/settings.py

@@ -3,7 +3,6 @@ from typing import Optional, cast
 import pickle
 import boto3
 import os
-
 from . import user
 
 class S3Storage:
@@ -86,6 +85,7 @@ def revive_storage():
                     os.environ['bucket_name'])
     
 storage = solara.reactive(revive_storage())  
+landslide_max_trials = solara.reactive(5)
 
 def storage_control(aws_access_key_id: str, aws_secret_access_key: str, region_name: str, bucket_name: str):
     storage.value = S3Storage(aws_access_key_id, aws_secret_access_key, region_name, bucket_name)
@@ -164,4 +164,7 @@ def Page(name: Optional[str] = None, page: int = 0, page_size=100):
     #    StorageViewer()
 
     if err_message != '':
-        solara.Error(err_message)
+        solara.Error(err_message)
+
+    with solara.Card(title='Landslide Parameters',subtitle='Choose the parameters for the landslide simulation'):
+        solara.SliderInt(label='Number of Monte-Carlo Trials', value=landslide_max_trials, min=1,max=100)