Completed (eq,fl,ls) x (bld,pwr,road) combinations

tomorrowcities/backend/engine.py

@@ -45,17 +45,40 @@ def compute_road_infra(buildings, household, individual,
     gdf_buildings = gpd.sjoin_nearest(gdf_buildings,gdf_nodes, 
                 how='left', rsuffix='road_node',distance_col='road_node_distance')
 
-    if hazard in ['flood', 'debris']:
-        roads_with_width = gdf_edges['geometry'].buffer(threshold_flood_distance)
-        for i, road in enumerate(roads_with_width):
-            within_ims = gdf_intensity.clip(road)
-            if len(within_ims) > 0:
-                max_im = within_ims['im'].max()
-            else:
-                max_im = 0
-            gdf_edges.loc[i,'im'] = max_im
-        gdf_edges.loc[gdf_edges['im'] > road_water_height_threshold, 'ds'] = 1
-        gdf_edges.loc[gdf_edges['im'] > road_water_height_threshold, 'is_damaged'] = True
+    if hazard in ['flood', 'debris','landslide']:
+        gdf_edges = gpd.sjoin_nearest(gdf_edges, gdf_intensity, how='left',
+                                          rsuffix='intensity',distance_col='distance')
+        # TODO: sjoin_nearest or the approach below: compare
+        #roads_with_width = gdf_edges['geometry'].buffer(threshold_flood_distance)
+        #for i, road in enumerate(roads_with_width):
+        #    print(i)
+        #    within_ims = gdf_intensity.clip(road)
+        #    if len(within_ims) > 0:
+        #        max_im = within_ims['im'].max()
+        #    else:
+        #        max_im = 0
+        #    gdf_edges.loc[i,'im'] = max_im
+        if hazard == 'landslide':
+            print('before')
+            print(gdf_edges.loc[0])
+            gdf_edges['susceptibility'] = 'low'
+            gdf_edges.loc[gdf_edges['im'] == 2.0, 'susceptibility'] = 'medium'
+            gdf_edges.loc[gdf_edges['im'] == 3.0, 'susceptibility'] = 'high'
+
+            fragility['landslide_expstr'] = fragility['expstr'].astype(str) + "+"+ fragility["susceptibility"].astype(str) 
+            gdf_edges['landslide_expstr'] = 'roads' 
+            gdf_edges['landslide_expstr'] = gdf_edges['landslide_expstr'] + "+" + gdf_edges['susceptibility'].astype(str)
+            gdf_edges = gdf_edges.merge(fragility, on='landslide_expstr', how='left')
+            gdf_edges['ds'] = DS_NO
+            gdf_edges['rnd'] = np.random.random((len(gdf_edges),1))
+            collapsed_idx = (gdf_edges['rnd'] < gdf_edges['collapse_probability']) 
+            gdf_edges.loc[collapsed_idx, 'ds'] = DS_COMPLETE
+            gdf_edges.loc[collapsed_idx, 'is_damaged'] = True
+            print('after')
+            print(gdf_edges.loc[0])
+        else:
+            gdf_edges.loc[gdf_edges['im'] > road_water_height_threshold, 'ds'] = 1
+            gdf_edges.loc[gdf_edges['im'] > road_water_height_threshold, 'is_damaged'] = True
     elif hazard == 'earthquake':
         fragility = fragility.rename(columns={"med_slight": "med_ds1", 
                         "med_moderate": "med_ds2",
@@ -124,11 +147,18 @@ def compute_road_infra(buildings, household, individual,
     # For each individual, find the closest road node (closest) of the
     # household and the facility
 
+    print('individua')
+    print(individual)
+    print('household_w_node_id')
+    print(household_w_node_id)
+    print('gdf_buildings')
+    print(gdf_buildings)
     individual_w_nodes = individual.merge(household_w_node_id[['hhid','node_id']],on='hhid',how='left')\
                         .rename(columns={'node_id':'household_node_id'})\
                         .merge(gdf_buildings[['bldid','node_id']],how='left',left_on='indivfacid',right_on='bldid')\
                         .rename(columns={'node_id':'facility_node_id'})
-
+    print('individual_w_nodes')
+    print(individual_w_nodes)
     # Calculate distances between all nodes in the damaged network
     shortest_distance = nx.shortest_path_length(G_dmg)
 
@@ -140,11 +170,13 @@ def compute_road_infra(buildings, household, individual,
             connection_dict[source][target] = True
 
     # Based on connectivity, fill-in facillity_access attribute in the individual layer
+    print(individual_w_nodes)
     individual_w_nodes['facility_access'] = individual_w_nodes.apply(lambda x: x['facility_node_id'] in connection_dict[x['household_node_id']].keys(),axis=1)
 
     return gdf_edges['ds'], gdf_edges['is_damaged'], gdf_buildings['node_id'], gdf_buildings['hospital_access'], household_w_node_id['node_id'], household_w_node_id['hospital_access'], individual_w_nodes['facility_access']
 
-def compute_power_infra(buildings, household, nodes,edges,intensity,fragility,hazard):
+def compute_power_infra(buildings, household, nodes,edges,intensity,fragility,hazard,
+                        threshold_flood, threshold_flood_distance):
     print('Computing power infrastructure')
     print(nodes.head())
     print(edges.head())
@@ -192,7 +224,8 @@ def compute_power_infra(buildings, household, nodes,edges,intensity,fragility,ha
                     "beta_moderate": "beta_ds2",
                     "beta_extensive": "beta_ds3",
                     "beta_complete": "beta_ds4"})
-        gdf_nodes = gdf_nodes.merge(fragility, how='left',left_on='eq_vuln',right_on='vuln_string')
+        #gdf_nodes = gdf_nodes.merge(fragility, how='left',left_on='eq_vuln',right_on='vuln_string')
+        gdf_nodes = gdf_nodes.merge(fragility, how='left',left_on='eq_frgl',right_on='vuln_string')
         nulls = gdf_nodes['med_ds1'].isna()
         gdf_nodes.loc[nulls, ['med_ds1','med_ds2','med_ds3','med_ds4']] = [99999,99999,99999,99999]
         gdf_nodes.loc[nulls, ['beta_ds1','beta_ds2','beta_ds3','beta_ds4']] = [1,1,1,1]
@@ -209,6 +242,39 @@ def compute_power_infra(buildings, household, nodes,edges,intensity,fragility,ha
             gdf_nodes[f'ds_{i}'] = np.abs(gdf_nodes[f'prob_ds{i-1}'] - gdf_nodes[f'prob_ds{i}'])
         df_ds = gdf_nodes[['ds_1','ds_2','ds_3','ds_4','ds_5']]
         gdf_nodes['ds'] = df_ds.idxmax(axis='columns').str.extract(r'ds_([0-9]+)').astype('int') - 1
+    elif hazard == 'landslide':
+        print(gdf_nodes.loc[0])
+        gdf_nodes['rnd'] = np.random.random((len(gdf_nodes),1))
+        if 'ls_susceptibility' in gdf_nodes.columns:
+            gdf_nodes['susceptibility'] = gdf_nodes['ls_susceptibility']
+        else:
+            gdf_nodes['susceptibility'] = 'low'
+            gdf_nodes.loc[gdf_nodes['im'] == 2.0, 'susceptibility'] = 'medium'
+            gdf_nodes.loc[gdf_nodes['im'] == 3.0, 'susceptibility'] = 'high'
+        fragility['landslide_expstr'] = fragility['expstr'].astype(str) + "+"+ fragility["susceptibility"].astype(str) 
+        gdf_nodes['landslide_expstr'] = gdf_nodes['ls_frgl'].astype(str) + "+" \
+                                                   + gdf_nodes['susceptibility'].astype(str)
+        gdf_nodes = gdf_nodes.merge(fragility, on='landslide_expstr', how='left')
+        gdf_nodes['ds'] = DS_NO
+        collapsed_idx = (gdf_nodes['rnd'] < gdf_nodes['collapse_probability']) 
+        gdf_nodes.loc[collapsed_idx, 'ds'] = DS_COMPLETE
+        print(gdf_nodes.loc[0])
+    elif hazard == 'flood':
+        away_from_flood = gdf_nodes['distance'] > threshold_flood_distance
+        print('threshold_flood_distance',threshold_flood_distance)
+        print('number of distant buildings', len(gdf_nodes.loc[away_from_flood, 'im']))
+        gdf_nodes.loc[away_from_flood, 'im'] = 0
+
+
+        gdf_nodes = gdf_nodes.merge(fragility, left_on='fl_vuln', right_on='expstr', how='left')
+        x = np.array([0,0.5,1,1.5,2,3,4,5,6])
+        y = gdf_nodes[['hw0','hw0_5','hw1','hw1_5','hw2','hw3','hw4','hw5','hw6']].to_numpy()
+        xnew = gdf_nodes['im'].to_numpy()
+        flood_mapping = interp1d(x,y,axis=1,kind='linear',bounds_error=False, fill_value=(0,1))
+        # TODO: find another way for vectorized interpolate
+        gdf_nodes['fl_prob'] = np.diag(flood_mapping(xnew))
+        gdf_nodes['ds'] = 0
+        gdf_nodes.loc[gdf_nodes['fl_prob'] > threshold_flood,'ds'] = 1
         
     # All Nodes
     all_nodes = set(gdf_nodes['node_id'])
@@ -346,8 +412,28 @@ def compute(gdf_landuse, gdf_buildings, df_household, df_individual,gdf_intensit
     gdf_building_intensity['rnd'] = np.random.random((len(gdf_building_intensity),1))
 
     if hazard_type == "landslide":
+        print('----------up side down prev', gdf_building_intensity.shape)
+        print(pd.unique(gdf_building_intensity['im']))
+        gdf_building_intensity['susceptibility'] = 'low'
+        gdf_building_intensity.loc[gdf_building_intensity['im'] == 2.0, 'susceptibility'] = 'medium'
+        gdf_building_intensity.loc[gdf_building_intensity['im'] == 3.0, 'susceptibility'] = 'high'
+        print(gdf_building_intensity.loc[0])
+        print(gdf_building_intensity.columns)
+        print(df_hazard.loc[0])
+        df_hazard['landslide_expstr'] = df_hazard['expstr'].astype(str) +"+"+ df_hazard["susceptibility"].astype(str) 
+
+        gdf_building_intensity['landslide_expstr'] = gdf_building_intensity['material'].astype(str)  + "+" \
+                                                   + gdf_building_intensity['code_level'].astype(str)  + "+" \
+                                                   + gdf_building_intensity["susceptibility"].astype(str) 
+        print('x1', gdf_building_intensity.columns)
+        print('x2', df_hazard.columns)
+
+        print(len(gdf_building_intensity))
         gdf_building_collapse_prob = gdf_building_intensity.merge(df_hazard, 
-                                        on=['expstr','susceptibility'], how='left')
+                                        on='landslide_expstr', how='left')
+        print('----------up side down ', gdf_building_collapse_prob.shape)
+        print(gdf_building_collapse_prob.loc[0])
+        print(len(gdf_building_collapse_prob))
         gdf_building_collapse_prob['ds'] = DS_NO
         collapsed_idx = (gdf_building_collapse_prob['rnd'] < gdf_building_collapse_prob['collapse_probability']) 
         gdf_building_collapse_prob.loc[collapsed_idx, 'ds'] = DS_COLLAPSED

tomorrowcities/components/article.py

@@ -6,9 +6,9 @@ from ..data import articles
 @solara.component
 def ArticleCard(name):
     article = articles[name]
-    with rv.Card(max_width="400px") as main:
+    with rv.Card(max_width="300px") as main:
         with solara.Link(f"/docs/{name}"):
-            rv.Img(height="250", src=article.image_url)
+            rv.Img(height="150", src=article.image_url)
         rv.CardTitle(children=[article.title])
         with rv.CardText():
             solara.Markdown(article.description)
@@ -21,7 +21,7 @@ def ArticleCard(name):
 def Overview():
     with solara.ColumnsResponsive(12) as main:
         with solara.Card():
-            with solara.ColumnsResponsive(12, small=6, large=4):
+            with solara.ColumnsResponsive(12, small=6, large=2):
                 for name in articles:
                     ArticleCard(name)
     return main

tomorrowcities/content/articles/data.md

@@ -43,7 +43,8 @@ in the GeoTIFF are labelled such as "PGA", "SA 0.3", "SA 1", etc.
 
 
 ## Layers
-The layers supported by tomorrowcities are listed below.  In this section, we will cover all of them and provide information so that the users are able to generate data compatible to web application.
+The layers supported by tomorrowcities are listed below.  In this section, we will cover all of them and provide information so that the users are able to generate data compatible to web application. Unless otherwise
+mentioned, all geo-spatial layers have WGS 84 coordinate reference system with EPSG:4326.
 
 * land use 
 * building
@@ -55,6 +56,17 @@ The layers supported by tomorrowcities are listed below.  In this section, we wi
 * power nodes
 * power edges
 
+### Landuse
+Landuse plan is presented as a GeoJSON format whose attributes are given below:
+
+
+|population|status|zone|densitycap|luf|landuse_lu|area|zoneid|floorarear|setback|avgincome|
+|----------|------|----|----------|---|----------|----|------|----------|-------|---------|
+0|None 	|Agricultural Zone 	|0.0 	|Agricultural Zone 	|Agricultural Zone 	|4099.24144134 	|4 	|0.0 	|0.0 	|None|
+0| None |Forest Zone 	|0.0 	|Forest Zone 	|Forest Zone 	|30843.9849839 	|5 	|0.0 	|0.0 	|None|
+7081 	|None 	|Residential Very Low Density 	|5.0 	|Residential-Very Low Density |	Agriculture Cum Resedential	|1416.16357154 	|226 	|0.0 	|0.0 	|lowIncome|
+
+
 ### Buildings
 Buildings are the core component of visioning scenarios. The features of the building with some example data are shown below:
 
@@ -86,6 +98,26 @@ where
 * **nind (integer)** is the number of individuals living in the household
 * **commfacid (integer)** is the building identifier of the community facility. In Tomorrow's Cities, it is used to define the hospital associated with the household. 
 
+### Individuals
+Individual layer is a tabular data which can be stored in Excel or JSON format.
+The attributes are:
+
+|hhid|individ|gender|age|head|eduattstat|indivfacid|
+|----|-------|------|---|----|----------|----------|
+|24448| 	1| 	2| 	8| 	1| 	3| 	-1|
+|1 	|5552 	|2 	|2 	|8 	|1 	|4 	|-1 	|
+|2 	|31586 	|3 	|2 	|9 	|1 	|5 	|-1 	|
+
+where 
+
+* **hhid** refers to household identifier where the individual lives in.
+* **individ** is the unique individual identifier
+* **gender** is a categorical variable for gender: 1: male, 2:female
+* **age** is the age category
+* **head** is a binary feature indicating that the individual is the head of the household
+* **eduattstat** is the education level
+* **indivfacid** is the facility (school or workspace) that the individual is associated with
+
 ### Intensity Measures
 Whether it is flood, debris or earthquake, every hazard map should contain at least two properties: a point geometry and intensity measure denotes by 'im'. The data can be provided via GeoTIFF or GeoJSON format. TIFF files should contain CRS 
 information so that the engine could map the coordinates to a common CRS to conduct calculations.

tomorrowcities/pages/engine.py

@@ -48,18 +48,7 @@ layers = solara.reactive({
             'pre_processing': identity_preprocess,
             'extra_cols': {},
             'attributes_required': [set(['expstr','susceptibility','minor','moderate','severe'])],
-            'attributes': [set(['expstr','susceptibility','minor','moderate','severe'])]},
-        'landslide susceptibility': {
-            'render_order': 21,
-            'map_info_tooltip': 'Number of zones in the landslide susceptibility map',
-            'data': solara.reactive(None),
-            'map_layer': solara.reactive(None),
-            'force_render': solara.reactive(False),
-            'visible': solara.reactive(False),
-            'pre_processing': identity_preprocess,
-            'extra_cols': {},
-            'attributes_required': [set(['id','susceptibility','geometry'])],
-            'attributes': [set(['id','susceptibility','geometry'])]},
+            'attributes': [set(['expstr','susceptibility','minor','moderate','severe','description'])]},
         'building': {
             'render_order': 50,
             'map_info_tooltip': 'Number of buildings',
@@ -159,7 +148,7 @@ layers = solara.reactive({
             'visible': solara.reactive(False),
             'pre_processing': identity_preprocess,
             'extra_cols': {'ds': 0, 'is_damaged': False, 'is_operational': True},
-            'attributes_required': [set(['geometry', 'node_id', 'pwr_plant', 'n_bldgs', 'eq_vuln'])],
+            'attributes_required': [set(['geometry', 'node_id', 'pwr_plant', 'n_bldgs'])],
             'attributes': [set(['geometry', 'fltytype', 'strctype', 'utilfcltyc', 'indpnode', 'guid', 
                          'node_id', 'x_coord', 'y_coord', 'pwr_plant', 'serv_area', 'n_bldgs', 
                          'income', 'eq_vuln'])]},
@@ -346,11 +335,6 @@ def load_app_state():
 def generic_layer_colors(feature):
     return None
 
-def susceptibility_colors(feature):
-    susceptibility = feature['properties']['susceptibility']
-    s_to_color = {'low': 'green', 'medium':'orange','high': 'red'}
-    return {'fillColor':  s_to_color[susceptibility], 'color':  s_to_color[susceptibility]}
-
 def generic_layer_click_handler(event=None, feature=None, id=None, properties=None):
     layers.value['map_info_detail'].set(properties)
     layers.value['map_info_button'].set("detail")  
@@ -486,9 +470,12 @@ def create_map_layer(df, name):
     if name == "intensity":
         # Take the largest 500_000 values to display
         im_col = 'pga' if 'pga' in df.columns else 'im'
-        df_limited = df.sort_values(by=im_col,ascending=False).head(500_000)
+        df_non_zero = df[df[im_col] > 0]
+        df_limited = df_non_zero.sample(min(len(df_non_zero),500_000))
+        df_limited[im_col] = df_limited[im_col] / df_limited[im_col].max()
+        #df_limited = df.sort_values(by=im_col,ascending=False).head(500_000)
         locs = np.array([df_limited.geometry.y.to_list(), df_limited.geometry.x.to_list(), df_limited[im_col].to_list()]).transpose().tolist()
-        map_layer = ipyleaflet.Heatmap(locations=locs, radius = 5) 
+        map_layer = ipyleaflet.Heatmap(locations=locs, radius = 5, blur = 1) 
     elif name == "landuse":
         map_layer = ipyleaflet.GeoJSON(data = json.loads(df.to_json()),
             style={'opacity': 1, 'dashArray': '9', 'fillOpacity': 0.5, 'weight': 1},
@@ -537,13 +524,7 @@ def create_map_layer(df, name):
             markers.append(marker)
         map_layer= ipyleaflet.MarkerCluster(markers=markers,
                                                    disable_clustering_at_zoom=5)
-        
-    elif name == 'landslide susceptibility':
-        map_layer = ipyleaflet.GeoJSON(data = json.loads(df.to_json()),
-            style={'opacity': 1, 'dashArray': '9', 'fillOpacity': 0.5, 'weight': 1},
-            hover_style={'color': 'white', 'dashArray': '0', 'fillOpacity': 0.5},
-            style_callback=susceptibility_colors)
-        map_layer.on_click(generic_layer_click_handler)     
+
     else:
         map_layer = ipyleaflet.GeoJSON(data = json.loads(df.to_json()),
             style={'opacity': 1, 'dashArray': '9', 'fillOpacity': 0.5, 'weight': 1},
@@ -1017,23 +998,35 @@ def ExecutePanel():
                 missing += list(set(["landuse","building","household","individual","intensity","fragility"]) - existing_layers)
         elif hazard == "flood":
             if "power" in  infra:
-                missing += list(set(["power edges","power nodes","intensity","power vulnerability"]) - existing_layers)
+                missing += list(set(["landuse","building","household","individual","power edges","power nodes","intensity","vulnerability"]) - existing_layers)
             if "road" in  infra:
-                missing += list(set(["road edges","road nodes","intensity"]) - existing_layers)
+                missing += list(set(["landuse","building","household","individual","road edges","road nodes","intensity"]) - existing_layers)
             if "building" in infra:
                 missing += list(set(["landuse","building","household","individual","intensity","vulnerability"]) - existing_layers)
         elif hazard == "landslide":
             if "power" in  infra:
-                missing += list(set(["power edges","power nodes","landslide fragility","landslide susceptibility"]) - existing_layers)
+                missing += list(set(["landuse","building","household","individual","power edges","power nodes","landslide fragility"]) - existing_layers)
             if "road" in  infra:
-                missing += list(set(["road edges","road nodes","landslide fragility","landslide susceptibility"]) - existing_layers)
+                missing += list(set(["landuse","building","household","individual","road edges","road nodes","landslide fragility"]) - existing_layers)
             if "building" in infra:
-                missing += list(set(["landuse","building","household","individual","landslide fragility","landslide susceptibility"]) - existing_layers)
+                missing += list(set(["landuse","building","household","individual","landslide fragility"]) - existing_layers)
  
         if infra == []:
             missing += ['You should select at least one of power, road or building']
         return missing == [], missing
     
+    def pre_compute_checks():
+        hazard = layers.value['hazard'].value
+        infra = layers.value['infra'].value
+        building = layers.value['layers']['building']['data'].value
+        household = layers.value['layers']['household']['data'].value
+
+        missing_buildings = set(household['bldid']) - set(building['bldid'])
+        print('missing buildings', missing_buildings)
+        if len(missing_buildings) > 0:
+            return False, f"There are {len(missing_buildings)} household without buildings. Please add buildings for these households."
+        
+        return True, ''
 
 
     def execute_engine():
@@ -1047,6 +1040,10 @@ def ExecutePanel():
             edges = layers.value['layers']['road edges']['data'].value
             intensity = layers.value['layers']['intensity']['data'].value
             fragility = layers.value['layers']['road fragility']['data'].value
+            if layers.value['hazard'].value == 'landslide':
+                fragility = layers.value['layers']['landslide fragility']['data'].value
+                trigger_level= layers.value['landslide_trigger_level'].value
+                fragility = fragility[['expstr','susceptibility',trigger_level]].rename(columns={trigger_level:'collapse_probability'})
             hazard = layers.value['hazard'].value
 
             edges['ds'] = 0
@@ -1080,9 +1077,16 @@ def ExecutePanel():
             nodes = layers.value['layers']['power nodes']['data'].value
             edges = layers.value['layers']['power edges']['data'].value
             intensity = layers.value['layers']['intensity']['data'].value
-            power_fragility = layers.value['layers']['power fragility']['data'].value
+            fragility = layers.value['layers']['power fragility']['data'].value
             hazard = layers.value['hazard'].value
 
+            if layers.value['hazard'].value == 'landslide':
+                fragility = layers.value['layers']['landslide fragility']['data'].value
+                trigger_level= layers.value['landslide_trigger_level'].value
+                fragility = fragility[['expstr','susceptibility',trigger_level]].rename(columns={trigger_level:'collapse_probability'})
+
+            if layers.value['hazard'].value == 'flood':
+                fragility = layers.value['layers']['vulnerability']['data'].value
 
             ds, is_damaged, is_operational, has_power, household_has_power, hospital_has_power = \
                 compute_power_infra(buildings,
@@ -1090,8 +1094,8 @@ def ExecutePanel():
                                     nodes, 
                                     edges,
                                     intensity,
-                                    power_fragility,
-                                    hazard)
+                                    fragility,
+                                    hazard, threshold_flood.value, threshold_flood_distance.value)
             
             #power_node_df =  dfs['Power Nodes'].copy()                         
             nodes['ds'] = list(ds)
@@ -1124,7 +1128,6 @@ def ExecutePanel():
             #print('policies',policies)
             if layers.value['hazard'].value == 'landslide':
                 fragility = layers.value['layers']['landslide fragility']['data'].value
-                intensity = layers.value['layers']['landslide susceptibility']['data'].value
                 trigger_level= layers.value['landslide_trigger_level'].value
                 df_bld_hazard = compute(
                     landuse,
@@ -1180,9 +1183,15 @@ def ExecutePanel():
             is_ready, missing = is_ready_to_run(layers.value['infra'].value, layers.value['hazard'].value)
             if not is_ready:
                 raise Exception(f'Missing {missing}')
+            #is_ready, message = pre_compute_checks()
+            #if not is_ready:
+            #    raise Exception(message)
             max_trials = landslide_max_trials.value  if layers.value['hazard'].value == "landslide" else 1
             for trial in range(1,max_trials+1):
-                set_progress_message('Running...')
+                if trial == 1:
+                    set_progress_message('Running...')
+                else:
+                    set_progress_message(f'Monte-Carlo trial {trial}/{max_trials}...')
                 if 'power' in layers.value['infra'].value:
                     nodes, buildings, household = execute_power()
                     layers.value['layers']['power nodes']['data'].set(nodes)