Added two policies

README.md

@@ -1,4 +1,16 @@
-# Release Notes
+---
+title: App Engine
+emoji: 🏆
+colorFrom: purple
+colorTo: blue
+sdk: docker
+pinned: false
+---
 
-## v0.0.1
-* 
+## What is New?
+
+### Excel support:
+You can  upload Excel files containing your tabular data such as individual, household, fragility or vulnerability data. However, processing excel files is very slower than processing JSON files so I definitely suggest working with JSON files. You can convert your Excel files via panda framework. The platform also does not try to convert the coordinates even if there is any in the Excel file because there is no way to know which columns represent the coordinates or coordinate reference systems without metadata. So Excel spreadsheets should be used to provide non-geo related tabular data.  So please use them only for data not containing any geo-specific information. 
+
+### Dar Es Salaam Study
+[Sample Dataset](https://drive.google.com/file/d/1BGPZQ2IKJHY9ExOCCHcNNrCTioYZ8D1y/view?usp=sharing) now contains some visioning scenario for Dar Es Salaam case.

pyproject.toml

@@ -17,7 +17,8 @@ dependencies = [
     "psycopg2-binary",
     "scipy", 
     "pandas",
-    "networkx"
+    "networkx",
+    "openpyxl"
 ]
 
 [tool.hatch.version]

tomorrowcities/backend/engine.py

@@ -1,13 +1,7 @@
-#%%
-import warnings
-import json
-import sys
-import argparse
-import io
 import os
+os.environ['USE_PYGEOS'] = '0'
 import pandas as pd
-import psycopg2
-import geopandas
+import geopandas as gpd
 import numpy as np
 from scipy.stats import norm
 from scipy.interpolate import interp1d
@@ -36,7 +30,7 @@ def compute_power_infra(nodes,edges,intensity,fragility):
     for _, edge in edges.iterrows():
         G_power.add_edge(*(edge.from_node, edge.to_node))  
 
-    nodes = geopandas.sjoin_nearest(nodes,intensity, 
+    nodes = gpd.sjoin_nearest(nodes,intensity, 
                 how='left', rsuffix='intensity',distance_col='distance')
 
     nodes = nodes.merge(eq_vuln, how='left',left_on='eq_vuln',right_on='vuln_string')
@@ -55,14 +49,14 @@ def compute_power_infra(nodes,edges,intensity,fragility):
     for i in [1,2,3,4,5]:
         nodes[f'ds_{i}'] = np.abs(nodes[f'prob_ds{i-1}'] - nodes[f'prob_ds{i}'])
     df_ds = nodes[['ds_1','ds_2','ds_3','ds_4','ds_5']]
-    nodes['eq_ds'] = df_ds.idxmax(axis='columns').str.extract(r'ds_([0-9]+)').astype('int')
+    nodes['eq_ds'] = df_ds.idxmax(axis='columns').str.extract(r'ds_([0-9]+)').astype('int') - 1
     
     # Damage State Codes
-    DS_NO = 1
-    DS_SLIGHT = 2
-    DS_MODERATE = 3
-    DS_EXTENSIVE = 4
-    DS_COMPLETE = 5
+    DS_NO = 0
+    DS_SLIGHT = 1
+    DS_MODERATE = 2
+    DS_EXTENSIVE = 3
+    DS_COMPLETE = 4
 
     # If a damage state is above this threshold (excluding), 
     # we consider the associated node as dead.
@@ -131,8 +125,7 @@ def compute_power_infra(nodes,edges,intensity,fragility):
 
     return nodes['eq_ds'], nodes['is_damaged'], nodes['is_operational']
 
-
-def compute(gdf_buildings, df_household, df_individual,gdf_intensity, df_hazard, hazard_type):
+def compute(gdf_landuse, gdf_buildings, df_household, df_individual,gdf_intensity, df_hazard, hazard_type, policies=[]):
 
     column_names = {'zoneID':'zoneid','bldID':'bldid','nHouse':'nhouse',
                     'specialFac':'specialfac','expStr':'expstr','repValue':'repvalue',
@@ -140,30 +133,28 @@ def compute(gdf_buildings, df_household, df_individual,gdf_intensity, df_hazard,
                     'CommFacID':'commfacid','indivId':'individ','eduAttStat':'eduattstat',
                     'indivFacID':'indivfacid','VALUE':'im'}
 
-    print(hazard_type, df_hazard.head())
+
+    gdf_landuse = gdf_landuse.rename(columns=column_names)
     gdf_buildings = gdf_buildings.rename(columns=column_names)
     df_household = df_household.rename(columns=column_names)
     df_individual = df_individual.rename(columns=column_names)
     gdf_intensity = gdf_intensity.rename(columns=column_names)
 
     # Damage States
-    DS_NO = 1
-    DS_SLIGHT = 2
-    DS_MODERATE = 3
-    DS_EXTENSIZE = 4
-    DS_COLLAPSED = 5
+    DS_NO = 0
+    DS_SLIGHT = 1
+    DS_MODERATE = 2
+    DS_EXTENSIZE = 3
+    DS_COLLAPSED = 4
 
     # Hazard Types 
     HAZARD_EARTHQUAKE = "earthquake"
     HAZARD_FLOOD = "flood"
     HAZARD_DEBRIS = "debris"
 
-    policies = []
-    threshold = 1
     threshold_flood = 0.2
     threshold_flood_distance = 10
     epsg = 3857 
-    #epsg = 21037 # Arc 1960 / UTM zone 37S
 
     # Replace strange TypeX LRS with RCi
     print('deneme',df_hazard.columns)
@@ -172,42 +163,40 @@ def compute(gdf_buildings, df_household, df_individual,gdf_intensity, df_hazard,
     number_of_unique_buildings = len(pd.unique(gdf_buildings['bldid']))
     print('number of unique building', number_of_unique_buildings)
     print('number of records in building layer ', len(gdf_buildings['bldid']))
-     
+
     # Convert both to the same target coordinate system
     gdf_buildings = gdf_buildings.set_crs("EPSG:4326",allow_override=True)
     gdf_intensity = gdf_intensity.set_crs("EPSG:4326",allow_override=True)
-    
+
     gdf_buildings = gdf_buildings.to_crs(f"EPSG:{epsg}")
     gdf_intensity = gdf_intensity.to_crs(f"EPSG:{epsg}")
 
-    #%%
-    gdf_building_intensity = geopandas.sjoin_nearest(gdf_buildings,gdf_intensity, 
+    gdf_building_intensity = gpd.sjoin_nearest(gdf_buildings,gdf_intensity, 
                 how='left', rsuffix='intensity',distance_col='distance')
-    #%%
     gdf_building_intensity = gdf_building_intensity.drop_duplicates(subset=['bldid'], keep='first')
-    # %%
+
+    gdf_building_intensity = gdf_building_intensity.merge(gdf_landuse[['zoneid','avgincome']],on='zoneid',how='left')
+
     # TODO: Check if the logic makes sense
     if hazard_type == HAZARD_FLOOD:
         away_from_flood = gdf_building_intensity['distance'] > threshold_flood_distance
         print('threshold_flood_distance',threshold_flood_distance)
         print('number of distant buildings', len(gdf_building_intensity.loc[away_from_flood, 'im']))
         gdf_building_intensity.loc[away_from_flood, 'im'] = 0
-    # %%
     gdf_building_intensity[['material','code_level','storeys','occupancy']] =  \
         gdf_building_intensity['expstr'].str.split('+',expand=True)
     gdf_building_intensity['height'] = gdf_building_intensity['storeys'].str.extract(r'([0-9]+)s').astype('int')
-    # %%
     lr = (gdf_building_intensity['height'] <= 4)
     mr = (gdf_building_intensity['height'] >= 5) & (gdf_building_intensity['height'] <= 8)
     hr = (gdf_building_intensity['height'] >= 9)
     gdf_building_intensity.loc[lr, 'height_level'] = 'LR'
     gdf_building_intensity.loc[mr, 'height_level'] = 'MR'
     gdf_building_intensity.loc[hr, 'height_level'] = 'HR'
-    # %% Earthquake uses simplified taxonomy
+    # Earthquake uses simplified taxonomy
     gdf_building_intensity['vulnstreq'] = \
         gdf_building_intensity[['material','code_level','height_level']] \
             .agg('+'.join,axis=1)
-    # %%
+    # 
     if hazard_type == HAZARD_EARTHQUAKE:
         bld_eq = gdf_building_intensity.merge(df_hazard, left_on='vulnstreq',right_on='expstr', how='left')
         nulls = bld_eq['muds1_g'].isna()
@@ -223,13 +212,18 @@ def compute(gdf_buildings, df_household, df_individual,gdf_intensity, df_hazard,
         for i in [1,2,3,4,5]:
             bld_eq[f'ds_{i}'] = np.abs(bld_eq[f'prob_ds{i-1}'] - bld_eq[f'prob_ds{i}'])
         df_ds = bld_eq[['ds_1','ds_2','ds_3','ds_4','ds_5']]
-        bld_eq['eq_ds'] = df_ds.idxmax(axis='columns').str.extract(r'ds_([0-9]+)').astype('int')
+        bld_eq['eq_ds'] = df_ds.idxmax(axis='columns').str.extract(r'ds_([0-9]+)').astype('int') - 1
 
+        if 1 in policies:
+            bld_eq.loc[bld_eq['occupancy'] == 'Res', 'eq_ds'] = 0
+        if 2 in policies:
+            bld_eq.loc[bld_eq['avgincome'] == 'lowIncomeA', 'eq_ds'] = 0
+            bld_eq.loc[bld_eq['avgincome'] == 'lowIncomeB', 'eq_ds'] = 0
         # Create a simplified building-hazard relation
-        bld_hazard = bld_eq[['bldid','occupancy','eq_ds']]
+        bld_hazard = bld_eq[['bldid','eq_ds']]
         bld_hazard = bld_hazard.rename(columns={'eq_ds':'ds'})
 
-        ds_str = {1: 'No Damage',2:'Low',3:'Medium',4:'High',5:'Collapsed'}
+        ds_str = {0: 'No Damage',1:'Low',2:'Medium',3:'High',4:'Collapsed'}
 
     elif hazard_type == HAZARD_FLOOD:
         bld_flood = gdf_building_intensity.merge(df_hazard, on='expstr', how='left')
@@ -241,128 +235,146 @@ def compute(gdf_buildings, df_household, df_individual,gdf_intensity, df_hazard,
         bld_flood['fl_prob'] = np.diag(flood_mapping(xnew))
         bld_flood['fl_ds'] = 0
         bld_flood.loc[bld_flood['fl_prob'] > threshold_flood,'fl_ds'] = 1
-
+        if 1 in policies:
+            bld_flood.loc[bld_flood['occupancy'] == 'Res', 'fl_ds'] = 0
+        if 2 in policies:
+            bld_flood.loc[bld_flood['avgincome'] == 'lowIncomeA', 'fl_ds'] = 0
+            bld_flood.loc[bld_flood['avgincome'] == 'lowIncomeB', 'fl_ds'] = 0
         # Create a simplified building-hazard relation
-        bld_hazard = bld_flood[['bldid','occupancy','fl_ds']]
+        bld_hazard = bld_flood[['bldid','fl_ds']]
         bld_hazard = bld_hazard.rename(columns={'fl_ds':'ds'})
 
         ds_str = {0: 'No Damage',1:'Flooded'}
-    # %%
-    bld_hazard['occupancy'] = pd.Categorical(bld_hazard['occupancy'])
-    for key, value in ds_str.items():
-        bld_hazard.loc[bld_hazard['ds'] == key,'damage_level'] = value
-    bld_hazard['damage_level'] = pd.Categorical(bld_hazard['damage_level'], list(ds_str.values()))
+    
+    return bld_hazard
+
+
+def calculate_metrics(gdf_buildings, df_household, df_individual, hazard_type, policies=[]):
+    # only use necessary columns
+    bld_hazard = gdf_buildings[['bldid','ds','expstr']]
+    bld_hazard[['material','code_level','storeys','occupancy']] =  \
+        bld_hazard['expstr'].str.split('+',expand=True).copy()
+    #bld_hazard['occupancy'] = bld_hazard['occupancy'].astype('category')
 
-    #%% Find the damage state of the building that the household is in
+    # Find the damage state of the building that the household is in
     df_household_bld = df_household.merge(bld_hazard[['bldid','ds']], on='bldid', how='left',validate='many_to_one')
 
-    #%% find the damage state of the hospital that the household is associated with
-    df_hospitals = df_household.merge(bld_hazard[['bldid','damage_level', 'ds']], 
+    # Find the damage state of the hospital that the household is associated with
+    df_hospitals = df_household.merge(bld_hazard[['bldid', 'ds']], 
             how='left', left_on='commfacid', right_on='bldid', suffixes=['','_comm'],
             validate='many_to_one')
 
-    #%%
-    df_individual_occupancy = df_individual.merge(bld_hazard[['bldid','occupancy','damage_level', 'ds']], 
+    # Find the occupancy of facility that the individual is associated
+    df_individual_occupancy = df_individual.merge(bld_hazard[['bldid','occupancy','ds']], 
                         how='inner',left_on='indivfacid',right_on='bldid',
                         suffixes=['_l','_r'],validate='many_to_one')
 
-    #%%
+    # Filtering working places
     df_workers = df_individual_occupancy.query('occupancy in ["Com","ResCom","Ind"]')
 
-    #%%
+    # Filtering schools
     df_students = df_individual_occupancy.query('occupancy in ["Edu"]')
 
-    #%%
-    df_indiv_hosp = df_individual.merge(df_hospitals[['hhid','ds','bldid']], 
-                    how='left', on='hhid', validate='many_to_one')
-    #%%
+    # connect individuals to damage state of associated hospitals
+    df_indiv_hosp = df_individual.merge(df_hospitals[['hhid','ds']], 
+                        how='left', on='hhid', validate='many_to_one')
 
     # get the ds of household that individual lives in
     df_indiv_household = df_individual[['hhid','individ']].merge(df_household_bld[['hhid','ds']])
 
+    # Collect all damage states in a single table
     df_displaced_indiv = df_indiv_hosp.rename(columns={'ds':'ds_hospital'})\
         .merge(df_workers[['individ','ds']].rename(columns={'ds':'ds_workplace'}),on='individ', how='left')\
         .merge(df_students[['individ','ds']].rename(columns={'ds':'ds_school'}), on='individ', how='left')\
-        .merge(df_indiv_household[['individ','ds']].rename(columns={'ds':'ds_household'}), on='individ',how='left')
+        .merge(df_indiv_household[['individ','ds']].rename(columns={'ds':'ds_household'}), on='individ',how='left')\
+        .merge(df_household[['hhid','bldid']],on='hhid',how='left')
+
+    DS_NO = 0
+    DS_SLIGHT = 1
+    DS_MODERATE = 2
+    DS_EXTENSIZE = 3
+    DS_COLLAPSED = 4
+
+    # Hazard Types 
+    HAZARD_EARTHQUAKE = "earthquake"
+    HAZARD_FLOOD = "flood"
+    HAZARD_DEBRIS = "debris"
 
-    # %%
-    #%%
     if hazard_type == HAZARD_EARTHQUAKE:
-        # Effect of policies on thresholds
-        # First get the global threshold
-        thresholds = {f'metric{id}': threshold for id in range(8)}
-        # Policy-1: Loans for reconstruction for minor to moderate damages
-        # Changes: Damage state thresholds for “displacement”
-        # Increase thresholds from “slight to moderate” as fewer people will be displaced.
-        if 21 in policies and thresholds['metric7'] == DS_NO:
-            thresholds['metric7'] = DS_SLIGHT
-
-        # Policy-3: Cat-bond agreement for education and health facilities
-        # Changes: Damage state thresholds for “loss of access to hospitals” and “loss of access to schools”
-        # Increase thresholds from “slight to moderate” as fewer people will be displaced.
-        if 23 in policies and thresholds['metric3'] == DS_NO:
-            thresholds['metric3'] = DS_SLIGHT
-        if 23 in policies and thresholds['metric2'] == DS_NO:
-            thresholds['metric2'] = DS_SLIGHT
-
-        # Policy-2: Knowledge sharing about DRR in public and private schools
-        # Changes: Damage state thresholds for “loss of school access”
-        # Increase thresholds loss of school access to beyond current scale. So that the impact will be downgraded to “0”.
-        if 22 in policies:
-            thresholds['metric2'] = DS_COLLAPSED
-    elif hazard_type == HAZARD_FLOOD:
-        # For flood, there are only two states: 0 or 1. 
-        # So threshold is set to 0. 
-        thresholds = {f'metric{id}': 0 for id in range(8)}
+    # Effect of policies on thresholds
+    # First get the global threshold
+        thresholds = {f'metric{id}': DS_SLIGHT for id in range(8)}
+    else:
+        # Default thresholds for flood and debris
+        # For flood, there are only two states: 0 or 1.
+        # So threshold is set to 0.
+        thresholds = {f'metric{id}': DS_NO for id in range(8)}
+
+    # metric 1 number of unemployed workers in each building
+    df_workers_per_building = df_workers[df_workers['ds'] > thresholds['metric1']][['individ','hhid','ds']].merge(
+        df_household[['hhid','bldid']],on='hhid',how='left').groupby(
+            'bldid',as_index=False).agg({'individ':'count'})
 
-    #%% metric 1 number of unemployed workers in each building
-    df_workers_per_building = df_workers[df_workers['ds'] > thresholds['metric1']].groupby('bldid',as_index=False).agg({'individ':'count'})
-    
     df_metric1 = bld_hazard.merge(df_workers_per_building,how='left',left_on='bldid',right_on = 'bldid')[['bldid','individ']]
     df_metric1.rename(columns={'individ':'metric1'}, inplace=True)
     df_metric1['metric1'] = df_metric1['metric1'].fillna(0).astype(int)
 
-    #%% metric 2 number of students in each building with no access to schools
-    df_students_per_building = df_students[df_students['ds'] > thresholds['metric2']].groupby('bldid',as_index=False).agg({'individ':'count'})
+    # metric 2 number of students in each building with no access to schools
+    df_students_per_building = df_students[df_students['ds'] > thresholds['metric2']][['individ','hhid','ds']].merge(
+        df_household[['hhid','bldid']],on='hhid',how='left').groupby(
+            'bldid',as_index=False).agg({'individ':'count'})
+
     df_metric2 = bld_hazard.merge(df_students_per_building,how='left',left_on='bldid',right_on = 'bldid')[['bldid','individ']]
     df_metric2.rename(columns={'individ':'metric2'}, inplace=True)
     df_metric2['metric2'] = df_metric2['metric2'].fillna(0).astype(int)
 
-    #%% metric 3 number of households in each building with no access to hospitals
-    df_hospitals_per_household = df_hospitals[df_hospitals['ds'] > thresholds['metric3']].groupby('bldid',as_index=False).agg({'hhid':'count'})
+    # metric 3 number of households in each building with no access to hospitals
+    df_hospitals_per_household = df_hospitals[df_hospitals['ds'] > thresholds['metric3']].groupby(
+        'bldid',as_index=False).agg({'hhid':'count'})
+
     df_metric3 = bld_hazard.merge(df_hospitals_per_household,how='left',left_on='bldid',right_on='bldid')[['bldid','hhid']]
     df_metric3.rename(columns={'hhid':'metric3'}, inplace=True)
     df_metric3['metric3'] = df_metric3['metric3'].fillna(0).astype(int)
 
-    #%% metric 4 number of individuals in each building with no access to hospitals
-    df_hospitals_per_individual = df_hospitals[df_hospitals['ds'] > thresholds['metric4']].groupby('bldid',as_index=False).agg({'nind':'sum'})
+    # metric 4 number of individuals in each building with no access to hospitals
+    df_hospitals_per_individual = df_hospitals[df_hospitals['ds'] > thresholds['metric4']].groupby(
+        'bldid',as_index=False).agg({'nind':'sum'})
+
     df_metric4 = bld_hazard.merge(df_hospitals_per_individual,how='left',left_on='bldid',right_on='bldid')[['bldid','nind']]
     df_metric4.rename(columns={'nind':'metric4'}, inplace=True)
     df_metric4['metric4'] = df_metric4['metric4'].fillna(0).astype(int)
 
-    #%% metric 5 number of damaged households in each building
-    df_homeless_households = df_household_bld[df_household_bld['ds'] > thresholds['metric5']].groupby('bldid',as_index=False).agg({'hhid':'count'})
+    # metric 5 number of damaged households in each building
+    df_homeless_households = df_household_bld[df_household_bld['ds'] > thresholds['metric5']].groupby(
+        'bldid',as_index=False).agg({'hhid':'count'})
+
     df_metric5 = bld_hazard.merge(df_homeless_households,how='left',left_on='bldid',right_on='bldid')[['bldid','hhid']]
     df_metric5.rename(columns={'hhid':'metric5'}, inplace=True)
     df_metric5['metric5'] = df_metric5['metric5'].fillna(0).astype(int)
 
-    #%% metric 6 number of homeless individuals in each building
-    df_homeless_individuals = df_household_bld[df_household_bld['ds'] > thresholds['metric6']].groupby('bldid',as_index=False).agg({'nind':'sum'})
+    # metric 6 number of homeless individuals in each building
+    df_homeless_individuals = df_household_bld[df_household_bld['ds'] > thresholds['metric6']].groupby(
+        'bldid',as_index=False).agg({'nind':'sum'})
+
     df_metric6 = bld_hazard.merge(df_homeless_individuals,how='left',left_on='bldid',right_on='bldid')[['bldid','nind']]
     df_metric6.rename(columns={'nind':'metric6'}, inplace=True)
     df_metric6['metric6'] = df_metric6['metric6'].fillna(0).astype(int)
 
-    #%% metric 7 the number of displaced individuals in each building
+    # metric 7 the number of displaced individuals in each building
     # more info: an individual is displaced if at least of the conditions below hold
     df_disp_per_bld = df_displaced_indiv[(df_displaced_indiv['ds_household'] > thresholds['metric6']) |
-                                        (df_displaced_indiv['ds_school'] > thresholds['metric7']) |
-                                        (df_displaced_indiv['ds_workplace'] > thresholds['metric7']) |
-                                        (df_displaced_indiv['ds_hospital'] > thresholds['metric7'])].groupby('bldid',as_index=False).agg({'individ':'count'})
+                                        (df_displaced_indiv['ds_school'] > thresholds['metric2']) |
+                                        (df_displaced_indiv['ds_workplace'] > thresholds['metric1']) |
+                                        (df_displaced_indiv['ds_hospital'] > thresholds['metric4'])]\
+                                            .groupby('bldid',as_index=False)\
+                                            .agg({'individ':'count'})
+
     df_metric7 = bld_hazard.merge(df_disp_per_bld,how='left',left_on='bldid',right_on='bldid')[['bldid','individ']]
     df_metric7.rename(columns={'individ':'metric7'}, inplace=True)
     df_metric7['metric7'] = df_metric7['metric7'].fillna(0).astype(int)
 
-    #%%
+
+
     df_metrics = {'metric1': df_metric1,
                 'metric2': df_metric2,
                 'metric3': df_metric3,
@@ -371,7 +383,7 @@ def compute(gdf_buildings, df_household, df_individual,gdf_intensity, df_hazard,
                 'metric6': df_metric6,
                 'metric7': df_metric7}
 
-    #%%
+
     number_of_workers = len(df_workers)
     print('number of workers', number_of_workers)
 
@@ -387,7 +399,7 @@ def compute(gdf_buildings, df_household, df_individual,gdf_intensity, df_hazard,
                 "metric2": {"desc": "Number of children with no access to education", "value": 0, "max_value": number_of_students},
                 "metric3": {"desc": "Number of households with no access to hospital", "value": 0, "max_value": number_of_households},
                 "metric4": {"desc": "Number of individuals with no access to hospital", "value": 0, "max_value": number_of_individuals},
-                "metric5": {"desc": "Number of homeless households", "value": 0, "max_value": number_of_households},
+                "metric5": {"desc": "Number of households displaced", "value": 0, "max_value": number_of_households},
                 "metric6": {"desc": "Number of homeless individuals", "value": 0, "max_value": number_of_individuals},
                 "metric7": {"desc": "Population displacement", "value": 0, "max_value": number_of_individuals},}
     metrics["metric1"]["value"] = int(df_metric1['metric1'].sum())
@@ -398,5 +410,4 @@ def compute(gdf_buildings, df_household, df_individual,gdf_intensity, df_hazard,
     metrics["metric6"]["value"] = int(df_metric6['metric6'].sum())
     metrics["metric7"]["value"] = int(df_metric7['metric7'].sum())
 
-    return metrics, df_metrics, bld_hazard
-    
+    return metrics, df_metrics

tomorrowcities/content/articles/metrics.md

@@ -0,0 +1,42 @@
+---
+author: huseyin.kaya
+title: Metrics
+description: A brief introduction to metrics
+image: https://images.unsplash.com/photo-1429041966141-44d228a42775?ixid=MXwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHw%3D&ixlib=rb-1.2.1&auto=format&fit=crop&w=2500&q=80
+thumbnail: https://images.unsplash.com/photo-1429041966141-44d228a42775?ixid=MXwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHw%3D&ixlib=rb-1.2.1&auto=format&fit=crop&w=350&q=80
+alt: "Metrics"
+createdAt: 2023-10-10
+duration: 6 min read
+category:
+  - general
+---
+
+## Metrics
+There are seven fundamental impact metrics displayed in the web application. For each them there is an associcated threhold but for the sake of simplicity, it is not explicitly stated.
+
+### Metric 1: Number of workers unemployed
+It denotes the number of invidivuals who last their jobs either due to a severe damage at the workplace 
+or lost access to a functional workplace. When the metrics is displayed on a building-level, it is the sum of
+such individuals living in that building.
+
+### Metric 2: Number of children with no access to education
+Similar to the first metrics but individual here refers to a child who is associated with a school.
+The metric becomes active if the school is damaged or not accessible.
+
+### Metric 3: Number of households with no access to hospital
+The number of households who lost its access to its associated hospital.
+Either hospital is damaged or the path between the household and the hospital
+is broken.
+
+### Metric 4: Number of individuals with no access to hospital
+It is derived from metric 3 by counting the individuals in the corresponding households.
+
+### Metric 5: Number of households displaced
+It is direct result of building damage state. If a building is damaged, then the households in it are also
+damaged.
+
+### Metric 6: Number of homeless individuals
+It is derived from metric 5.
+
+### Metric 7: Population displacement
+An individual is assumed to be displaced if the associated household is damaged, or he/she lost access to workplace, schoold or hospital. 

tomorrowcities/content/articles/policies.md

@@ -0,0 +1,22 @@
+---
+author: huseyin.kaya
+title: Policies in a nutshell
+description: A brief introduction to policies
+image: https://images.unsplash.com/photo-1429041966141-44d228a42775?ixid=MXwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHw%3D&ixlib=rb-1.2.1&auto=format&fit=crop&w=2500&q=80
+thumbnail: https://images.unsplash.com/photo-1429041966141-44d228a42775?ixid=MXwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHw%3D&ixlib=rb-1.2.1&auto=format&fit=crop&w=350&q=80
+alt: "Policies"
+createdAt: 2023-10-10
+duration: 6 min read
+category:
+  - general
+---
+
+### Policy 1: Land and tenure security program
+When this policy applied, the code levels of the residential buildings are set to higher standards 
+which effectively reduces the damage states during hazard simulations. Since this policy only changes
+the code levels, the computing engine is required to run again to see its effect.
+
+### Policy 2: State-led upgrading/retrofitting of low-income/informal housing
+When applied, this policy increase the code level of the buildings located in low income A and B zones.
+This effectively reduces the damage states, and hence several impact metrics, starting from building-level
+metrics.

tomorrowcities/content/articles/welcome.md

@@ -14,6 +14,17 @@ category:
 ## Tomorrow's Cities Decision Support Environment (TCDSE)
 TCDSE is a web application designed to conduct computational tasks to generate information needed for decision mechanisms in designing future cities. The web application, which will be referred as TCDSE for short, contains a computational engine capable of executing several hazard scenarios on different exposure datasets and infrastructures. 
 
+## What is New?
+
+### Policy Support
+Two new policies are added to the computing engine. More info is [here](/docs/policies)
+
+### Excel support:
+You can  upload Excel files containing your tabular data such as individual, household, fragility or vulnerability data. However, processing excel files is very slower than processing JSON files so I definitely suggest working with JSON files. You can convert your Excel files via panda framework. The platform also does not try to convert the coordinates even if there is any in the Excel file because there is no way to know which columns represent the coordinates or coordinate reference systems without metadata. So Excel spreadsheets should be used to provide non-geo related tabular data.  So please use them only for data not containing any geo-specific information. 
+
+### Dar Es Salaam Study
+[Sample Dataset](https://drive.google.com/file/d/1BGPZQ2IKJHY9ExOCCHcNNrCTioYZ8D1y/view?usp=sharing) now contains some visioning scenario for Dar Es Salaam case.
+
 ## Features
 General capabilities/features of the web application can be summarized as follows:
 

tomorrowcities/pages/engine.py

@@ -10,10 +10,10 @@ from typing import Tuple, Optional
 import ipyleaflet
 from ipyleaflet import AwesomeIcon, Marker
 import numpy as np
-import sys
-
-from ..backend.engine import compute, compute_power_infra
+import logging, sys
+#logging.basicConfig(stream=sys.stderr, level=logging.INFO)
 
+from ..backend.engine import compute, compute_power_infra, calculate_metrics
 
 layers = solara.reactive({
     'layers' : {
@@ -23,6 +23,7 @@ layers = solara.reactive({
             'force_render': solara.reactive(False),
             'visible': solara.reactive(False),
             'extra_cols': {'ds': 0, 'metric1': 0, 'metric2': 0, 'metric3': 0,'metric4': 0, 'metric5': 0,'metric6': 0,'metric7': 0},
+            'cols_required': set(['residents', 'fptarea', 'repvalue', 'nhouse', 'zoneid', 'expstr', 'bldid', 'geometry', 'specialfac']),
             'cols': set(['residents', 'fptarea', 'repvalue', 'nhouse', 'zoneid', 'expstr', 'bldid', 'geometry', 'specialfac'])},
         'landuse': {
             'df': solara.reactive(None),
@@ -30,6 +31,7 @@ layers = solara.reactive({
             'force_render': solara.reactive(False),
             'visible': solara.reactive(False),
             'extra_cols': {},
+            'cols_required': set(['geometry', 'zoneid', 'luf', 'population', 'densitycap', 'avgincome']),
             'cols': set(['geometry', 'zoneid', 'luf', 'population', 'densitycap', 'floorarat', 'setback', 'avgincome'])},
         'household': {
             'df': solara.reactive(None),
@@ -37,6 +39,7 @@ layers = solara.reactive({
             'force_render': solara.reactive(False),
             'visible': solara.reactive(False),
             'extra_cols': {},
+            'cols_required':set(['hhid', 'nind', 'income', 'bldid', 'commfacid']),
             'cols':set(['hhid', 'nind', 'income', 'bldid', 'commfacid'])},
         'individual': {
             'df': solara.reactive(None),
@@ -44,6 +47,7 @@ layers = solara.reactive({
             'force_render': solara.reactive(False),
             'visible': solara.reactive(False),
             'extra_cols': {},
+            'cols_required': set(['individ', 'hhid', 'gender', 'age', 'eduattstat', 'head', 'indivfacid']),
             'cols': set(['individ', 'hhid', 'gender', 'age', 'eduattstat', 'head', 'indivfacid'])},
         'intensity': {
             'df': solara.reactive(None),
@@ -51,6 +55,7 @@ layers = solara.reactive({
             'force_render': solara.reactive(False),
             'visible': solara.reactive(False),
             'extra_cols': {},
+            'cols_required': set(['geometry','im']),
             'cols': set(['geometry','im'])},
         'fragility': {
             'df': solara.reactive(None),
@@ -58,6 +63,7 @@ layers = solara.reactive({
             'force_render': solara.reactive(False),
             'visible': solara.reactive(False),
             'extra_cols': {},
+            'cols_required': set(['expstr','muds1_g','muds2_g','muds3_g','muds4_g','sigmads1','sigmads2','sigmads3','sigmads4']),
             'cols': set(['expstr','muds1_g','muds2_g','muds3_g','muds4_g','sigmads1','sigmads2','sigmads3','sigmads4'])},
         'vulnerability': {
             'df': solara.reactive(None),
@@ -65,6 +71,7 @@ layers = solara.reactive({
             'force_render': solara.reactive(False),
             'visible': solara.reactive(False),
             'extra_cols': {},
+            'cols_required': set(['expstr', 'hw0', 'hw0_5', 'hw1', 'hw1_5', 'hw2', 'hw3', 'hw4', 'hw5','hw6']),
             'cols': set(['expstr', 'hw0', 'hw0_5', 'hw1', 'hw1_5', 'hw2', 'hw3', 'hw4', 'hw5','hw6'])},
         'power nodes': {
             'df': solara.reactive(None),
@@ -72,6 +79,9 @@ layers = solara.reactive({
             'force_render': solara.reactive(False),
             'visible': solara.reactive(False),
             'extra_cols': {'ds': 0, 'is_damaged': False, 'is_operational': True},
+            'cols_required': set(['geometry', 'fltytype', 'strctype', 'utilfcltyc', 'indpnode', 'guid', 
+                         'node_id', 'x_coord', 'y_coord', 'pwr_plant', 'serv_area', 'n_bldgs', 
+                         'income', 'eq_vuln']),
             'cols': set(['geometry', 'fltytype', 'strctype', 'utilfcltyc', 'indpnode', 'guid', 
                          'node_id', 'x_coord', 'y_coord', 'pwr_plant', 'serv_area', 'n_bldgs', 
                          'income', 'eq_vuln'])},
@@ -81,6 +91,8 @@ layers = solara.reactive({
             'force_render': solara.reactive(False),
             'visible': solara.reactive(False),
             'extra_cols': {},
+            'cols_required': set(['from_node', 'direction', 'pipetype', 'edge_id', 'guid', 'capacity', 
+                         'geometry', 'to_node', 'length']),
             'cols': set(['from_node', 'direction', 'pipetype', 'edge_id', 'guid', 'capacity', 
                          'geometry', 'to_node', 'length'])},
         'power fragility': {
@@ -89,6 +101,8 @@ layers = solara.reactive({
             'force_render': solara.reactive(False),
             'visible': solara.reactive(False),
             'extra_cols': {},
+            'cols_required': set(['vuln_string', 'med_slight', 'med_moderate', 'med_extensive', 'med_complete', 
+                         'beta_slight', 'beta_moderate', 'beta_extensive', 'beta_complete']),
             'cols': set(['vuln_string', 'med_slight', 'med_moderate', 'med_extensive', 'med_complete', 
                          'beta_slight', 'beta_moderate', 'beta_extensive', 'beta_complete', 'description'])}
             },
@@ -96,12 +110,16 @@ layers = solara.reactive({
     'selected_layer' : solara.reactive(None),
     'render_count': solara.reactive(0),
     'bounds': solara.reactive(None),
+    'policies': {
+        '1': {'id':1, 'label': 'Policy 1', 'description': 'Land and tenure security program', 'applied': solara.reactive(False)},
+        '2': {'id':2, 'label': 'Policy 2', 'description': 'State-led upgrading/retrofitting of low-income/informal housing', 'applied': solara.reactive(False)},
+    },
     'metrics': {
         "metric1": {"desc": "Number of workers unemployed", "value": 0, "max_value": 100},
         "metric2": {"desc": "Number of children with no access to education", "value": 0, "max_value": 100},
         "metric3": {"desc": "Number of households with no access to hospital", "value": 0, "max_value": 100},
         "metric4": {"desc": "Number of individuals with no access to hospital", "value": 0, "max_value": 100},
-        "metric5": {"desc": "Number of homeless households", "value": 0, "max_value": 100},
+        "metric5": {"desc": "Number of households displaced", "value": 0, "max_value": 100},
         "metric6": {"desc": "Number of homeless individuals", "value": 0, "max_value": 100},
         "metric7": {"desc": "Population displacement", "value": 0, "max_value":100},}})
 
@@ -200,21 +218,35 @@ def MetricWidget(name, description, value, max_value, render_count):
             solara.FigureEcharts(option=options, attributes={ "style": "height: 100px; width: 100px" })
 
 
-def import_data(data: Optional[bytes]):
-    json_string = data.decode('utf-8')
-    json_data = json.loads(json_string)
-    if "features" in json_data.keys():
-        df = gpd.GeoDataFrame.from_features(json_data['features'])
+def import_data(fileinfo: solara.components.file_drop.FileInfo):
+    data = fileinfo['data']
+    extension = fileinfo['name'].split('.')[-1]
+    if extension == 'xlsx':
+        df = pd.read_excel(data)
     else:
-        df = pd.read_json(json_string)
+        json_string = data.decode('utf-8')
+        json_data = json.loads(json_string)
+        if "features" in json_data.keys():
+            df = gpd.GeoDataFrame.from_features(json_data['features'])
+        else:
+            df = pd.read_json(json_string)
 
     df.columns = df.columns.str.lower()
 
+    # in the first pass, look for exact column match
     name = None
     for layer_name, layer in layers.value['layers'].items():
         if layer['cols'] == set(df.columns):
             name = layer_name
             break
+    # if not, check only the required columns
+    if name is None:
+        for layer_name, layer in layers.value['layers'].items():
+            if layer['cols_required'].issubset(set(df.columns)):
+                name = layer_name
+                logging.debug('There are extra columns', set(df.columns) - layer['cols_required'])
+                break
+    
 
     # Inject columns
     if name is not None:
@@ -232,7 +264,7 @@ def FileDropZone():
     def load():
         if fileinfo is not None:
             print('processing file')
-            name, df = import_data(fileinfo['data'])
+            name, df = import_data(fileinfo)
             if name is not None and df is not None:
                 layers.value['layers'][name]['df'].set(df)
                 layers.value['selected_layer'].set(name)
@@ -278,7 +310,7 @@ def FileDropZone():
 
 @solara.component
 def LayerDisplayer():
-
+    print(f'{layers.value["bounds"].value}')
     nonempty_layers = {name: layer for name, layer in layers.value['layers'].items() if layer['df'].value is not None}
     nonempty_layer_names = list(nonempty_layers.keys())
     selected = layers.value['selected_layer'].value
@@ -290,9 +322,13 @@ def LayerDisplayer():
     if selected is None and len(nonempty_layer_names) > 0:
         set_selected(nonempty_layer_names[0])
     if selected is not None:
-        DataframeDisplayer(nonempty_layers[selected]['df'].value, 
-                           layers.value['render_count'].value,
-                           layers.value['bounds'].value)
+        df = nonempty_layers[selected]['df'].value
+        if "geometry" in df.columns:
+            ((ymin,xmin),(ymax,xmax)) = layers.value['bounds'].value
+            solara.DataFrame(df.cx[xmin:xmax,ymin:ymax].drop(columns='geometry'))
+        else:
+            solara.DataFrame(df)
+
 
 @solara.component
 def MetricPanel():
@@ -310,6 +346,8 @@ def MetricPanel():
                          filtered_metrics[name], 
                          metric['max_value'],
                          layers.value['render_count'].value)
+        with solara.Link("/docs/metrics"):
+            solara.Button(icon_name="mdi-help-circle-outline", icon=True)
   
 
 @solara.component
@@ -329,9 +367,6 @@ def MapViewer():
     zoom, set_zoom = solara.use_state(default_zoom)
     #center, set_center = solara.use_state(default_center)
 
-    def set_bounds(bounds):
-        layers.value['bounds'].set(bounds)
-
     base_map = ipyleaflet.basemaps["Stamen"]["Watercolor"]
     base_layer = ipyleaflet.TileLayer.element(url=base_map.build_url())
     map_layers = [base_layer]
@@ -350,7 +385,7 @@ def MapViewer():
     ipyleaflet.Map.element(
         zoom=zoom,
         on_zoom=set_zoom,
-        on_bounds=set_bounds,
+        on_bounds=layers.value['bounds'].set,
         center=layers.value['center'].value,
         on_center=layers.value['center'].set,
         scroll_wheel_zoom=True,
@@ -361,21 +396,11 @@ def MapViewer():
         keyboard=True if random.random() > 0.5 else False,
         layers=map_layers
         )
-    
-
-@solara.component
-def DataframeDisplayer(df, render_count, bounds):
-    df, set_df = solara.use_state_or_update(df)
-    if "geometry" in df.columns:
-        ((ymin,xmin),(ymax,xmax)) = bounds
-        solara.DataFrame(df.cx[xmin:xmax,ymin:ymax].drop(columns='geometry'))
-    else:
-        solara.DataFrame(df)
-    
+        
 @solara.component
 def ExecutePanel():
-    infra, set_infra = solara.use_state(["power"])
-    hazard, set_hazard = solara.use_state("earthquake")
+    infra, set_infra = solara.use_state(["building"])
+    hazard, set_hazard = solara.use_state("flood")
 
 
     execute_counter, set_execute_counter = solara.use_state(0)
@@ -394,12 +419,12 @@ def ExecutePanel():
             if "power" in  infra:
                 missing += list(set(["power edges","power nodes","intensity","power fragility"]) - existing_layers)
             if "building" in infra:
-                missing += list(set(["building","household","individual","intensity","fragility"]) - existing_layers)
+                missing += list(set(["landuse","building","household","individual","intensity","fragility"]) - existing_layers)
         elif hazard == "flood":
             if "power" in  infra:
                 missing += list(set(["power edges","power nodes","intensity","power vulnerability"]) - existing_layers)
             if "building" in infra:
-                missing += list(set(["building","household","individual","intensity","vulnerability"]) - existing_layers)
+                missing += list(set(["landuse","building","household","individual","intensity","vulnerability"]) - existing_layers)
  
         if infra == []:
             missing += ['You should select power and/or building']
@@ -429,6 +454,7 @@ def ExecutePanel():
             return nodes
 
         def execute_building():
+            landuse = layers.value['layers']['landuse']['df'].value
             buildings = layers.value['layers']['building']['df'].value
             household = layers.value['layers']['household']['df'].value
             individual = layers.value['layers']['individual']['df'].value
@@ -436,20 +462,27 @@ def ExecutePanel():
 
             fragility = layers.value['layers']['fragility']['df'].value
             vulnerability = layers.value['layers']['vulnerability']['df'].value
-            computed_metrics, df_metrics, df_bld_hazard = compute(
+
+            policies = [p['id'] for id, p in layers.value['policies'].items() if p['applied'].value]
+
+            print('policies',policies)
+            df_bld_hazard = compute(
+                landuse,
                 buildings, 
                 household, 
                 individual,
                 intensity,
                 fragility if hazard == "earthquake" else vulnerability, 
-                hazard)
-
+                hazard,policies=policies)
+            buildings['ds'] = list(df_bld_hazard['ds'])
+            computed_metrics, df_metrics = calculate_metrics(buildings, household, individual, hazard, policies=policies)
+        
             print(computed_metrics)
             for metric in df_metrics.keys():
                 buildings[metric] = list(df_metrics[metric][metric])
                 layers.value['metrics'][metric]['value'] = computed_metrics[metric]['value']
                 layers.value['metrics'][metric]['max_value'] = computed_metrics[metric]['max_value']
-            buildings['ds'] = list(df_bld_hazard['ds'])
+            
             return buildings
 
         if execute_counter > 0 :
@@ -477,10 +510,12 @@ def ExecutePanel():
     result = solara.use_thread(execute_engine, dependencies=[execute_counter])
 
     with solara.Row(justify="center"):
-        solara.ToggleButtonsMultiple(value=infra, on_value=set_infra, values=["power","building"])
+        solara.ToggleButtonsMultiple(value=infra, on_value=set_infra, values=["building","power"])
     with solara.Row(justify="center"):
         solara.ToggleButtonsSingle(value=hazard, on_value=set_hazard, values=["earthquake","flood"])
 
+    PolicyPanel()
+    solara.ProgressLinear(value=False)
     solara.Button("Calculate", on_click=on_click, outlined=True,
                   disabled=execute_btn_disabled)
     # The statements in this block are passed several times during thread execution
@@ -499,13 +534,22 @@ def ExecutePanel():
         set_execute_btn_disabled(False)
         solara.ProgressLinear(value=False)
 
+@solara.component
+def PolicyPanel():
+    with solara.Row():
+        for policy_key, policy in layers.value['policies'].items():
+            with solara.Tooltip(tooltip=policy['description']):
+                solara.Checkbox(label=policy['label'], 
+                                value=policy['applied'])
+        with solara.Link("/docs/policies"):
+            solara.Button(icon_name="mdi-help-circle-outline", icon=True)
+
 
 @solara.component
 def WebApp():
 
     with solara.Columns([30,60]):
         with solara.Column():
-
             solara.Markdown('[Download Sample Dataset](https://drive.google.com/file/d/1BGPZQ2IKJHY9ExOCCHcNNrCTioYZ8D1y/view?usp=sharing)')
             FileDropZone()
             ExecutePanel()