Add AWS S3 support to save objects

pyproject.toml

@@ -20,8 +20,8 @@ dependencies = [
     "networkx",
     "openpyxl",
     "rasterio",
-    "xml",
     "boto3",
+    "cryptography",
 ]
 
 [tool.hatch.version]

tomorrowcities/content/articles/welcome.md

@@ -17,7 +17,8 @@ category:
 TCDSE is a web application designed to conduct computational tasks to generate information needed for decision mechanisms in designing future cities. The web application, which will be referred as TCDSE for short, contains a computational engine capable of executing several hazard scenarios on different exposure datasets and infrastructures. 
 
 ## What is New?
-* **New** The engine now can parse, display vulnerability curves located on [Global Vulnerability Model Reposity](https://github.com/gem/global_vulnerability_model) maintained by [Global Earthquake Model Foundation)](https://www.globalquakemodel.org/gem). To see the new fatures, download one of the XML files in  [Global Vulnerability Model Reposity](https://github.com/gem/global_vulnerability_model) and drag and drop to [Engine](/engine). The engine will read all the vulnerability functions defined in the XML file and display them. 
+* **New** Amazon S3 support to save session or policy database.
+* The engine now can parse, display vulnerability curves located on [Global Vulnerability Model Reposity](https://github.com/gem/global_vulnerability_model) maintained by [Global Earthquake Model Foundation)](https://www.globalquakemodel.org/gem). To see the new fatures, download one of the XML files in  [Global Vulnerability Model Reposity](https://github.com/gem/global_vulnerability_model) and drag and drop to [Engine](/engine). The engine will read all the vulnerability functions defined in the XML file and display them. 
 * basemap is changed to ESri.WorldImagery to see the landscapes especially rivers.
 * utilities page is added.
 * Excel to GeoJSON converted is added to utilities page.

tomorrowcities/pages/__init__.py

@@ -10,7 +10,7 @@ from cryptography.fernet import Fernet
 
 from ..data import articles
 
-route_order = ["/", "docs","engine","utilities","settings","account"]
+route_order = ["/", "docs","engine","utilities","policies","settings","account"]
 
 def check_auth(route, children):
     # This can be replaced by a custom function that checks if the user is
@@ -18,9 +18,8 @@ def check_auth(route, children):
 
     # routes that are public or only for admin
     # the rest only requires login
-    public_paths = ["/","docs","engine","utilities","account"]
-    admin_paths = ["settings"]
-
+    public_paths = ["/","docs","engine","utilities","policies","settings","account"]
+    admin_paths = [""]
 
     if route.path in public_paths:
         children_auth = children
@@ -34,121 +33,6 @@ def check_auth(route, children):
                 children_auth = children
     return children_auth
 
-
-class S3Storage:
-    def __init__(self, aws_access_key_id, aws_secret_access_key, region_name, bucket_name):
-        self.aws_access_key_id = aws_access_key_id
-        self.aws_secret_access_key = aws_secret_access_key
-        self.region_name = region_name
-        self.bucket_name = bucket_name
-        self.s3 = self.connect()
-    def connect(self):
-        session = boto3.Session(
-            aws_access_key_id=self.aws_access_key_id,
-            aws_secret_access_key=self.aws_secret_access_key,
-            region_name=self.region_name
-        )
-        return session.client('s3')
-
-    def is_alive(self):
-        try:
-            buckets = self.s3.list_buckets()
-            for bucket in buckets['Buckets']:
-                if bucket['Name'] == self.bucket_name:
-                    return True
-            return False
-        except:
-            return False
-        
-    def upload_file(self, file_name, object_name=None):
-        if object_name is None:
-            object_name = file_name
-        self.s3.upload_file(file_name, self.bucket_name, object_name)
-        return f"https://{self.bucket_name}.s3.amazonaws.com/{object_name}"
-
-    def load_metadata(self, session_name):
-        # Use the get_object method to read the file
-        self.s3.download_file(self.bucket_name, f'{session_name}.metadata', f'/tmp/{session_name}.metadata')
-
-        with open(f'/tmp/{session_name}.metadata', 'rb') as fileObj:
-            # Access the content of the file from the response
-            metadata = pickle.load(fileObj)
-
-        print(type(metadata))
-
-        return metadata
-    
-    def load_object(self, object_name):
-        # Use the get_object method to read the file
-        self.s3.download_file(self.bucket_name, object_name, f'/tmp/{object_name}')
-        print(f'Downloading {object_name}')
-
-        with open(f'/tmp/{object_name}', 'rb') as fileObj:
-            # Access the content of the file from the response
-            data = pickle.load(fileObj)
-        return data
-    
-    def load_data(self, session_name):
-        return self.load_object(f'{session_name}.data')
-    
-    def list_objects(self): 
-        objects = self.s3.list_objects(Bucket=self.bucket_name)
-        objects_array = set()
-        for obj in objects.get('Contents', []):
-            objects_array.add(obj["Key"].split('.')[:-1][0])
-        return [a for a in objects_array]
-    
-    def list_sessions(self):
-        objects = self.list_objects()
-        return [o for o in objects if "TCDSE_SESSION" in o]
-
-def get_S3(master_key:str, aws_access_key_id: str, aws_secret_access_key: str, region_name: str, bucket_name: str):
-    fernet = Fernet(master_key)
-
-    dec_aws_access_key_id = fernet.decrypt(aws_access_key_id).decode()
-    dec_aws_secret_access_key = fernet.decrypt(aws_secret_access_key).decode()
-    dec_region_name = fernet.decrypt(region_name).decode()
-    dec_bucket_name = fernet.decrypt(bucket_name).decode()
-    s3 = S3Storage(dec_aws_access_key_id, dec_aws_secret_access_key, dec_region_name, dec_bucket_name)
-    s3.connect()
-    return s3
-
-def revive_storage():
-    if 'master_key' in os.environ:
-        print('reading storage from env')
-        return get_S3(os.environ['master_key'],
-                    os.environ['aws_access_key_id'],
-                    os.environ['aws_secret_access_key'],
-                    os.environ['region_name'],
-                    os.environ['bucket_name'])
-    return None
-    
-
-storage = solara.reactive(cast(Optional[S3Storage], revive_storage()))  
-
-
-
-
-
-
-def storage_control(master_key:str, aws_access_key_id: str, aws_secret_access_key: str, region_name: str, bucket_name: str):
-    fernet = Fernet(master_key)
-
-    dec_aws_access_key_id = fernet.decrypt(aws_access_key_id).decode()
-    dec_aws_secret_access_key = fernet.decrypt(aws_secret_access_key).decode()
-    dec_region_name = fernet.decrypt(region_name).decode()
-    dec_bucket_name = fernet.decrypt(bucket_name).decode()
-    storage.value = S3Storage(dec_aws_access_key_id, dec_aws_secret_access_key, dec_region_name, dec_bucket_name)
-    storage.value.connect()
-    os.environ['master_key'] = master_key
-    os.environ['aws_access_key_id'] = aws_access_key_id
-    os.environ['aws_secret_access_key'] = aws_secret_access_key
-    os.environ['region_name'] = region_name
-    os.environ['bucket_name'] = bucket_name  
-
-def storage_disconnect():
-    storage.value = None
-
 @dataclasses.dataclass
 class User:
     username: str
@@ -179,8 +63,6 @@ def LoginForm():
         solara.Markdown(
             """
         This is an example login form.
-
-          * use admin/admin to login as admin.
           * use test/test to login as a normal user.
         """
         )
@@ -208,12 +90,6 @@ def Layout(children=[]):
             with solara.lab.Tabs(align="center"):
                 for route in routes:
                     name = route.path if route.path != "/" else "Welcome"
-                    is_admin = user.value and user.value.admin
-                    # we could skip the admin tab if the user is not an admin
-                    if route.path == "settings" and not is_admin:
-                         continue
-                    if user.value is not None and route.path == "logon":
-                        continue
                     # in this case we disable the tab
                     solara.lab.Tab(name, path_or_route=route, disabled=False)
             if user.value:

tomorrowcities/pages/account.py

@@ -5,55 +5,10 @@ import pprint
 
 from . import user
 from . import LoginForm
-from . import storage, storage_control, storage_disconnect
-from .engine import layers, load_from_state
 
 @solara.component
-def Page(name: Optional[str] = None, page: int = 0, page_size=100):
-    aws_access_key_id = solara.use_reactive('gAAAAABlNYcieCqS_iKRhX_J4LXea2hZ7UOqml4ebclJuGJaNpf0h_vOIYiYqLvPmHly8y8hcxBYr-_YUroLb5UG95xpPYTMjEhH_roobgY7hd7hTxJ9mB4=')
-    aws_secret_access_key = solara.use_reactive('gAAAAABlNYeL1SvCXf6DePlh2I6U_rrd9Izv1QlR6U9eev00bloKdUApf1r29l3UUh6G1PS9DagGLF228f3peWCrWdgZgT_-gUV3ueV6nHR9_QERYCWr0iaAUpcmqPVzIy81ESHqNZ2Z')
-    master_key = solara.use_reactive('')
-    region_name = solara.use_reactive('gAAAAABlNYfC41QcaN-_OwD0XwHGOY8BD38YtNXlvd8dWT74aUoLeERtw1OADP_jIKMqDAqvRMnoioqHONI6-um3NCeKlbG2rw==')
-    bucket_name = solara.use_reactive('gAAAAABlNYfYJwQCh6S9bO2npj7Qd1r9riEsGHxw6LAK5xz1DreQv7cpHdmepFtLhB8DlNDEKRDsRsFXD9zRPVMMc5JDcYiMQQ==')
-
-    session_name = solara.use_reactive(None)
-    session_list = solara.use_reactive([])
-
-
-    
-
-
-    def load_selected_session():
-        data = storage.value.load_data(session_name.value)
-        pprint.pprint(data)
-        load_from_state(data)
-
-    def refresh_session_list():
-        session_list.set(sorted(storage.value.list_sessions(),reverse=True))
-
-    #for k in ['aws_access_key_id', 'aws_secret_access_key','region_name','bucket_name']:
-    #    solara.Text(f'{k} --> {os.environ[k]}')
+def Page():
     solara.Title("TCDSE » Account")
     if user.value is None:
         LoginForm()
-    else:    
-        if storage.value is None:
-            with solara.Card(title='Attaching AWS S3',subtitle='Please attach an S3 bucket to save workspace sessions'):
-                solara.InputText(label='Master Key',value=master_key, password=True)
-                solara.InputText(label='AWS Access Key ID', value=aws_access_key_id, password=True,disabled=True)
-                solara.InputText(label='AWS Secret Access Key', value=aws_secret_access_key,password=True,disabled=True)
-                solara.InputText(label='AWS Region Name', value=region_name,password=True,disabled=True)
-                solara.InputText(label='Bucket Name', value=bucket_name,password=True,disabled=True)
-                solara.Button(label="Connect to S3", 
-                            on_click=lambda: storage_control(master_key.value, aws_access_key_id.value, 
-                                                            aws_secret_access_key.value, 
-                                                            region_name.value,
-                                                         bucket_name.value))
-        else:
-            with solara.Card(title='Load Session', subtitle='Choose a session from storage'):
-                solara.Select(label='Choose session',value=session_name.value, values=session_list.value,
-                            on_value=session_name.set)
-                solara.Button(label="Load", on_click=lambda: load_selected_session())
-                solara.Button(label="Refresh", on_click=lambda: refresh_session_list())
-            solara.Button(label="Disconnect from S3",on_click=lambda: storage_disconnect())
 

tomorrowcities/pages/engine.py

@@ -18,7 +18,7 @@ import logging, sys
 #logging.basicConfig(stream=sys.stderr, level=logging.INFO)
 import pickle
 import datetime
-from . import storage, storage_control, storage_disconnect
+from .settings import storage
 from ..backend.engine import compute, compute_power_infra, calculate_metrics
 
 
@@ -793,7 +793,7 @@ def MapViewer():
     zoom, set_zoom = solara.use_state(default_zoom)
     #center, set_center = solara.use_state(default_center)
 
-    base_map = ipyleaflet.basemaps["Stamen"]["Watercolor"]
+    base_map = ipyleaflet.basemaps["Esri"]["WorldStreetMap"]
     base_layer = ipyleaflet.TileLayer.element(url=base_map.build_url())
     map_layers = [base_layer]
 

tomorrowcities/pages/policies.py

@@ -0,0 +1,5 @@
+import solara
+
+@solara.component
+def Page():
+    solara.Markdown('Policies')

tomorrowcities/pages/settings.py

@@ -1,8 +1,167 @@
 import solara
+from typing import Optional, cast
+import pickle
+import boto3
+import os
 
 from . import user
 
+class S3Storage:
+    def __init__(self, aws_access_key_id, aws_secret_access_key, region_name, bucket_name):
+        self.aws_access_key_id = aws_access_key_id
+        self.aws_secret_access_key = aws_secret_access_key
+        self.region_name = region_name
+        self.bucket_name = bucket_name
+        self.s3 = self._connect()
+
+    def _connect(self):
+        session = boto3.Session(
+            aws_access_key_id=self.aws_access_key_id,
+            aws_secret_access_key=self.aws_secret_access_key,
+            region_name=self.region_name
+        )
+        return session.client('s3')
+
+    def is_alive(self):
+        print(self.aws_access_key_id)
+        print(self.aws_secret_access_key)
+        print(self.region_name)
+        print(self.bucket_name)
+        buckets = self.s3.list_buckets()
+        for bucket in buckets['Buckets']:
+            if bucket['Name'] == self.bucket_name:
+                return True
+        return False
+
+        
+    def upload_file(self, file_name, object_name=None):
+        if object_name is None:
+            object_name = file_name
+        self.s3.upload_file(file_name, self.bucket_name, object_name)
+        return f"https://{self.bucket_name}.s3.amazonaws.com/{object_name}"
+
+    def load_metadata(self, session_name):
+        # Use the get_object method to read the file
+        self.s3.download_file(self.bucket_name, f'{session_name}.metadata', f'/tmp/{session_name}.metadata')
+
+        with open(f'/tmp/{session_name}.metadata', 'rb') as fileObj:
+            # Access the content of the file from the response
+            metadata = pickle.load(fileObj)
+
+        print(type(metadata))
+
+        return metadata
+    
+    def load_object(self, object_name):
+        # Use the get_object method to read the file
+        self.s3.download_file(self.bucket_name, object_name, f'/tmp/{object_name}')
+        print(f'Downloading {object_name}')
+
+        with open(f'/tmp/{object_name}', 'rb') as fileObj:
+            # Access the content of the file from the response
+            data = pickle.load(fileObj)
+        return data
+    
+    def load_data(self, session_name):
+        return self.load_object(f'{session_name}.data')
+    
+    def list_objects(self): 
+        objects = self.s3.list_objects(Bucket=self.bucket_name)
+        objects_array = set()
+        for obj in objects.get('Contents', []):
+            objects_array.add(obj["Key"].split('.')[:-1][0])
+        return [a for a in objects_array]
+    
+    def list_sessions(self):
+        objects = self.list_objects()
+        return [o for o in objects if "TCDSE_SESSION" in o]
+
+def revive_storage():
+    if 'aws_access_key_id' in os.environ:
+        print('reviving storage from env')
+        return S3Storage(
+                    os.environ['aws_access_key_id'],
+                    os.environ['aws_secret_access_key'],
+                    os.environ['region_name'],
+                    os.environ['bucket_name'])
+    
+storage = solara.reactive(revive_storage())  
+
+def storage_control(aws_access_key_id: str, aws_secret_access_key: str, region_name: str, bucket_name: str):
+    storage.value = S3Storage(aws_access_key_id, aws_secret_access_key, region_name, bucket_name)
+
+
+def storage_disconnect():
+    print('Disconnecting S3')
+    storage.set(None)
+    session_name.set(None)
+    session_list.set([])
+    os.environ.pop('aws_access_key_id')
+    os.environ.pop('aws_secret_access_key')
+    os.environ.pop('region_name')
+    os.environ.pop('bucket_name')
+    if 'aws_access_key_id' in os.environ.keys():
+        print('After removing',os.environ['aws_access_key_id'])
+
+session_name = solara.reactive(None)
+session_list = solara.reactive([])
+
+def refresh_session_list():
+    session_list.set(sorted(storage.value.list_sessions(),reverse=True))
+
+@solara.component
+def StorageViewer():
+    with solara.Card(title='Load Session', subtitle='Choose a session from storage'):
+            solara.Select(label='Choose session',value=session_name.value, values=session_list.value,
+                        on_value=session_name.set)
+            solara.Button(label="Refresh", on_click=lambda: refresh_session_list())
+
+
 @solara.component
-def Page():
-    assert user.value is not None
-    solara.Markdown(f"Hi {user.value.username}, you are an admin")
+def Page(name: Optional[str] = None, page: int = 0, page_size=100):
+    aws_access_key_id = solara.use_reactive('')
+    aws_secret_access_key = solara.use_reactive('')
+    region_name = solara.use_reactive('eu-west-3')
+    bucket_name = solara.use_reactive('tcdse')
+    err_message, set_err_message = solara.use_state('')
+
+    def connect_storage(aws_access_key_id,
+                        aws_secret_access_key, 
+                        region_name,
+                        bucket_name):
+        print('connecting to s3')
+        try:
+            s3 = S3Storage(aws_access_key_id, aws_secret_access_key, region_name, bucket_name)
+            if s3.is_alive():
+                storage.value = s3
+                # Save for later to revive storage
+                os.environ['aws_access_key_id'] = aws_access_key_id
+                os.environ['aws_secret_access_key'] = aws_secret_access_key
+                os.environ['region_name'] = region_name
+                os.environ['bucket_name'] = bucket_name  
+            set_err_message('')
+        except Exception as e:
+            set_err_message(str(e))
+            print(e)
+
+    if storage.value is None:
+        s3_object = revive_storage()
+        if s3_object is not None and s3_object.is_alive():
+            storage.set(s3_object)
+            StorageViewer()
+        else:
+            with solara.Card(title='Attaching AWS S3',subtitle='Please attach an S3 bucket to save workspace sessions'):
+                solara.InputText(label='AWS Access Key ID', value=aws_access_key_id, on_value=aws_access_key_id.set)
+                solara.InputText(label='AWS Secret Access Key', value=aws_secret_access_key,password=True, on_value=aws_secret_access_key.set)
+                solara.InputText(label='AWS Region Name', value=region_name,on_value=region_name.set)
+                solara.InputText(label='Bucket Name', value=bucket_name, on_value=bucket_name.set)
+                solara.Button(label="Connect to S3", 
+                            on_click=lambda: connect_storage(aws_access_key_id.value, 
+                                                            aws_secret_access_key.value, 
+                                                            region_name.value,
+                                                            bucket_name.value))
+    else:
+        StorageViewer()
+
+    if err_message != '':
+        solara.Error(err_message)