Update main.py
Browse files
main.py
CHANGED
|
@@ -24,23 +24,40 @@ config_fig = {
|
|
| 24 |
}
|
| 25 |
}
|
| 26 |
|
| 27 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
|
| 29 |
# Add the read-in data from the yaml file
|
| 30 |
-
def read_config(filename):
|
| 31 |
-
with open(filename, 'r') as yaml_file:
|
| 32 |
-
config = yaml.safe_load(yaml_file)
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
config = read_config(config_path)
|
| 36 |
-
path_parquet = config.get("path_parquet")
|
| 37 |
-
conditions = config.get("conditions")
|
| 38 |
-
col_features = config.get("col_features")
|
| 39 |
-
col_counts = config.get("col_counts")
|
| 40 |
-
col_mt = config.get("col_mt")
|
| 41 |
|
| 42 |
# Import the data from one .parquet file
|
| 43 |
-
df = pl.read_parquet(path_parquet)
|
| 44 |
#df = df.rename({"__index_level_0__": "Unnamed: 0"})
|
| 45 |
|
| 46 |
# Setup the app
|
|
|
|
| 24 |
}
|
| 25 |
}
|
| 26 |
|
| 27 |
+
from adlfs import AzureBlobFileSystem
|
| 28 |
+
mountpount=os.environ['AZURE_MOUNT_POINT'],
|
| 29 |
+
accountkey=os.environ['AZURE_STORAGE_ACCESS_KEY'],
|
| 30 |
+
accountname=os.environ['AZURE_STORAGE_ACCOUNT'],
|
| 31 |
+
|
| 32 |
+
storage_options={'account_name': ACCOUNT_NAME, 'anon': False}
|
| 33 |
+
df = pl.read_parquet(path="liu/filename.parquet",storage_options=storage_options)
|
| 34 |
+
#abfs = AzureBlobFileSystem(account_name=accountname,account_key=accountkey)
|
| 35 |
+
|
| 36 |
+
#pq.write_table(polars_dataframe.to_arrow(), 'liu/file_name.parquet', filesystem=abfs)
|
| 37 |
+
#storage_options = {
|
| 38 |
+
# "aws_access_key_id": ,
|
| 39 |
+
# "aws_secret_access_key": os.environ['AZURE_STORAGE_ACCESS_KEY'],
|
| 40 |
+
#}
|
| 41 |
+
#df = pl.scan_parquet(source, storage_options=storage_options)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
#config_path = "./azure/data/config.yaml"
|
| 45 |
|
| 46 |
# Add the read-in data from the yaml file
|
| 47 |
+
#def read_config(filename):
|
| 48 |
+
# with open(filename, 'r') as yaml_file:
|
| 49 |
+
# config = yaml.safe_load(yaml_file)
|
| 50 |
+
# return config
|
| 51 |
+
|
| 52 |
+
#config = read_config(config_path)
|
| 53 |
+
#path_parquet = config.get("path_parquet")
|
| 54 |
+
#conditions = config.get("conditions")
|
| 55 |
+
#col_features = config.get("col_features")
|
| 56 |
+
#col_counts = config.get("col_counts")
|
| 57 |
+
#col_mt = config.get("col_mt")
|
| 58 |
|
| 59 |
# Import the data from one .parquet file
|
| 60 |
+
#df = pl.read_parquet(path_parquet)
|
| 61 |
#df = df.rename({"__index_level_0__": "Unnamed: 0"})
|
| 62 |
|
| 63 |
# Setup the app
|