Update main.py
Browse files
main.py
CHANGED
|
@@ -37,31 +37,21 @@ storage_options={'account_name': AZURE_STORAGE_ACCOUNT, 'account_key': AZURE_STO
|
|
| 37 |
|
| 38 |
df = pl.read_parquet(filepath,storage_options=storage_options)
|
| 39 |
|
| 40 |
-
#
|
| 41 |
-
|
| 42 |
-
# "aws_access_key_id": ,
|
| 43 |
-
# "aws_secret_access_key": os.environ['AZURE_STORAGE_ACCESS_KEY'],
|
| 44 |
-
#}
|
| 45 |
-
#df = pl.scan_parquet(source, storage_options=storage_options)
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
#config_path = "./azure/data/config.yaml"
|
| 49 |
|
| 50 |
# Add the read-in data from the yaml file
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
|
| 56 |
-
|
| 57 |
#path_parquet = config.get("path_parquet")
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
# Import the data from one .parquet file
|
| 64 |
-
#df = pl.read_parquet(path_parquet)
|
| 65 |
#df = df.rename({"__index_level_0__": "Unnamed: 0"})
|
| 66 |
|
| 67 |
# Setup the app
|
|
|
|
| 37 |
|
| 38 |
df = pl.read_parquet(filepath,storage_options=storage_options)
|
| 39 |
|
| 40 |
+
# Load in config file
|
| 41 |
+
config_path = "./data/config.yaml"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
|
| 43 |
# Add the read-in data from the yaml file
|
| 44 |
+
def read_config(filename):
|
| 45 |
+
with open(filename, 'r') as yaml_file:
|
| 46 |
+
config = yaml.safe_load(yaml_file)
|
| 47 |
+
return config
|
| 48 |
|
| 49 |
+
config = read_config(config_path)
|
| 50 |
#path_parquet = config.get("path_parquet")
|
| 51 |
+
conditions = config.get("conditions")
|
| 52 |
+
col_features = config.get("col_features")
|
| 53 |
+
col_counts = config.get("col_counts")
|
| 54 |
+
col_mt = config.get("col_mt")
|
|
|
|
|
|
|
|
|
|
| 55 |
#df = df.rename({"__index_level_0__": "Unnamed: 0"})
|
| 56 |
|
| 57 |
# Setup the app
|