update scripts with path

utils/add_missing.py

@@ -1,9 +1,12 @@
+from pathlib import Path
+
 import pandas as pd
-import os
 
-MISSING_ENRICHED = os.path.join("data", "missing_enriched.csv")
-CITIES_ENRICHED = os.path.join("data", "cities_enriched_final.csv")
-CITIES_ENRICHED_MANUALLY = os.path.join("data", "cities_enriched_manually.csv")
+MISSING_ENRICHED = Path(__file__).parent.parent / "data" / "missing_enriched.csv"
+CITIES_ENRICHED = Path(__file__).parent.parent / "data" / "cities_enriched_final.csv"
+CITIES_ENRICHED_MANUALLY = (
+    Path(__file__).parent.parent / "data" / "cities_enriched_manually.csv"
+)
 
 
 def merge_missing_coord(df, missing):

utils/add_missing_coordinates_app.py

@@ -1,11 +1,16 @@
-import pandas as pd
 import os
-import streamlit as st
+from pathlib import Path
 
+import pandas as pd
+import streamlit as st
 
-CITIES_ENRICHED_FINAL = os.path.join("data", "cities_enriched_final.csv")
-CITIES_ENRICHED_MANUALLY = os.path.join("data", "cities_enriched_manually.csv")
-MISSING_ENRICHED = os.path.join("data", "missing_enriched.csv")
+CITIES_ENRICHED_FINAL = (
+    Path(__file__).parent.parent / "data" / "cities_enriched_final.csv"
+)
+CITIES_ENRICHED_MANUALLY = (
+    Path(__file__).parent.parent / "data" / "cities_enriched_manually.csv"
+)
+MISSING_ENRICHED = Path(__file__).parent.parent / "data" / "missing_enriched.csv"
 
 
 def load_df(path: str) -> pd.DataFrame:
@@ -20,7 +25,7 @@ def get_missing(df: pd.DataFrame) -> pd.DataFrame:
 
 st.header("Impute missing coordinates")
 st.write(
-    f"""This is an application to manually add and save missing 
+    """This is an application to manually add and save missing 
          coordinates to cities.csv file for the Musterdatenkatalog.
          Missing coordinates are cities with empty or missing Geometry column.
          The reason for this can be that the city administation unit has changed and,

utils/compare_old_coord.py

@@ -1,6 +1,7 @@
-import os
-import pandas as pd
 import logging
+from pathlib import Path
+
+import pandas as pd
 
 # define logger
 logging.basicConfig(
@@ -13,10 +14,12 @@ logging.basicConfig(
 )
 
 
-CITIES_ENRICHED_OLD = os.path.join("data", "cities_enriched_old.csv")
-CITIES_ENRICHED_NEW = os.path.join("data", "cities_enriched.csv")
-CITIES_ENRICHED_FINAL = os.path.join("data", "cities_enriched_final.csv")
-MISSING = os.path.join("data", "missing_final.csv")
+CITIES_ENRICHED_OLD = Path(__file__).parent.parent / "data" / "cities_enriched_old.csv"
+CITIES_ENRICHED_NEW = Path(__file__).parent.parent / "data" / "cities_enriched.csv"
+CITIES_ENRICHED_FINAL = (
+    Path(__file__).parent.parent / "data" / "cities_enriched_final.csv"
+)
+MISSING = Path(__file__).parent.parent / "data" / "missing_final.csv"
 
 
 def load_data(path: str) -> pd.DataFrame:

utils/get_coordinates.py

@@ -1,11 +1,9 @@
-import pandas as pd
-import os
 import logging
-import numpy as np
-import ast
-import math
 from pathlib import Path
 
+import numpy as np
+import pandas as pd
+
 # define logger
 logging.basicConfig(
     level=logging.INFO,
@@ -17,18 +15,25 @@ logging.basicConfig(
 )
 
 
-CITIES_DATA = os.path.join("data", "raw", "2024_08_20_cities_1310_v5.csv")
-DATA_ENRICHED = os.path.join("data", "cities_enriched.csv")
+CITIES_DATA = (
+    Path(__file__).parent.parent / "data" / "raw" / "2025_06_12_cities_1474_v6.csv"
+)
+DATA_ENRICHED = Path(__file__).parent.parent / "data" / "cities_enriched.csv"
 
 # meta data for kreis codes ( variable in coordinates table)
-NAME_CODE_DATA = os.path.join("data", "raw", "name_kreiscode.csv")
-CODES_KOMMUNEN = os.path.join("data", "raw", "Deutschlandatlas.csv")
+NAME_CODE_DATA = Path(__file__).parent.parent / "data" / "raw" / "name_kreiscode.csv"
+CODES_KOMMUNEN = Path(__file__).parent.parent / "data" / "raw" / "Deutschlandatlas.csv"
+
 # coordinates for Gemeinden
-COORDINATES = os.path.join("data", "raw", "coordinates_plz_kreiscode.csv")
-MISSING = os.path.join("data", "missing_first_parser.csv")
+COORDINATES = (
+    Path(__file__).parent.parent / "data" / "raw" / "coordinates_plz_kreiscode.csv"
+)
+MISSING = Path(__file__).parent.parent / "data" / "missing_first_parser.csv"
+
 
-if not os.path.exists(os.path.join("data", "preprocessed")):
-    Path(os.path.join("data", "preprocessed")).mkdir(parents=True, exist_ok=True)
+preprocessed_path = Path(__file__).parent.parent / "data" / "preprocessed"
+if not preprocessed_path.exists():
+    preprocessed_path.mkdir(parents=True, exist_ok=True)
 
 
 def load_cities(path: str) -> pd.DataFrame:
@@ -60,7 +65,6 @@ def create_code_mapper(path: str) -> dict:
 def map_code(org_name, code_mapper):
     # Split the org_name string into parts
     parts = org_name.split()
-    # print(parts, type(parts[0]))
     # Find a key in code_mapper that contains all parts of the split org_name
     for key in code_mapper.keys():
         # look first for whole name (cases like "Landkreis München" , "kreisfreie Stadt München")
@@ -166,7 +170,7 @@ def merge_coordinates(df: pd.DataFrame, coordinates: pd.DataFrame) -> pd.DataFra
                     modified_row = row
         modified_rows.append(modified_row)
     df["Geometry"] = geometries
-    # print(modified_rows)
+
     modified = pd.DataFrame(modified_rows)
     modified["Geometry"] = geometries
     return modified
@@ -179,7 +183,7 @@ def aggregate_coordinates(geo_element: str) -> list:
     else:
         actual_list = geo_element  # ast.literal_eval(geo_element)
         processed_list = [list(map(float, coord.split(", "))) for coord in actual_list]
-        # print(processed_list)
+
         if len(processed_list) > 1:
             coordinates = np.mean(np.array(processed_list), axis=0)
         else:
@@ -195,16 +199,17 @@ if __name__ == "__main__":
     missing = data[data["Code"].isnull()]
     logging.info(f"Missing values Gebietscode: {len(missing)}")
     data.to_csv(
-        os.path.join("data", "preprocessed", "cities_enriched_with_code.csv"),
+        Path(__file__).parent.parent / "data" / "preprocessed" / "cities_enriched.csv",
         index=False,
     )
-    # data = pd.read_csv(
-    #    os.path.join("data", "preprocessed", "cities_enriched_with_code.csv"))
     data["Code"] = data["Code"].apply(lambda x: int(x) if pd.notna(x) else None)
     coordinates = load_coordinates(COORDINATES)
     data = merge_coordinates(data, coordinates)
     data.to_csv(
-        os.path.join("data", "preprocessed", "cities_enriched_with_coordinates.csv"),
+        Path(__file__).parent.parent
+        / "data"
+        / "preprocessed"
+        / "cities_enriched_with_coordinates.csv",
         index=False,
     )
     logging.info("Coordinates merged")
@@ -222,6 +227,5 @@ if __name__ == "__main__":
     logging.info(f"Missing geometry: {len(missing_geometry)}")
     missing_geometry.to_csv(MISSING, index=False)
 
-    # data = pd.read_csv(os.path.join("data", "cities_enriched_manually.csv"))
     data["Geometry"] = data["Geometry"].apply(aggregate_coordinates)
     data.to_csv(DATA_ENRICHED, index=False)

utils/process_data.py

@@ -1,7 +1,7 @@
-import os
 import logging
-import pandas as pd
+from pathlib import Path
 
+import pandas as pd
 
 # define logger
 logging.basicConfig(
@@ -13,11 +13,12 @@ logging.basicConfig(
     ],
 )
 
+
 # change these to paths if you want to generate the map_data.csv separately from the app
-DATA_RAW = os.path.join("2024-08-21_musterdatenkatalog.json")
-CITIES_ENRICHED = os.path.join("data", "cities_enriched_manually.csv")
+DATA_RAW = Path(__file__).parent.parent / "2025-06-13_musterdatenkatalog.json"
+CITIES_ENRICHED = Path(__file__).parent.parent / "data" / "cities_enriched_manually.csv"
 
-OUTPUT = os.path.join("data", "preprocessed", "map_data.csv")
+OUTPUT = Path(__file__).parent.parent / "data" / "preprocessed" / "map_data.csv"
 
 
 def load_data(path: str = DATA_RAW) -> pd.DataFrame: