tarted working on database

docker-compose.yaml

@@ -0,0 +1,36 @@
+version: '3.8'
+services:
+  db:
+    image: postgres:16-alpine
+    container_name: sd_postgres
+    restart: unless-stopped
+    ports:
+      - "${DB_PORT:-5432}:5432"
+    environment:
+      POSTGRES_USER: hjun
+      POSTGRES_PASSWORD: alsdfjwpoejfkd
+      POSTGRES_DB: sd_roster_real
+    volumes:
+      - db_data:/var/lib/postgresql/data
+      - ./docker/init:/docker-entrypoint-initdb.d:ro   # 초기 스키마/권한 스크립트
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
+      interval: 5s
+      timeout: 3s
+      retries: 20
+
+  pgadmin:
+    image: dpage/pgadmin4
+    container_name: sd_pgadmin
+    restart: unless-stopped
+    environment:
+      PGADMIN_DEFAULT_EMAIL: hjun@unicef.org
+      PGADMIN_DEFAULT_PASSWORD: alsdfjwpoejfkd
+    ports:
+      - "${PGADMIN_PORT:-5050}:80"
+    depends_on:
+      db:
+        condition: service_healthy
+
+volumes:
+  db_data:

docker/init/001_init.sql

@@ -0,0 +1,16 @@
+CREATE SCHEMA IF NOT EXISTS stg;
+CREATE SCHEMA IF NOT EXISTS dim;
+CREATE SCHEMA IF NOT EXISTS fact;
+CREATE SCHEMA IF NOT EXISTS rej;
+CREATE SCHEMA IF NOT EXISTS meta;
+
+CREATE TABLE IF NOT EXISTS meta.batch_log (
+    batch_id       BIGSERIAL PRIMARY KEY,
+    source_file    TEXT NOT NULL,
+    source_hash    TEXT,
+    rows_read      INT,
+    rows_loaded    INT,
+    rows_rejected  INT,
+    started_at     TIMESTAMPTZ DEFAULT NOW(),
+    finished_at    TIMESTAMPTZ
+);

pyproject.toml

@@ -11,13 +11,6 @@ requires-python = ">=3.10,<3.11"
 dependencies = [
     "pandas>=2.1.3",
     "or-tools>=10.0.0",
-    "numpy>=1.26.4",
-    "matplotlib>=3.8.0",
-    "seaborn>=0.13.2",
-    "scipy>=1.13.0",
-    "scikit-learn>=1.3.2",
-    "statsmodels>=0.14.4",
-    "plotly>=5.19.0",
 ]
 
 

requirements.txt

@@ -254,3 +254,10 @@ xyzservices==2025.1.0
 yarl==1.20.0
 zipp==3.21.0
 zstandard==0.23.0
+
+# Database and ETL dependencies
+sqlalchemy==2.0.36
+psycopg2-binary==2.9.9
+python-dotenv==1.0.0
+pydantic==2.10.6
+alembic==1.14.0

src/config/optimization_config.py

@@ -1,8 +1,17 @@
-DATE_SPAN = list(range(1, 8))
+import pandas as pd
+import etl.transform as transformed_data
+
+
+DATE_SPAN = list(range(1, 8))  # Default 7 days - other than this - user input
 PRODUCT_LIST = ["a", "b", "c"]  # COOIS_Planned_and_Released.csv
-EMPLOYEE_TYPE_LIST = ["Fixed", "Humanizer"]  # WH_Workforce_Hourly_payment_scale.csv
-SHIFT_LIST = [1, 2, 3]  # WH_Workforce_Hourly_Pay_Scale.csv
-LINE_LIST = ["long", "short"]  # WH_Workforce_Hourly_Pay_Scale.csv
+PRODUCT_LIST = transformed_data.get_product_list()
+EMPLOYEE_TYPE_LIST = ["Fixed", "Humanizer"]  # WH_Workforce_Hourly_payment_scale.csv -> This is default setting. User should be able to manipulate
+EMPLOYEE_TYPE_LIST = transformed_data.get_employee_type_list()
+
+# WH_Workforce_Hourly_payment_scale.csv -> This is default setting. User should be able to manipulate
+SHIFT_LIST = [1, 2, 3]  # WH_Workforce_Hourly_Pay_Scale.csv -> This is default setting. User should be able to manipulate
+LINE_LIST = ["long", "short"]  # WH_Workforce_Hourly_Pay_Scale.csv -> This is default setting. User should be able to manipulate
+
 LINE_LIST_PER_TYPE = {
     "long": 2,
     "short": 3,

src/etl/db.py

@@ -0,0 +1,17 @@
+from sqlalchemy import create_engine
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+USER = os.getenv("POSTGRES_USER", "myuser")
+PWD = os.getenv("POSTGRES_PASSWORD", "mypass")
+DB = os.getenv("POSTGRES_DB", "mydb")
+PORT = os.getenv("DB_PORT", "5432")
+HOST = "localhost"
+
+engine = create_engine(
+    f"postgresql+psycopg2://{USER}:{PWD}@{HOST}:{PORT}/{DB}", future=True
+)
+if __name__ == "__main__":
+    with engine.begin() as conn:
+        print(conn.execute("select version();").scalar())

src/etl/extract.py

@@ -0,0 +1,58 @@
+import pandas as pd
+import datetime
+from datetime import date, timedelta
+
+START_DATE = pd.Timestamp(2025, 7, 7)
+END_DATE = pd.Timestamp(2025, 7, 11)
+
+
+def read_excel(path: str) -> pd.DataFrame:
+    return pd.read_excel(path, dtype={"id": "Int64"})
+
+
+def read_demand_data(
+    path="data/real_data_excel/converted_csv/COOIS_Planned_and_Released.csv",
+) -> pd.DataFrame:
+    df = pd.read_csv(path)
+    df["Basic start date"] = pd.to_datetime(df["Basic start date"])
+    df["Basic finish date"] = pd.to_datetime(df["Basic finish date"])
+    df = df[
+        (df["Basic start date"] >= START_DATE) & (df["Basic finish date"] <= END_DATE)
+    ]
+
+    return df
+
+
+def read_employee_data(
+    path="data/real_data_excel/converted_csv/WH_Workforce_Hourly_Pay_Scale.csv",
+) -> pd.DataFrame:
+    return pd.read_csv(path)
+
+
+def read_shift_data(
+    path="data/real_data_excel/converted_csv/Work_Shifts_timing.csv",
+) -> pd.DataFrame:
+    return pd.read_csv(path)
+
+
+def read_kit_data(
+    path="data/real_data_excel/converted_csv/Kits__Calculation.csv",
+) -> pd.DataFrame:
+    return pd.read_csv(path)
+
+
+def read_work_center_capacity(
+    path="data/real_data_excel/converted_csv/Work_Centre_Capacity.csv",
+) -> pd.DataFrame:
+    return pd.read_csv(path)
+
+
+def read_material_master(
+    path="data/real_data_excel/converted_csv/Material_Master_WMS.csv",
+) -> pd.DataFrame:
+    return pd.read_csv(path)
+
+
+if __name__ == "__main__":
+    demand_data = read_demand_data()
+    print(demand_data.head())

src/etl/models.py

@@ -0,0 +1 @@
+

src/etl/transform.py

@@ -0,0 +1,20 @@
+import pandas as pd
+import extract as ex
+
+
+def get_product_list():
+    demand = ex.read_demand_data()
+    print(demand["Material Number"].unique())
+    return demand["Material Number"].unique()
+
+
+def get_employee_list():
+    employee = ex.read_employee_data()
+    employee = employee["Description"]
+
+    return employee["Employee_Type"].unique()
+
+
+if __name__ == "__main__":
+    get_product_list()
+    # get_employee_list()

src/project

@@ -0,0 +1 @@
+Subproject commit 2e1b97c9d8196552a23dd5a4c536f25e53c033dc