Convert to DuckDB for efficient data storage and access

- Converted 389MB CSV files to 100MB DuckDB (74% compression)
- Added DuckDB dependency to pyproject.toml
- Modified EDA pipeline to load data directly from DuckDB
- Updated .gitignore to track DuckDB file
- Faster loading: DuckDB vs CSV parsing overhead
- Git-friendly: Single binary file under 100MB limit
- Better performance for RL parameter extraction

.gitignore

@@ -29,3 +29,4 @@ Data/test_verification/
 # Keep essential data
 !Data/README.md
 !pyproject.toml
+!Data/court_data.duckdb

pyproject.toml

@@ -20,7 +20,8 @@ dependencies = [
     "scipy>=1.14",
     "scikit-learn>=1.5",
     "streamlit>=1.28",
-    "altair>=5.0"
+    "altair>=5.0",
+    "duckdb>=1.4.2",
 ]
 
 [project.optional-dependencies]

src/eda_config.py

@@ -9,8 +9,7 @@ from pathlib import Path
 # Paths and versioning
 # -------------------------------------------------------------------
 DATA_DIR = Path("Data")
-CASES_FILE = DATA_DIR / "ISDMHack_Cases_WPfinal.csv"
-HEAR_FILE = DATA_DIR / "ISDMHack_Hear.csv"
+DUCKDB_FILE = DATA_DIR / "court_data.duckdb"
 
 REPORTS_DIR = Path("reports")
 FIGURES_DIR = REPORTS_DIR / "figures"

src/eda_load_clean.py

@@ -11,10 +11,10 @@ Responsibilities:
 from datetime import timedelta
 
 import polars as pl
+import duckdb
 from src.eda_config import (
     CASES_CLEAN_PARQUET,
-    CASES_FILE,
-    HEAR_FILE,
+    DUCKDB_FILE,
     HEARINGS_CLEAN_PARQUET,
     NULL_TOKENS,
     RUN_TS,
@@ -56,19 +56,22 @@ def _null_summary(df: pl.DataFrame, name: str) -> None:
 # Main logic
 # -------------------------------------------------------------------
 def load_raw() -> tuple[pl.DataFrame, pl.DataFrame]:
-    print("Loading raw data with Polars...")
-    cases = pl.read_csv(
-        CASES_FILE,
-        try_parse_dates=True,
-        null_values=NULL_TOKENS,
-        infer_schema_length=100_000,
-    )
-    hearings = pl.read_csv(
-        HEAR_FILE,
-        try_parse_dates=True,
-        null_values=NULL_TOKENS,
-        infer_schema_length=100_000,
-    )
+    print(f"Loading raw data from DuckDB: {DUCKDB_FILE}")
+    
+    if not DUCKDB_FILE.exists():
+        raise FileNotFoundError(f"DuckDB file not found: {DUCKDB_FILE}")
+    
+    # Connect to DuckDB and load data
+    conn = duckdb.connect(str(DUCKDB_FILE))
+    
+    # Load cases as Polars DataFrame
+    cases = pl.from_pandas(conn.execute("SELECT * FROM cases").df())
+    
+    # Load hearings as Polars DataFrame
+    hearings = pl.from_pandas(conn.execute("SELECT * FROM hearings").df())
+    
+    conn.close()
+    
     print(f"Cases shape: {cases.shape}")
     print(f"Hearings shape: {hearings.shape}")
     return cases, hearings