Spaces:
Sleeping
Sleeping
| """Run the full offline pipeline: raw CSV -> small artifacts the app serves. | |
| python -m src.build_artifacts # run from the repo root | |
| Outputs: | |
| data/processed/hotspots.parquet per-cell stats + CII | |
| data/processed/forecast.parquet next-day predicted intensity per cell | |
| data/processed/meta.json config, date range, model metrics | |
| models/lgbm_intensity.txt saved LightGBM model | |
| """ | |
| import json | |
| from src import config | |
| from src.data_prep import load_clean | |
| from src.features import add_h3 | |
| from src.hotspots import build_cell_stats | |
| from src.impact_index import add_cii | |
| from src.model import train_and_forecast | |
| from src.offenders import build_offender_stats, offender_summary | |
| from src.trends import build_trends, build_byday | |
| def main(): | |
| config.DATA_PROCESSED.mkdir(parents=True, exist_ok=True) | |
| config.MODELS_DIR.mkdir(parents=True, exist_ok=True) | |
| print("[1/4] Loading & cleaning ...") | |
| df = load_clean() | |
| print(f" clean parking records: {len(df):,}") | |
| df = add_h3(df) | |
| total_days = (df["ts"].dt.date.max() - df["ts"].dt.date.min()).days + 1 | |
| print("[2/4] Building hotspot stats & CII ...") | |
| stats = add_cii(build_cell_stats(df, total_days)) | |
| stats.to_parquet(config.HOTSPOTS_PARQUET, index=False) | |
| print(f" {len(stats):,} cells -> {config.HOTSPOTS_PARQUET.name}") | |
| print("[2b] Repeat-offender intelligence ...") | |
| offenders = build_offender_stats(df, top_n=500) | |
| offenders.to_parquet(config.OFFENDERS_PARQUET, index=False) | |
| off_summary = offender_summary(df) | |
| print(f" {len(offenders):,} offenders -> {config.OFFENDERS_PARQUET.name} | {off_summary}") | |
| print("[2c] Trend aggregates ...") | |
| build_trends(df).to_parquet(config.TRENDS_PARQUET, index=False) | |
| build_byday(df).to_parquet(config.TRENDS_BYDAY_PARQUET, index=False) | |
| print(f" -> {config.TRENDS_PARQUET.name}, {config.TRENDS_BYDAY_PARQUET.name}") | |
| print("[3/4] Training LightGBM & forecasting ...") | |
| model, forecast, metrics = train_and_forecast(df) | |
| model.save_model(str(config.MODEL_PATH)) | |
| forecast.to_parquet(config.FORECAST_PARQUET, index=False) | |
| print(f" {len(forecast):,} cells -> {config.FORECAST_PARQUET.name}") | |
| print(f" metrics: {metrics}") | |
| print("[4/4] Writing meta.json ...") | |
| # small real daily series for the KPI sparklines | |
| dord = sorted(df["date"].unique()) | |
| gd = df.groupby("date") | |
| kpi_sparks = dict( | |
| violations=gd["id"].size().reindex(dord, fill_value=0).astype(int).tolist(), | |
| zones=gd["h3"].nunique().reindex(dord, fill_value=0).astype(int).tolist(), | |
| peak=(df[df["is_peak"]].groupby("date")["id"].size() | |
| .reindex(dord, fill_value=0).astype(int).tolist()), | |
| ) | |
| meta = dict( | |
| date_range=[df["ts"].dt.date.min().isoformat(), | |
| df["ts"].dt.date.max().isoformat()], | |
| total_days=total_days, | |
| n_records=int(len(df)), | |
| n_cells=int(len(stats)), | |
| h3_res=config.H3_RES, | |
| severity_weights=config.PARKING_SEVERITY, | |
| cii_weights=config.CII_WEIGHTS, | |
| cii_junction_alpha=config.CII_JUNCTION_ALPHA, | |
| offender_summary=off_summary, | |
| model_metrics=metrics, | |
| kpi_sparks=kpi_sparks, | |
| ) | |
| config.META_JSON.write_text(json.dumps(meta, indent=2)) | |
| print(f" -> {config.META_JSON.name}") | |
| print("Done. You can now run: streamlit run app.py") | |
| if __name__ == "__main__": | |
| main() | |