Spaces:
Sleeping
Sleeping
File size: 3,472 Bytes
c8a8b27 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | """Run the full offline pipeline: raw CSV -> small artifacts the app serves.
python -m src.build_artifacts # run from the repo root
Outputs:
data/processed/hotspots.parquet per-cell stats + CII
data/processed/forecast.parquet next-day predicted intensity per cell
data/processed/meta.json config, date range, model metrics
models/lgbm_intensity.txt saved LightGBM model
"""
import json
from src import config
from src.data_prep import load_clean
from src.features import add_h3
from src.hotspots import build_cell_stats
from src.impact_index import add_cii
from src.model import train_and_forecast
from src.offenders import build_offender_stats, offender_summary
from src.trends import build_trends, build_byday
def main():
config.DATA_PROCESSED.mkdir(parents=True, exist_ok=True)
config.MODELS_DIR.mkdir(parents=True, exist_ok=True)
print("[1/4] Loading & cleaning ...")
df = load_clean()
print(f" clean parking records: {len(df):,}")
df = add_h3(df)
total_days = (df["ts"].dt.date.max() - df["ts"].dt.date.min()).days + 1
print("[2/4] Building hotspot stats & CII ...")
stats = add_cii(build_cell_stats(df, total_days))
stats.to_parquet(config.HOTSPOTS_PARQUET, index=False)
print(f" {len(stats):,} cells -> {config.HOTSPOTS_PARQUET.name}")
print("[2b] Repeat-offender intelligence ...")
offenders = build_offender_stats(df, top_n=500)
offenders.to_parquet(config.OFFENDERS_PARQUET, index=False)
off_summary = offender_summary(df)
print(f" {len(offenders):,} offenders -> {config.OFFENDERS_PARQUET.name} | {off_summary}")
print("[2c] Trend aggregates ...")
build_trends(df).to_parquet(config.TRENDS_PARQUET, index=False)
build_byday(df).to_parquet(config.TRENDS_BYDAY_PARQUET, index=False)
print(f" -> {config.TRENDS_PARQUET.name}, {config.TRENDS_BYDAY_PARQUET.name}")
print("[3/4] Training LightGBM & forecasting ...")
model, forecast, metrics = train_and_forecast(df)
model.save_model(str(config.MODEL_PATH))
forecast.to_parquet(config.FORECAST_PARQUET, index=False)
print(f" {len(forecast):,} cells -> {config.FORECAST_PARQUET.name}")
print(f" metrics: {metrics}")
print("[4/4] Writing meta.json ...")
# small real daily series for the KPI sparklines
dord = sorted(df["date"].unique())
gd = df.groupby("date")
kpi_sparks = dict(
violations=gd["id"].size().reindex(dord, fill_value=0).astype(int).tolist(),
zones=gd["h3"].nunique().reindex(dord, fill_value=0).astype(int).tolist(),
peak=(df[df["is_peak"]].groupby("date")["id"].size()
.reindex(dord, fill_value=0).astype(int).tolist()),
)
meta = dict(
date_range=[df["ts"].dt.date.min().isoformat(),
df["ts"].dt.date.max().isoformat()],
total_days=total_days,
n_records=int(len(df)),
n_cells=int(len(stats)),
h3_res=config.H3_RES,
severity_weights=config.PARKING_SEVERITY,
cii_weights=config.CII_WEIGHTS,
cii_junction_alpha=config.CII_JUNCTION_ALPHA,
offender_summary=off_summary,
model_metrics=metrics,
kpi_sparks=kpi_sparks,
)
config.META_JSON.write_text(json.dumps(meta, indent=2))
print(f" -> {config.META_JSON.name}")
print("Done. You can now run: streamlit run app.py")
if __name__ == "__main__":
main()
|