Spaces:
Sleeping
Sleeping
File size: 2,156 Bytes
61b4af1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | # scripts/build_training_data.py
import pandas as pd
from pathlib import Path
RAW_DATA_PATH = Path("merged_crop_data_with_weather.csv")
OUT_PATH = Path("data/processed/training_data.parquet")
def add_lag_and_target(df: pd.DataFrame) -> pd.DataFrame:
"""
For each (Mandi, Commodity) group:
- add price lags (1,2,3,7 days)
- add target = next day's ModalPrice
"""
df = df.sort_values(["Mandi", "Commodity", "date"]).copy()
def process_group(g: pd.DataFrame) -> pd.DataFrame:
g = g.sort_values("date").copy()
for lag in [1, 2, 3, 7]:
g[f"lag_{lag}"] = g["ModalPrice"].shift(lag)
# target is next day's price
g["target_price"] = g["ModalPrice"].shift(-1)
return g
df = df.groupby(["Mandi", "Commodity"], group_keys=False).apply(process_group)
# remove rows where target or important lags are NaN (start/end of series)
lag_cols = [f"lag_{l}" for l in [1, 2, 3, 7]]
df = df.dropna(subset=lag_cols + ["target_price"])
return df
def add_time_features(df: pd.DataFrame) -> pd.DataFrame:
"""Add calendar features like day of week, month."""
df["dayofweek"] = df["date"].dt.dayofweek # 0=Mon, 6=Sun
df["month"] = df["date"].dt.month
return df
def main():
RAW_DATA_PATH.parent.mkdir(parents=True, exist_ok=True)
print(f"Loading raw data from {RAW_DATA_PATH} ...")
df = pd.read_csv(RAW_DATA_PATH)
# parse dates
df["date"] = pd.to_datetime(df["date"])
# keep only columns we need for now
needed_cols = ["date", "Mandi", "Commodity", "ModalPrice"]
df = df[needed_cols].copy()
print("Adding lag and target columns...")
df = add_lag_and_target(df)
print("Adding time features...")
df = add_time_features(df)
# treat Mandi & Commodity as categories (LightGBM can handle this)
df["Mandi"] = df["Mandi"].astype("category")
df["Commodity"] = df["Commodity"].astype("category")
OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
df.to_parquet(OUT_PATH, index=False)
print(f"Saved training data with shape {df.shape} to {OUT_PATH}")
if __name__ == "__main__":
main()
|