scripts/02_preprocess.py

"""Script 02: Preprocess raw data (clean + inner join review with meta + stratified split)."""
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from src.utils import setup_logging
from src.preprocess import preprocess_pipeline, stratified_split
from src import config as cfg


def main():
    setup_logging()
    processed = preprocess_pipeline()

    train, val, test = stratified_split(processed)
    train.to_parquet(cfg.TRAIN_PATH, index=False)
    val.to_parquet(cfg.VAL_PATH, index=False)
    test.to_parquet(cfg.TEST_PATH, index=False)
    print(f"Saved splits:")
    print(f"  train -> {cfg.TRAIN_PATH}  ({len(train)} rows)")
    print(f"  val   -> {cfg.VAL_PATH}  ({len(val)} rows)")
    print(f"  test  -> {cfg.TEST_PATH}  ({len(test)} rows)")


if __name__ == "__main__":
    main()