File size: 847 Bytes
8f1213e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | """Script 02: Preprocess raw data (clean + inner join review with meta + stratified split)."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from src.utils import setup_logging
from src.preprocess import preprocess_pipeline, stratified_split
from src import config as cfg
def main():
setup_logging()
processed = preprocess_pipeline()
train, val, test = stratified_split(processed)
train.to_parquet(cfg.TRAIN_PATH, index=False)
val.to_parquet(cfg.VAL_PATH, index=False)
test.to_parquet(cfg.TEST_PATH, index=False)
print(f"Saved splits:")
print(f" train -> {cfg.TRAIN_PATH} ({len(train)} rows)")
print(f" val -> {cfg.VAL_PATH} ({len(val)} rows)")
print(f" test -> {cfg.TEST_PATH} ({len(test)} rows)")
if __name__ == "__main__":
main()
|