feat: add preprocessing. #3

.gitignore

@@ -0,0 +1,57 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+.Python
+*.egg-info/
+dist/
+build/
+*.egg
+
+# Entornos virtuales
+.env
+.venv
+env/
+venv/
+ENV/
+
+# Jupyter
+.ipynb_checkpoints/
+*.ipynb_checkpoints
+
+# Datos (no subir datos al repo)
+data/raw/*
+data/processed/*
+!data/raw/.gitkeep
+!data/processed/.gitkeep
+
+# Modelos entrenados
+models/*
+!models/.gitkeep
+
+# Logs
+logs/*
+!logs/.gitkeep
+
+# Variables de entorno
+.env
+*.env
+secrets.yaml
+
+# IDEs
+.vscode/settings.json
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# MLflow
+mlruns/
+mlartifacts/
+
+#jony
+02_preprocessing_v2.ipynb

configs/features.yaml

@@ -12,11 +12,11 @@ preprocessing:
 vectorization:
   method: tfidf  # tfidf | bow | both
   tfidf:
-    max_features: 10000
+    max_features: 5000
     ngram_range: [1, 2]
     sublinear_tf: true
-    min_df: 2
+    min_df: 3
   bow:
-    max_features: 10000
+    max_features: 5000
     ngram_range: [1, 1]
-    min_df: 2
+    min_df: 3

configs/pipeline.yaml

@@ -0,0 +1,19 @@
+pipeline:
+  random_state: 42
+  test_size: 0.2
+  cv_folds: 5
+
+data:
+  raw_path: data/raw/youtoxic_english_1000.csv
+  processed_path: data/processed/v1/comments_with_stats.csv
+  target_binary: IsToxic
+  target_multilabel:
+    - IsAbusive
+    - IsProvocative
+    - IsHatespeech
+    - IsRacist
+    - IsObscene
+  text_column: Text
+  id_column: CommentId
+
+mode: binary  # binary | multilabel