Create feature_engineering.py

src/feature_engineering.py

@@ -0,0 +1,19 @@
+# src/feature_engineering.py
+from transformers import DistilBertTokenizer
+import torch
+from src.config import MAX_LENGTH
+import logging
+
+def setup_logging():
+    logging.basicConfig(filename="logs/app.log", level=logging.INFO, 
+                        format="%(asctime)s - %(levelname)s - %(message)s")
+
+def tokenize_texts(texts):
+    """Tokenize texts using DistilBERT tokenizer."""
+    setup_logging()
+    tokenizer = DistilBertTokenizer.from_pretrained("distilbert-base-uncased")
+    logging.info("Tokenizing texts")
+    encodings = tokenizer(
+        texts.tolist(), truncation=True, padding=True, max_length=MAX_LENGTH, return_tensors="pt"
+    )
+    return encodings