feat: Phase 2 - preprocessing and EDA pipeline

data/.gitignore

@@ -1,2 +1 @@
 /raw
-/processed

data/processed.dvc

@@ -1,6 +0,0 @@
-outs:
-- md5: d751713988987e9331980363e24189ce.dir
-  size: 0
-  nfiles: 0
-  hash: md5
-  path: processed

data/raw.dvc

@@ -1,6 +1,6 @@
 outs:
-- md5: 0219de7f981995ce70939659481c9757.dir
-  size: 5750191
-  nfiles: 6
+- md5: 0d10b4997c93bd1bce4a9d53a6e4748e.dir
+  size: 7943297
+  nfiles: 21
   hash: md5
   path: raw

dvc.lock

@@ -0,0 +1,68 @@
+schema: '2.0'
+stages:
+  preprocess_semeval:
+    cmd: PYTHONPATH=. python src/data/dataset.py
+    deps:
+    - path: data/raw/semeval_laptops
+      hash: md5
+      md5: 834bf3cc7b3d9216331adae54c9239c6.dir
+      size: 1104115
+      nfiles: 9
+    - path: data/raw/semeval_restaurants
+      hash: md5
+      md5: 07f540b34a0561f20908aaa70012defb.dir
+      size: 1625908
+      nfiles: 9
+    - path: src/data/dataset.py
+      hash: md5
+      md5: 4199a5c7d112040180779168f7da1a4a
+      size: 2761
+    - path: src/data/lang_detect.py
+      hash: md5
+      md5: ca10825874c1eeb68e199e0e178d8f94
+      size: 1053
+    - path: src/data/preprocess.py
+      hash: md5
+      md5: c01401432dfeb6e32e928d47570caa21
+      size: 855
+    - path: src/data/transliterate.py
+      hash: md5
+      md5: 2ab5e443eb6123188986e975586f42b8
+      size: 1404
+    outs:
+    - path: data/processed/semeval_test.jsonl
+      hash: md5
+      md5: 2c4581091b4aeb29a0734e44fd3a5656
+      size: 263576
+    - path: data/processed/semeval_train.jsonl
+      hash: md5
+      md5: 04b5486dc6a7c62d1289804bd373eeaa
+      size: 971123
+  preprocess_hindi:
+    cmd: PYTHONPATH=. python src/data/hindi_loader.py
+    deps:
+    - path: data/raw/amazon_hindi/hindi_sentiment.jsonl
+      hash: md5
+      md5: 152317207b7015c46f6260a93c2cd756
+      size: 1140222
+    - path: src/data/hindi_loader.py
+      hash: md5
+      md5: 6ffae4a0c61907d396b4e693beb6a23a
+      size: 1826
+    - path: src/data/lang_detect.py
+      hash: md5
+      md5: ca10825874c1eeb68e199e0e178d8f94
+      size: 1053
+    - path: src/data/preprocess.py
+      hash: md5
+      md5: c01401432dfeb6e32e928d47570caa21
+      size: 855
+    - path: src/data/transliterate.py
+      hash: md5
+      md5: 2ab5e443eb6123188986e975586f42b8
+      size: 1404
+    outs:
+    - path: data/processed/amazon_hindi.jsonl
+      hash: md5
+      md5: 29825aafaee80993c9fcb89669bb27ce
+      size: 304642

dvc.yaml

@@ -0,0 +1,24 @@
+stages:
+  preprocess_semeval:
+    cmd: PYTHONPATH=. python src/data/dataset.py
+    deps:
+      - src/data/dataset.py
+      - src/data/preprocess.py
+      - src/data/lang_detect.py
+      - src/data/transliterate.py
+      - data/raw/semeval_restaurants
+      - data/raw/semeval_laptops
+    outs:
+      - data/processed/semeval_train.jsonl
+      - data/processed/semeval_test.jsonl
+
+  preprocess_hindi:
+    cmd: PYTHONPATH=. python src/data/hindi_loader.py
+    deps:
+      - src/data/hindi_loader.py
+      - src/data/preprocess.py
+      - src/data/lang_detect.py
+      - src/data/transliterate.py
+      - data/raw/amazon_hindi/hindi_sentiment.jsonl
+    outs:
+      - data/processed/amazon_hindi.jsonl

src/data/dataset.py

@@ -0,0 +1,70 @@
+import json
+from pathlib import Path
+from datasets import load_from_disk
+from collections import defaultdict
+from src.config import RAW_DIR, SEMEVAL_TRAIN_PATH, SEMEVAL_TEST_PATH
+from src.data.preprocess import clean
+from src.data.lang_detect import detect_language
+
+def process_semeval():
+    rest_path = RAW_DIR / "semeval_restaurants"
+    lap_path = RAW_DIR / "semeval_laptops"
+    
+    rest_data = load_from_disk(str(rest_path))
+    lap_data = load_from_disk(str(lap_path))
+    
+    train_samples = defaultdict(list)
+    test_samples = defaultdict(list)
+    
+    for ds_name, ds, source_name in [("train", rest_data["train"], "restaurants"), 
+                                     ("test", rest_data["test"], "restaurants"),
+                                     ("train", lap_data["train"], "laptops"),
+                                     ("test", lap_data["test"], "laptops")]:
+        target = train_samples if ds_name == "train" else test_samples
+        for row in ds:
+            text = row["text"]
+            span = row["span"]
+            label = row["label"]
+            
+            target[(text, source_name)].append({
+                "term": span,
+                "polarity": label
+            })
+            
+    SEMEVAL_TRAIN_PATH.parent.mkdir(parents=True, exist_ok=True)
+    
+    for path, data_dict in [(SEMEVAL_TRAIN_PATH, train_samples), (SEMEVAL_TEST_PATH, test_samples)]:
+        total = 0
+        lang_counts = defaultdict(int)
+        with open(path, "w", encoding="utf-8") as f:
+            for (text, source), aspects in data_dict.items():
+                lang = detect_language(text)
+                cleaned_text = clean(text, lang)
+                lang_counts[lang] += 1
+                
+                final_aspects = []
+                for aspect in aspects:
+                    term_clean = clean(aspect["term"], lang)
+                    from_idx = cleaned_text.find(term_clean)
+                    to_idx = from_idx + len(term_clean) if from_idx != -1 else -1
+                    final_aspects.append({
+                        "term": term_clean,
+                        "polarity": aspect["polarity"],
+                        "from": from_idx,
+                        "to": to_idx
+                    })
+                
+                sample = {
+                    "text": cleaned_text,
+                    "language": lang,
+                    "aspect_terms": final_aspects,
+                    "source": source
+                }
+                f.write(json.dumps(sample, ensure_ascii=False) + "\n")
+                total += 1
+                
+        print(f"SemEval {path.stem} samples: {total}")
+        print(f"SemEval {path.stem} languages: {dict(lang_counts)}")
+
+if __name__ == "__main__":
+    process_semeval()

src/data/hindi_loader.py

@@ -0,0 +1,55 @@
+import json
+from pathlib import Path
+from src.config import RAW_DIR, AMAZON_HINDI_PATH
+from src.data.preprocess import clean
+from src.data.lang_detect import detect_language
+
+def process_hindi():
+    raw_path = RAW_DIR / "amazon_hindi" / "hindi_sentiment.jsonl"
+    if not raw_path.exists():
+        print(f"File not found: {raw_path}")
+        return
+        
+    AMAZON_HINDI_PATH.parent.mkdir(parents=True, exist_ok=True)
+    
+    total = 0
+    lang_counts = {"hi": 0, "hinglish": 0, "en": 0, "other": 0}
+    
+    with open(raw_path, "r", encoding="utf-8") as fin, \
+         open(AMAZON_HINDI_PATH, "w", encoding="utf-8") as fout:
+        for line in fin:
+            row = json.loads(line)
+            text = row.get("INDIC REVIEW", row.get("text", ""))
+            if not text:
+                continue
+                
+            label = str(row.get("LABEL", row.get("label", ""))).lower()
+            if label == "0" or label == "negative":
+                label = "negative"
+            elif label == "1" or label == "positive":
+                label = "positive"
+            else:
+                label = "neutral"
+                
+            lang = detect_language(text)
+            if lang in lang_counts:
+                lang_counts[lang] += 1
+            else:
+                lang_counts[lang] = 1
+            
+            cleaned_text = clean(text, lang)
+            
+            sample = {
+                "text": cleaned_text,
+                "language": lang,
+                "label": label,
+                "source": "amazon_hindi"
+            }
+            fout.write(json.dumps(sample, ensure_ascii=False) + "\n")
+            total += 1
+            
+    print(f"Hindi samples processed: {total}")
+    print(f"Hindi language distribution: {lang_counts}")
+
+if __name__ == "__main__":
+    process_hindi()

src/data/lang_detect.py

@@ -0,0 +1,41 @@
+import re
+import fasttext
+from src.config import FASTTEXT_MODEL_PATH
+from pathlib import Path
+
+_model = None
+
+def get_model():
+    global _model
+    if _model is None:
+        _model = fasttext.load_model(str(FASTTEXT_MODEL_PATH))
+    return _model
+
+def detect_language(text: str) -> str:
+    """Detects if text is en, hi, hinglish, or other."""
+    if not text or not text.strip():
+        return "other"
+        
+    has_alpha = bool(re.search(r'[^\W\d_]', text))
+    if not has_alpha:
+        return "other"
+        
+    text = text.replace('\n', ' ')
+    model = get_model()
+    predictions = model.predict(text, k=1)
+    label = predictions[0][0].replace("__label__", "")
+    
+    has_latin = bool(re.search(r'[a-zA-Z]', text))
+    has_devanagari = bool(re.search(r'[\u0900-\u097F]', text))
+    
+    is_hinglish = has_latin and has_devanagari
+    
+    if is_hinglish and label in ["en", "hi"]:
+        return "hinglish"
+        
+    if label == "en":
+        return "en"
+    elif label == "hi":
+        return "hi"
+    else:
+        return "other"

src/data/preprocess.py

@@ -0,0 +1,33 @@
+import re
+import unicodedata
+from src.data.transliterate import transliterate
+
+def clean(text: str, language: str) -> str:
+    """Clean text by lowercasing, removing URLs/mentions/hashtags, normalizing unicode, stripping whitespace."""
+    if not text:
+        return ""
+        
+    # lowercase
+    text = text.lower()
+    
+    # remove URLs
+    text = re.sub(r'http\S+|www\.\S+', '', text)
+    
+    # remove mentions
+    text = re.sub(r'@\w+', '', text)
+    
+    # remove hashtags
+    text = re.sub(r'#\w+', '', text)
+    
+    # Apply transliteration only for hi/hinglish inputs
+    if language in ["hi", "hinglish"]:
+        text = transliterate(text, language)
+        
+    # normalize unicode
+    text = unicodedata.normalize("NFKC", text)
+    
+    # strip whitespace
+    text = text.strip()
+    text = re.sub(r'\s+', ' ', text)
+    
+    return text

src/data/transliterate.py

@@ -0,0 +1,40 @@
+import logging
+import unicodedata
+import re
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+try:
+    from indicnlp.transliterate.unicode_transliterate import ItransTransliterator
+    HAS_INDIC_NLP = True
+except ImportError:
+    HAS_INDIC_NLP = False
+    logger.warning("indic-nlp-library not found. Transliteration will fallback to basic unicode handling.")
+
+def transliterate(text: str, src_lang: str) -> str:
+    """Romanize Devanagari text."""
+    if src_lang not in ["hi", "hinglish"]:
+        return text
+        
+    if HAS_INDIC_NLP:
+        try:
+            # We will process word by word if needed, but itrans translates string.
+            # actually to_itrans takes devanagari and romanizes it.
+            roman_text = ItransTransliterator.to_itrans(text, "hi")
+        except Exception as e:
+            logger.warning(f"indicnlp transliteration failed: {e}")
+            roman_text = text
+    else:
+        # Fallback to basic unicode normalization
+        roman_text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode("utf-8")
+
+    if not roman_text:
+        roman_text = text
+        
+    # Normalize common Hinglish spellings
+    # replace acha / accha -> achha
+    roman_text = re.sub(r'\baccha\b', 'achha', roman_text, flags=re.IGNORECASE)
+    roman_text = re.sub(r'\bacha\b', 'achha', roman_text, flags=re.IGNORECASE)
+    
+    return roman_text

tests/test_lang_detect.py

@@ -0,0 +1,18 @@
+from src.data.lang_detect import detect_language
+
+def test_detect_language():
+    samples = [
+        ("This is a simple English sentence.", "en"),
+        ("The food was amazing!", "en"),
+        ("यह एक हिंदी वाक्य है।", "hi"),
+        ("मुझे यह उत्पाद बहुत पसंद आया।", "hi"),
+        ("The phone is great but battery life kharab hai.", "en"), # no devanagari -> en
+        ("Phone bahut badhiya hai, लेकिन battery is bad.", "hinglish"),
+        ("I love this! मुझे यह पसंद है", "hinglish"),
+        ("Bonjour tout le monde", "other"),
+        ("12345 67890 !@#", "other"),
+        ("Just english text with 123", "en"),
+        ("सिर्फ हिंदी 123", "hi")
+    ]
+    for text, expected in samples:
+        assert detect_language(text) == expected, f"Failed on '{text}', expected {expected}"