feat: Phase 1 - project scaffold and data download

notebooks/01_data_exploration.ipynb

@@ -4,16 +4,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# 01 - Data Exploration\n",
-    "Exploring the SemEval 2014 ABSA and Amazon Hindi datasets."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Setup\n",
-    "Imports, config, and dataset loading."
+    "# Setup"
    ]
   },
   {
@@ -27,8 +18,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Load Data\n",
-    "Load SemEval restaurants, laptops, and Amazon Hindi datasets."
+    "# Load Data"
    ]
   },
   {
@@ -42,8 +32,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Language Distribution\n",
-    "Plot language distribution across datasets."
+    "# Language Distribution"
    ]
   },
   {
@@ -57,8 +46,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Sample Reviews\n",
-    "Display random samples from each dataset."
+    "# Sample Reviews"
    ]
   },
   {
@@ -72,8 +60,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Aspect Term Analysis\n",
-    "Analyze aspect term frequency, sentiment distribution, and review lengths."
+    "# Aspect Term Analysis"
    ]
   },
   {
@@ -91,10 +78,18 @@
    "name": "python3"
   },
   "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
    "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
    "version": "3.11.0"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 4
+ "nbformat_minor": 2
 }

requirements.txt

@@ -1,4 +1,4 @@
-transformers==4.39.0
+transformers==4.40.0
 datasets==2.19.0
 torch==2.3.0
 onnxruntime==1.18.0
@@ -12,7 +12,7 @@ pandas==2.2.2
 numpy==1.26.4
 seqeval==1.2.2
 mlflow==2.13.0
-dvc==3.67.1
+dvc==3.51.1
 evidently==0.4.30
 fastapi==0.111.0
 uvicorn==0.29.0

scripts/download_data.py

@@ -1,105 +1,45 @@
+import os
 import urllib.request
-from pathlib import Path
-import json
-import pandas as pd
 from datasets import load_dataset
-
-import sys
-sys.path.insert(0, str(Path(__file__).parent.parent))
-from src.config import RAW_DIR, DATA_DIR
-
-FASTTEXT_URL = "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz"
-FASTTEXT_DEST = DATA_DIR / "models" / "lid.176.ftz"
-
-
-def download_fasttext() -> dict:
-    FASTTEXT_DEST.parent.mkdir(parents=True, exist_ok=True)
-    if FASTTEXT_DEST.exists():
-        size = FASTTEXT_DEST.stat().st_size
-        print(f"  ✓ fasttext LID model already exists ({size / 1e6:.1f} MB)")
-        return {"status": "skipped", "size_mb": round(size / 1e6, 1)}
-    print(f"  Downloading fasttext LID model from {FASTTEXT_URL}...")
-    urllib.request.urlretrieve(FASTTEXT_URL, FASTTEXT_DEST)
-    size = FASTTEXT_DEST.stat().st_size
-    print(f"  ✓ Downloaded ({size / 1e6:.1f} MB)")
-    return {"status": "downloaded", "size_mb": round(size / 1e6, 1)}
-
-
-def download_semeval_laptops() -> dict:
-    dest = RAW_DIR / "semeval_laptops"
-    dest.mkdir(parents=True, exist_ok=True)
-    print("  Loading SemEval 2014 Laptops...")
-    dataset = load_dataset("jakartaresearch/semeval-absa", "laptop")
-    counts = {}
-    for split in dataset:
-        path = dest / f"{split}.jsonl"
-        dataset[split].to_json(path)
-        counts[split] = len(dataset[split])
-        print(f"  ✓ {split}: {len(dataset[split])} samples -> {path}")
-    return counts
-
-
-def download_semeval_restaurants() -> dict:
-    dest = RAW_DIR / "semeval_restaurants"
-    dest.mkdir(parents=True, exist_ok=True)
-    print("  Loading SemEval 2014 Restaurants...")
-    dataset = load_dataset("jakartaresearch/semeval-absa", "restaurant")
-    counts = {}
-    for split in dataset:
-        path = dest / f"{split}.jsonl"
-        dataset[split].to_json(path)
-        counts[split] = len(dataset[split])
-        print(f"  ✓ {split}: {len(dataset[split])} samples -> {path}")
-    return counts
-
-
-def download_amazon_hindi() -> dict:
-    dest = RAW_DIR / "amazon_hindi"
-    dest.mkdir(parents=True, exist_ok=True)
-    print("  Loading Hindi Amazon reviews...")
-
-    base_url = "https://raw.githubusercontent.com/Udrasht/Hindi-Sentiment-Analysis-Corpus-from-Amazon-Reviews/main/data"
-
-    for split, fname in [("train", "train.xlsx"), ("test", "test.xlsx")]:
-        url = f"{base_url}/{fname}"
-        print(f"  Downloading {split} from {url}...")
-        df = pd.read_excel(url)
-        records = []
-        for _, row in df.iterrows():
-            records.append({
-                "text": row["content_hindi"],
-                "title": row["title_hindi"],
-                "rating": int(row["rating"]),
-                "label": row["labels"],
-            })
-        path = dest / f"{split}.jsonl"
-        with open(path, "w", encoding="utf-8") as f:
-            for r in records:
-                f.write(json.dumps(r, ensure_ascii=False) + "\n")
-        print(f"  ✓ {split}: {len(records)} samples -> {path}")
-
-    return {"train": 3527, "test": 884}
-
-
-def main():
-    results = {}
-    print("\n=== Downloading fasttext LID model ===")
-    results["fasttext"] = download_fasttext()
-
-    print("\n=== Downloading SemEval 2014 Laptops ===")
-    results["semeval_laptops"] = download_semeval_laptops()
-
-    print("\n=== Downloading SemEval 2014 Restaurants ===")
-    results["semeval_restaurants"] = download_semeval_restaurants()
-
-    print("\n=== Downloading Hindi Amazon reviews ===")
-    results["amazon_hindi"] = download_amazon_hindi()
-
-    print("\n" + "=" * 50)
-    print("DOWNLOAD SUMMARY")
-    print("=" * 50)
-    print(json.dumps(results, indent=2))
-
+from src.config import DATA_DIR, RAW_DIR, FASTTEXT_MODEL_PATH
+
+def download_fasttext():
+    url = "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz"
+    os.makedirs(FASTTEXT_MODEL_PATH.parent, exist_ok=True)
+    if not FASTTEXT_MODEL_PATH.exists():
+        print("Downloading fastText LID model...")
+        urllib.request.urlretrieve(url, FASTTEXT_MODEL_PATH)
+    else:
+        print("fastText LID model already exists.")
+    print(f"fastText model size: {os.path.getsize(FASTTEXT_MODEL_PATH) / 1024 / 1024:.2f} MB")
+
+def download_semeval():
+    print("Downloading SemEval datasets...")
+    restaurants = load_dataset("tomaarsen/absa-semeval-2014-restaurants")
+    laptops = load_dataset("tomaarsen/absa-semeval-2014-laptops")
+    
+    rest_path = RAW_DIR / "semeval_restaurants"
+    lap_path = RAW_DIR / "semeval_laptops"
+    
+    restaurants.save_to_disk(str(rest_path))
+    laptops.save_to_disk(str(lap_path))
+    
+    print(f"SemEval Restaurants train samples: {len(restaurants['train'])}")
+    print(f"SemEval Laptops train samples: {len(laptops['train'])}")
+
+def download_amazon_hindi():
+    print("Downloading Amazon Hindi dataset...")
+    # Load just 5000 from train
+    amz_hi = load_dataset("amazon_reviews_multi", "hi", split="train[:5000]")
+    
+    amz_path = RAW_DIR / "amazon_hindi"
+    amz_hi.save_to_disk(str(amz_path))
+    
+    print(f"Amazon Hindi train samples: {len(amz_hi)}")
 
 if __name__ == "__main__":
-    main()
+    os.makedirs(RAW_DIR, exist_ok=True)
+    download_fasttext()
+    download_semeval()
+    download_amazon_hindi()
+    print("Download complete.")