Update make_data.py

make_data.py

@@ -1,22 +1,48 @@
 import pandas as pd
+from sklearn.model_selection import train_test_split
 
-df = pd.read_csv("hbl_nadra_clean.csv")
 
+DATA_PATH = "hbl_nadra_clean.csv"
 
-print("Total samples:", len(df))
+# ============================================================
+# LIMIT ROWS FOR EXPERIMENTS
+# Set to None for full dataset
+# Example: 50000 for testing
+# ============================================================
+LIMIT_ROWS = None
 
-print("\nColumns:")
-print(df.columns)
 
+df = pd.read_csv(DATA_PATH)
 
-print("\nPipe distribution Urdu:")
-print(df["urdu"].str.count(r"\|").value_counts())
 
+if LIMIT_ROWS:
+    df = df.head(LIMIT_ROWS)
 
-print("\nPipe distribution Roman:")
-print(df["roman"].str.count(r"\|").value_counts())
 
+print("Total rows:", len(df))
 
-print("\nLength statistics:")
-print(df["urdu"].str.len().describe())
-print(df["roman"].str.len().describe())
+
+# ============================================================
+# TRAIN / VALIDATION / TEST SPLIT
+# 95% train
+# 3% validation
+# 2% test
+# ============================================================
+
+train_df, temp_df = train_test_split(
+    df,
+    test_size=0.05,
+    random_state=42
+)
+
+
+val_df, test_df = train_test_split(
+    temp_df,
+    test_size=0.40,
+    random_state=42
+)
+
+
+print("Train:", len(train_df))
+print("Validation:", len(val_df))
+print("Test:", len(test_df))