File size: 3,183 Bytes
b31728c 3733c61 b31728c e6f08d5 b31728c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | import torch
from torch .utils .data import Dataset
import pandas as pd
class UrduTextDataset (Dataset ):
def __init__ (self ,csv_paths ,tokenizer ,max_length =128 ,task ="sentiment"):
"""
Custom PyTorch Dataset for Urdu Sentiment and Emotion Analysis.
NOTE: Tokenization here does NOT pad. Padding is applied per-batch by a
DataCollatorWithPadding in the training script (dynamic padding), which
is much faster than padding every sample to max_length -- especially on
CPU, where wasted FLOPs on padding tokens dominate runtime.
Args:
csv_paths (list or str): Path(s) to the cleaned CSV files.
tokenizer: HuggingFace tokenizer (XLM-RoBERTa).
max_length (int): Max token length for truncation.
task (str): 'sentiment' or 'emotion'.
"""
self .tokenizer =tokenizer
self .max_length =max_length
self .task =task
if isinstance (csv_paths ,str ):
csv_paths =[csv_paths ]
dfs =[pd .read_csv (path )for path in csv_paths ]
self .data =pd .concat (dfs ,ignore_index =True )
before =len (self .data )
self .data =self .data .dropna (subset =['text','label'])
if task =="sentiment":
label_map ={
'P':2 ,'O':1 ,'N':0 ,
0 :2 ,1 :0 ,2 :1 ,
'0':2 ,'1':0 ,'2':1 ,
}
elif task =="emotion":
label_map ={'joy':0 ,'anger':1 ,'fear':2 ,'sadness':3 }
else :
raise ValueError ("Task must be either 'sentiment' or 'emotion'")
self .data ['label']=self .data ['label'].map (label_map )
unmapped =int (self .data ['label'].isna ().sum ())
self .data =self .data .dropna (subset =['label'])
self .data ['label']=self .data ['label'].astype (int )
after =len (self .data )
print (f" [{task }] kept {after }/{before } rows "
f"({before -after } dropped, {unmapped } had invalid labels)")
print (f" [{task }] class counts: "
f"{self .data ['label'].value_counts ().sort_index ().to_dict ()}")
self .texts =self .data ['text'].astype (str ).tolist ()
self .labels =self .data ['label'].tolist ()
def __len__ (self ):
return len (self .texts )
def __getitem__ (self ,idx ):
encoding =self .tokenizer (
self .texts [idx ],
add_special_tokens =True ,
max_length =self .max_length ,
truncation =True ,
)
return {
'input_ids':encoding ['input_ids'],
'attention_mask':encoding ['attention_mask'],
'labels':torch .tensor (self .labels [idx ],dtype =torch .long ),
}
def get_class_weights (self ,num_labels ):
"""Inverse-frequency class weights for a weighted loss (handles imbalance)."""
counts =self .data ['label'].value_counts ().sort_index ()
counts =counts .reindex (range (num_labels ),fill_value =0 )
total =counts .sum ()
weights =total /(num_labels *counts .replace (0 ,1 ))
return torch .tensor (weights .values ,dtype =torch .float )
|