Add README, tokenizer, results

requirements.txt

@@ -0,0 +1,7 @@
+tensorflow>=2.13.0
+numpy>=1.24.0
+pandas>=2.0.0
+scikit-learn>=1.3.0
+matplotlib>=3.7.0
+seaborn>=0.12.0
+huggingface_hub>=0.20.0

results/all_strategies_results.csv

@@ -1,25 +1,25 @@
 Strategy,Phase,Accuracy,Balanced Acc,Precision,Recall,Specificity,F1,ROC-AUC
-english -> hindi -> hinglish -> Full,english,0.77007700770077,0.7701428019217391,0.7630083078268474,0.7762455516014235,0.7640400522420548,0.7695700110253583,0.8503583507499337
-english -> hindi -> hinglish -> Full,hindi,0.5506896551724138,0.5,0.0,0.0,1.0,0.0,0.568860406431668
-english -> hindi -> hinglish -> Full,hinglish,0.6780383795309168,0.6361898079670295,0.6443850267379679,0.4295900178253119,0.8427895981087471,0.5155080213903743,0.6691424044365222
-english -> hindi -> hinglish -> Full,Full,0.6796204247627655,0.667785959382029,0.7243319268635724,0.5009727626459144,0.8345991561181435,0.5922944220816562,0.7599073515408232
-english -> hinglish -> hindi -> Full,english,0.772057205720572,0.7722537093873023,0.7587532023911187,0.7904804270462633,0.7540269917283413,0.7742919389978213,0.8524885275084165
-english -> hinglish -> hindi -> Full,hinglish,0.6631130063965884,0.6369894185914211,0.5900621118012422,0.5080213903743316,0.7659574468085106,0.5459770114942529,0.6898774983881367
-english -> hinglish -> hindi -> Full,hindi,0.5810344827586207,0.5618840679305163,0.5497737556561086,0.372985418265541,0.7507827175954915,0.4444444444444444,0.5974623370469668
-english -> hinglish -> hindi -> Full,Full,0.6813149570718482,0.6739596159845014,0.6899087967049132,0.5702821011673151,0.7776371308016877,0.6244175209692451,0.7535184660723374
-hindi -> english -> hinglish -> Full,hindi,0.5662068965517242,0.5318909063473292,0.5490196078431373,0.1933998465080583,0.8703819661865999,0.286038592508513,0.5747905584675026
-hindi -> english -> hinglish -> Full,english,0.777997799779978,0.7783349626698604,0.7580174927113703,0.8096085409252669,0.7470613844144537,0.7829640782964078,0.8549286590741134
-hindi -> english -> hinglish -> Full,hinglish,0.6879886282871357,0.657374748738954,0.6367713004484304,0.5062388591800356,0.8085106382978723,0.564051638530288,0.7172264994542843
-hindi -> english -> hinglish -> Full,Full,0.6853818346136467,0.680189709238372,0.6810368349249659,0.6070038910505836,0.7533755274261603,0.6418927607046419,0.7528032084749381
-hindi -> hinglish -> english -> Full,hindi,0.5779310344827586,0.5523552170680732,0.5561877667140825,0.3000767459708365,0.80463368816531,0.3898305084745763,0.5971876950786947
-hindi -> hinglish -> english -> Full,hinglish,0.6986496090973703,0.6524285407264131,0.7020648967551623,0.4242424242424242,0.8806146572104019,0.5288888888888889,0.7108801827199825
-hindi -> hinglish -> english -> Full,english,0.777997799779978,0.7782685368661275,0.7610619469026548,0.8033807829181495,0.7531562908141054,0.7816489937243021,0.8562821574481336
-hindi -> hinglish -> english -> Full,Full,0.6865115228197017,0.6801490748493655,0.6899687411196362,0.5904669260700389,0.7698312236286919,0.6363517232341764,0.7507057916317786
-hinglish -> english -> hindi -> Full,hinglish,0.6652452025586354,0.6273530043867966,0.6113861386138614,0.4402852049910873,0.8144208037825059,0.5119170984455959,0.6691697955778055
-hinglish -> english -> hindi -> Full,english,0.7716171617161716,0.7722691054555144,0.7387529597474349,0.8327402135231317,0.7117979973878973,0.7829360100376411,0.8484048898687285
-hinglish -> english -> hindi -> Full,hindi,0.5637931034482758,0.5265203703605811,0.550531914893617,0.1588641596316193,0.8941765810895429,0.2465753424657534,0.5981714083053846
-hinglish -> english -> hindi -> Full,Full,0.6778129236330773,0.6717689504014186,0.6767676767676768,0.5865758754863813,0.7569620253164557,0.6284523189161021,0.7521193758311581
-hinglish -> hindi -> english -> Full,hinglish,0.6837242359630419,0.6460221741823744,0.645,0.4598930481283422,0.8321513002364066,0.5369406867845994,0.6929347711575496
-hinglish -> hindi -> english -> Full,hindi,0.5924137931034483,0.5742645818545997,0.5665566556655666,0.3952417498081351,0.7532874139010645,0.4656419529837251,0.5963745818497942
-hinglish -> hindi -> english -> Full,english,0.7764576457645764,0.776778023168081,0.7573099415204678,0.8064946619217082,0.7470613844144537,0.7811288237828522,0.8534031895230821
-hinglish -> hindi -> english -> Full,Full,0.6844780840488025,0.6774609458372326,0.6917708636231462,0.5785505836575876,0.7763713080168776,0.6301152165276123,0.7547951144330066
+english -> hindi -> hinglish -> Full,english,0.7701,0.7701,0.763,0.7762,0.764,0.7696,0.8504
+english -> hindi -> hinglish -> Full,hindi,0.5507,0.5,0.0,0.0,1.0,0.0,0.5689
+english -> hindi -> hinglish -> Full,hinglish,0.678,0.6362,0.6444,0.4296,0.8428,0.5155,0.6691
+english -> hindi -> hinglish -> Full,Full,0.6796,0.6678,0.7243,0.501,0.8346,0.5923,0.7599
+english -> hinglish -> hindi -> Full,english,0.7721,0.7723,0.7588,0.7905,0.754,0.7743,0.8525
+english -> hinglish -> hindi -> Full,hinglish,0.6631,0.637,0.5901,0.508,0.766,0.546,0.6899
+english -> hinglish -> hindi -> Full,hindi,0.581,0.5619,0.5498,0.373,0.7508,0.4444,0.5975
+english -> hinglish -> hindi -> Full,Full,0.6813,0.674,0.6899,0.5703,0.7776,0.6244,0.7535
+hindi -> english -> hinglish -> Full,hindi,0.5662,0.5319,0.549,0.1934,0.8704,0.286,0.5748
+hindi -> english -> hinglish -> Full,english,0.778,0.7783,0.758,0.8096,0.7471,0.783,0.8549
+hindi -> english -> hinglish -> Full,hinglish,0.688,0.6574,0.6368,0.5062,0.8085,0.5641,0.7172
+hindi -> english -> hinglish -> Full,Full,0.6854,0.6802,0.681,0.607,0.7534,0.6419,0.7528
+hindi -> hinglish -> english -> Full,hindi,0.5779,0.5524,0.5562,0.3001,0.8046,0.3898,0.5972
+hindi -> hinglish -> english -> Full,hinglish,0.6986,0.6524,0.7021,0.4242,0.8806,0.5289,0.7109
+hindi -> hinglish -> english -> Full,english,0.778,0.7783,0.7611,0.8034,0.7532,0.7816,0.8563
+hindi -> hinglish -> english -> Full,Full,0.6865,0.6801,0.69,0.5905,0.7698,0.6364,0.7507
+hinglish -> english -> hindi -> Full,hinglish,0.6652,0.6274,0.6114,0.4403,0.8144,0.5119,0.6692
+hinglish -> english -> hindi -> Full,english,0.7716,0.7723,0.7388,0.8327,0.7118,0.7829,0.8484
+hinglish -> english -> hindi -> Full,hindi,0.5638,0.5265,0.5505,0.1589,0.8942,0.2466,0.5982
+hinglish -> english -> hindi -> Full,Full,0.6778,0.6718,0.6768,0.5866,0.757,0.6285,0.7521
+hinglish -> hindi -> english -> Full,hinglish,0.6837,0.646,0.645,0.4599,0.8322,0.5369,0.6929
+hinglish -> hindi -> english -> Full,hindi,0.5924,0.5743,0.5666,0.3952,0.7533,0.4656,0.5964
+hinglish -> hindi -> english -> Full,english,0.7765,0.7768,0.7573,0.8065,0.7471,0.7811,0.8534
+hinglish -> hindi -> english -> Full,Full,0.6845,0.6775,0.6918,0.5786,0.7764,0.6301,0.7548

results/english_to_hindi_to_hinglish_results.csv

@@ -0,0 +1,5 @@
+Strategy,Phase,Accuracy,Balanced Acc,Precision,Recall,Specificity,F1,ROC-AUC
+english -> hindi -> hinglish -> Full,english,0.7701,0.7701,0.763,0.7762,0.764,0.7696,0.8504
+english -> hindi -> hinglish -> Full,hindi,0.5507,0.5,0.0,0.0,1.0,0.0,0.5689
+english -> hindi -> hinglish -> Full,hinglish,0.678,0.6362,0.6444,0.4296,0.8428,0.5155,0.6691
+english -> hindi -> hinglish -> Full,Full,0.6796,0.6678,0.7243,0.501,0.8346,0.5923,0.7599

results/english_to_hinglish_to_hindi_results.csv

@@ -0,0 +1,5 @@
+Strategy,Phase,Accuracy,Balanced Acc,Precision,Recall,Specificity,F1,ROC-AUC
+english -> hinglish -> hindi -> Full,english,0.7721,0.7723,0.7588,0.7905,0.754,0.7743,0.8525
+english -> hinglish -> hindi -> Full,hinglish,0.6631,0.637,0.5901,0.508,0.766,0.546,0.6899
+english -> hinglish -> hindi -> Full,hindi,0.581,0.5619,0.5498,0.373,0.7508,0.4444,0.5975
+english -> hinglish -> hindi -> Full,Full,0.6813,0.674,0.6899,0.5703,0.7776,0.6244,0.7535

results/hindi_to_english_to_hinglish_results.csv

@@ -0,0 +1,5 @@
+Strategy,Phase,Accuracy,Balanced Acc,Precision,Recall,Specificity,F1,ROC-AUC
+hindi -> english -> hinglish -> Full,hindi,0.5662,0.5319,0.549,0.1934,0.8704,0.286,0.5748
+hindi -> english -> hinglish -> Full,english,0.778,0.7783,0.758,0.8096,0.7471,0.783,0.8549
+hindi -> english -> hinglish -> Full,hinglish,0.688,0.6574,0.6368,0.5062,0.8085,0.5641,0.7172
+hindi -> english -> hinglish -> Full,Full,0.6854,0.6802,0.681,0.607,0.7534,0.6419,0.7528

results/hindi_to_hinglish_to_english_results.csv

@@ -0,0 +1,5 @@
+Strategy,Phase,Accuracy,Balanced Acc,Precision,Recall,Specificity,F1,ROC-AUC
+hindi -> hinglish -> english -> Full,hindi,0.5779,0.5524,0.5562,0.3001,0.8046,0.3898,0.5972
+hindi -> hinglish -> english -> Full,hinglish,0.6986,0.6524,0.7021,0.4242,0.8806,0.5289,0.7109
+hindi -> hinglish -> english -> Full,english,0.778,0.7783,0.7611,0.8034,0.7532,0.7816,0.8563
+hindi -> hinglish -> english -> Full,Full,0.6865,0.6801,0.69,0.5905,0.7698,0.6364,0.7507

results/hinglish_to_english_to_hindi_results.csv

@@ -0,0 +1,5 @@
+Strategy,Phase,Accuracy,Balanced Acc,Precision,Recall,Specificity,F1,ROC-AUC
+hinglish -> english -> hindi -> Full,hinglish,0.6652,0.6274,0.6114,0.4403,0.8144,0.5119,0.6692
+hinglish -> english -> hindi -> Full,english,0.7716,0.7723,0.7388,0.8327,0.7118,0.7829,0.8484
+hinglish -> english -> hindi -> Full,hindi,0.5638,0.5265,0.5505,0.1589,0.8942,0.2466,0.5982
+hinglish -> english -> hindi -> Full,Full,0.6778,0.6718,0.6768,0.5866,0.757,0.6285,0.7521

results/hinglish_to_hindi_to_english_results.csv

@@ -0,0 +1,5 @@
+Strategy,Phase,Accuracy,Balanced Acc,Precision,Recall,Specificity,F1,ROC-AUC
+hinglish -> hindi -> english -> Full,hinglish,0.6837,0.646,0.645,0.4599,0.8322,0.5369,0.6929
+hinglish -> hindi -> english -> Full,hindi,0.5924,0.5743,0.5666,0.3952,0.7533,0.4656,0.5964
+hinglish -> hindi -> english -> Full,english,0.7765,0.7768,0.7573,0.8065,0.7471,0.7811,0.8534
+hinglish -> hindi -> english -> Full,Full,0.6845,0.6775,0.6918,0.5786,0.7764,0.6301,0.7548

results/plan_b_results.csv

@@ -0,0 +1,7 @@
+Strategy,Accuracy,Balanced Acc,Precision,Recall,Specificity,F1,ROC-AUC
+english -> hindi -> hinglish -> Full,0.6836,0.6759,0.6951,0.5679,0.784,0.6251,0.7578
+english -> hinglish -> hindi -> Full,0.688,0.6836,0.6795,0.6214,0.7458,0.6491,0.7581
+hindi -> english -> hinglish -> Full,0.6841,0.6765,0.6957,0.5688,0.7842,0.6259,0.7563
+hindi -> hinglish -> english -> Full,0.6855,0.6796,0.6855,0.5968,0.7624,0.6381,0.7534
+hinglish -> english -> hindi -> Full,0.6859,0.6782,0.6987,0.5696,0.7869,0.6275,0.7623
+hinglish -> hindi -> english -> Full,0.6698,0.6584,0.7044,0.4983,0.8186,0.5837,0.7428