Upload 24 files

Llemma_Finetuned.py

@@ -0,0 +1,36 @@
+import transformers
+import torch
+# from transformers import BitsAndBytesConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from deepeval.models import DeepEvalBaseLLM
+
+
+class Llemma_Finetuned(DeepEvalBaseLLM):
+    def __init__(
+        self,
+        model,
+        tokenizer
+    ):
+        self.model = model
+        self.tokenizer = tokenizer
+
+    def load_model(self):
+        return self.model
+
+    def generate(self, prompt: str) -> str:
+        model = self.load_model()
+
+        device = "cuda" # the device to load the model onto
+
+        model_inputs = self.tokenizer([prompt], return_tensors="pt").to(device)
+        model.to(device)
+
+        generated_ids = model.generate(**model_inputs, max_new_tokens=100, do_sample=True)
+        return self.tokenizer.batch_decode(generated_ids)[0]
+
+    async def a_generate(self, prompt: str) -> str:
+        return self.generate(prompt)
+
+    def get_model_name(self):
+        return "Llemma Finetuned"

before_run.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# module load python/3.11
+# pip install virtualenv
+
+# python -m venv mse_env
+source ./mse_env/Scripts/activate
+# pip3 install pnglatex
+# pip install --upgrade pip
+pip install -r requirements.txt
+# pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128
+
+# pip3 install transformers datasets accelerate
+# pip uninstall install llm-toolkit
+# pip install -q -U transformers accelerate bitsandbytes seqeval evaluate trl peft
+# pip3 install -q -U bitsandbytes==0.42.0
+# pip3 install -q -U peft==0.8.2
+# pip3 install -q -U trl==0.7.10
+# pip3 install -q -U accelerate==0.27.1
+# pip3 install -q -U datasets==2.17.0
+# pip3 install -q -U transformers==4.38.0
+# pip3 install zope.interface=e0n<
+# pip3 install jsonl2json

dataset_row_stl.txt

@@ -0,0 +1,3243 @@
+3200
+246
+2680
+1090
+2739
+2385
+199
+3193
+2757
+64
+2670
+720
+3023
+3137
+2682
+249
+2073
+2646
+754
+732
+3215
+2800
+2731
+3034
+1940
+2791
+3190
+2324
+2302
+212
+2978
+768
+2817
+1729
+548
+2337
+2393
+2456
+762
+594
+2779
+253
+2986
+220
+11
+265
+2960
+1621
+2343
+1064
+3241
+778
+2782
+5
+2804
+3064
+1514
+3044
+256
+2729
+3025
+1646
+2773
+1519
+2314
+3051
+2745
+2637
+75
+2777
+3126
+267
+2650
+932
+3138
+774
+2266
+3192
+2396
+88
+1151
+340
+1292
+2384
+2716
+1656
+2642
+3170
+779
+3177
+2251
+3063
+1611
+701
+271
+283
+1800
+478
+3069
+3173
+91
+1223
+2356
+1126
+2676
+3046
+131
+282
+3011
+2056
+1283
+1986
+2234
+2813
+1362
+3065
+2715
+1980
+2169
+734
+1590
+2191
+285
+844
+1544
+1518
+2188
+2806
+2816
+3232
+3004
+3005
+782
+1605
+1239
+2700
+2766
+3220
+3057
+3080
+70
+919
+2945
+79
+1456
+47
+264
+270
+2464
+375
+606
+257
+1999
+2494
+1709
+2645
+73
+1222
+1645
+2301
+1630
+290
+2048
+16
+1871
+2349
+2934
+1050
+217
+3007
+3128
+251
+2674
+3058
+3168
+1062
+2737
+42
+2636
+3098
+2954
+59
+215
+1275
+1982
+1674
+3032
+52
+2661
+288
+662
+933
+45
+2699
+30
+2693
+742
+222
+25
+3214
+3157
+793
+2752
+1998
+747
+2552
+43
+3198
+2336
+244
+1898
+146
+258
+529
+1521
+3087
+304
+2924
+2244
+247
+2072
+1961
+2501
+2955
+2669
+1695
+1
+338
+3227
+2296
+3240
+1265
+1419
+3205
+2759
+3076
+204
+954
+2678
+161
+3155
+302
+1500
+1884
+2673
+250
+1619
+2798
+276
+999
+2107
+1974
+1862
+2794
+3037
+2078
+1750
+2537
+320
+3130
+2122
+2476
+32
+287
+787
+1610
+2253
+1677
+721
+1665
+2662
+2068
+1320
+902
+922
+3213
+789
+1985
+231
+512
+576
+19
+274
+1209
+2792
+2183
+2354
+1505
+1523
+1603
+1919
+147
+2320
+2815
+2472
+1959
+2217
+1499
+3219
+1797
+2708
+3002
+1679
+2623
+1056
+2760
+2225
+728
+1658
+908
+2571
+2735
+1893
+1529
+248
+2640
+2802
+34
+685
+1650
+2131
+1620
+2778
+396
+1299
+6
+584
+1334
+1615
+1453
+2717
+1584
+216
+289
+2338
+208
+2229
+2796
+2814
+851
+90
+224
+505
+3071
+353
+2937
+849
+2339
+1385
+1641
+751
+2298
+2684
+3093
+1414
+1587
+2560
+9
+1673
+755
+2775
+749
+2663
+7
+1023
+2382
+2600
+2608
+668
+2363
+2165
+2173
+1796
+446
+3187
+2630
+2807
+2194
+2204
+1368
+2134
+2836
+3082
+522
+2486
+931
+2358
+2990
+487
+949
+1531
+1648
+597
+2672
+462
+2820
+3133
+725
+2063
+2523
+1407
+1520
+1383
+2111
+972
+869
+3127
+1296
+1885
+2533
+3206
+1113
+1686
+1773
+1854
+3226
+166
+763
+1445
+2923
+2941
+3142
+947
+2053
+3150
+664
+625
+1107
+2999
+479
+561
+1714
+2602
+2305
+3078
+501
+907
+2237
+2647
+2656
+121
+1541
+1623
+3
+252
+2748
+3180
+562
+2242
+129
+723
+424
+1604
+1627
+1540
+2784
+1617
+3135
+233
+568
+2695
+566
+1446
+2285
+84
+351
+1443
+3021
+3059
+3061
+273
+460
+3062
+520
+1558
+1708
+2318
+588
+764
+2554
+2681
+57
+514
+2720
+2765
+1516
+1457
+711
+1682
+2386
+65
+1701
+2310
+1378
+1430
+279
+3166
+1706
+739
+55
+2015
+40
+371
+905
+225
+998
+1330
+1426
+3235
+278
+3160
+493
+759
+1957
+1864
+842
+1628
+1997
+2087
+2141
+241
+2683
+51
+906
+2572
+2723
+2365
+2145
+117
+1870
+2439
+2624
+2742
+3070
+209
+880
+1672
+2935
+3101
+20
+1034
+1256
+2770
+1911
+296
+2457
+103
+2088
+495
+339
+48
+578
+2411
+2761
+1236
+182
+400
+1032
+2772
+3230
+1063
+1613
+36
+2489
+2345
+2569
+331
+1743
+2092
+2541
+2334
+1081
+1659
+1415
+158
+676
+1260
+1827
+3243
+1551
+1805
+1950
+2548
+1237
+1836
+2399
+2781
+1031
+1660
+1953
+136
+2991
+1536
+1703
+2992
+1084
+2484
+1533
+2261
+961
+1427
+3031
+1490
+750
+124
+2089
+3231
+1294
+485
+1547
+2379
+3239
+2732
+137
+560
+1020
+3050
+3111
+196
+1622
+2397
+912
+1764
+975
+2853
+991
+1125
+255
+1338
+1693
+2998
+1506
+1822
+3204
+2518
+441
+719
+827
+1863
+29
+1033
+2725
+119
+349
+1038
+3088
+2513
+2539
+3113
+239
+692
+3008
+2193
+436
+359
+1070
+781
+2135
+551
+1626
+2829
+3075
+3117
+1073
+707
+2987
+227
+1357
+263
+3132
+2002
+3163
+1371
+1400
+1602
+2012
+1865
+2601
+2149
+164
+3154
+2521
+1996
+2487
+2655
+714
+3194
+286
+1545
+1598
+2168
+2730
+1455
+3019
+3146
+717
+1293
+1449
+2556
+627
+2498
+2593
+1944
+2746
+2162
+2211
+2707
+307
+890
+81
+786
+2245
+2437
+2181
+2660
+2067
+82
+3077
+2090
+3033
+1286
+1924
+2085
+360
+2309
+235
+1253
+1437
+2155
+26
+2270
+532
+1120
+3164
+412
+2254
+1218
+2381
+2026
+2429
+1241
+938
+1006
+2006
+202
+1329
+2267
+894
+2696
+1906
+2736
+3222
+1638
+2788
+3079
+46
+165
+492
+730
+1634
+2125
+1583
+599
+2677
+541
+1101
+1683
+1697
+2733
+559
+818
+2156
+2801
+2138
+2370
+544
+2220
+3125
+797
+1060
+1279
+1089
+2333
+2371
+336
+1012
+2691
+415
+2942
+595
+737
+1347
+1819
+2187
+262
+523
+605
+1392
+2207
+2795
+2809
+2344
+1579
+2989
+12
+450
+3237
+2126
+2264
+2512
+3020
+1552
+1887
+1812
+313
+1848
+2926
+3199
+2367
+346
+1737
+1823
+1945
+519
+1436
+2062
+2195
+1333
+2767
+1306
+2164
+2808
+1208
+1904
+2789
+655
+2128
+2185
+411
+1849
+92
+780
+2103
+2166
+297
+1304
+2011
+957
+1069
+1314
+1393
+2419
+1342
+1018
+1991
+452
+757
+1818
+1943
+3040
+1692
+293
+942
+1798
+1696
+2274
+2573
+234
+982
+2785
+2966
+429
+607
+838
+1112
+3056
+2634
+2127
+145
+1652
+997
+1971
+2461
+3072
+567
+1966
+327
+1791
+187
+580
+619
+530
+2819
+2947
+1667
+342
+1083
+2639
+1832
+2689
+421
+2083
+2391
+2633
+2671
+2038
+83
+769
+2066
+2762
+2198
+3009
+384
+2407
+3152
+2988
+439
+2380
+2535
+2953
+1909
+792
+1761
+2743
+652
+1281
+1918
+2776
+41
+826
+706
+326
+639
+1203
+2416
+1847
+2698
+260
+558
+1067
+1205
+1364
+3233
+93
+2424
+612
+475
+2321
+118
+2615
+923
+2561
+409
+2364
+466
+1762
+468
+1785
+301
+3182
+2352
+358
+2958
+393
+1624
+2307
+10
+2516
+841
+1542
+2865
+691
+1612
+3112
+2
+14
+3122
+144
+556
+1450
+2093
+604
+617
+2532
+1914
+632
+683
+834
+317
+486
+703
+1441
+3153
+292
+2199
+614
+2750
+143
+322
+2741
+1197
+1367
+2041
+3195
+573
+1233
+1363
+2589
+303
+344
+2036
+1507
+3092
+3119
+2226
+206
+2413
+122
+1735
+2751
+936
+1469
+2738
+986
+3038
+2617
+324
+2297
+2553
+3048
+213
+1121
+2405
+3167
+60
+240
+1734
+3178
+3188
+427
+709
+2299
+621
+1527
+1394
+1560
+1923
+1004
+2342
+2566
+948
+1336
+3169
+545
+2332
+2536
+2562
+1245
+2143
+968
+1569
+1962
+2178
+1784
+2154
+2744
+953
+1016
+2206
+2287
+37
+211
+1572
+888
+1440
+2281
+1916
+3054
+1308
+1684
+1804
+2701
+78
+2042
+770
+2366
+2369
+1402
+197
+2054
+2171
+2944
+645
+1353
+1770
+2939
+416
+3225
+542
+1776
+1929
+2973
+565
+935
+1666
+1369
+291
+1625
+105
+1289
+210
+854
+2787
+1866
+207
+385
+718
+1043
+1076
+694
+729
+2361
+3221
+653
+54
+788
+1280
+3131
+2709
+539
+3124
+1657
+939
+969
+1756
+2390
+203
+521
+510
+1332
+2279
+672
+1915
+1931
+39
+426
+791
+1794
+928
+930
+1021
+1905
+3238
+1710
+1309
+1724
+1890
+1405
+710
+98
+1424
+1668
+2544
+134
+1573
+1698
+2102
+2451
+2705
+2110
+2248
+2410
+2690
+2208
+1517
+1565
+2119
+644
+2249
+2706
+800
+966
+1219
+2549
+3104
+77
+1287
+2594
+35
+2189
+3022
+1954
+3042
+2120
+2564
+1897
+2108
+623
+2152
+341
+1841
+2417
+1305
+214
+2526
+68
+1843
+589
+1324
+1015
+1681
+1315
+2104
+2177
+335
+232
+506
+513
+3097
+1130
+1829
+1395
+1726
+8
+72
+1801
+1851
+2510
+1227
+2897
+577
+2282
+2300
+1082
+2209
+853
+1272
+2582
+17
+62
+753
+3201
+2970
+1928
+1509
+1472
+2101
+1471
+1799
+28
+767
+945
+1760
+2665
+1664
+2074
+1040
+1288
+1307
+1820
+398
+1783
+929
+1717
+2485
+2590
+2638
+1972
+2130
+2139
+2505
+2983
+1564
+2213
+2580
+647
+2065
+102
+195
+499
+700
+1616
+1873
+1007
+2493
+2278
+2768
+722
+1325
+1495
+1958
+2856
+86
+2167
+2482
+363
+1846
+1589
+925
+2376
+1699
+2981
+184
+2201
+3149
+3099
+1094
+1274
+1360
+2432
+3162
+2740
+3116
+1399
+185
+3159
+2230
+2447
+2950
+712
+2275
+74
+699
+1853
+1377
+2029
+3147
+3197
+973
+1277
+2515
+2129
+2749
+261
+399
+1561
+2121
+205
+940
+1136
+900
+1700
+2543
+3174
+308
+773
+2968
+2450
+1595
+831
+1268
+1831
+2659
+50
+618
+1316
+772
+2288
+445
+1370
+3189
+2005
+2940
+1512
+1732
+862
+951
+1341
+1359
+1951
+2928
+1880
+2247
+2685
+2075
+1599
+2575
+2205
+2558
+2112
+552
+613
+269
+281
+1432
+937
+295
+1358
+13
+391
+2402
+2946
+1538
+2697
+3129
+1312
+1988
+1476
+2529
+3212
+474
+189
+1661
+1425
+63
+494
+2769
+2949
+2982
+916
+1705
+2057
+2557
+2584
+579
+1212
+1596
+3029
+1606
+1711
+1147
+1431
+679
+1763
+715
+2182
+1041
+1231
+2428
+1494
+1078
+1380
+2621
+2805
+154
+1388
+1331
+1444
+2269
+2392
+135
+142
+498
+2722
+1303
+2372
+1354
+2401
+1497
+2308
+2980
+150
+2153
+2899
+1742
+2702
+525
+1420
+1480
+2438
+1153
+1935
+2398
+2902
+1567
+2844
+3183
+631
+677
+1036
+1814
+2008
+2236
+1990
+2163
+2453
+509
+1976
+2454
+3066
+3067
+1028
+2018
+1777
+1526
+959
+2284
+1642
+741
+2626
+3083
+2100
+89
+174
+128
+731
+3172
+1011
+2588
+316
+2007
+2293
+1489
+2459
+1834
+2896
+71
+2406
+2667
+2811
+1170
+2443
+2753
+735
+1291
+1386
+1562
+2357
+2694
+1586
+333
+1201
+1575
+1725
+1872
+2734
+1224
+1403
+2077
+2664
+58
+2440
+2657
+2718
+2976
+76
+2522
+200
+1845
+2239
+172
+1418
+1671
+1903
+2679
+2577
+1815
+637
+1555
+1609
+2879
+329
+2097
+839
+1297
+1478
+586
+2184
+1556
+149
+1704
+377
+56
+572
+716
+2260
+1027
+1807
+1095
+1727
+3000
+660
+3242
+1580
+630
+1895
+2001
+2252
+2711
+1257
+857
+1373
+1461
+2466
+3100
+563
+3185
+1165
+2724
+1326
+1886
+2142
+1766
+1746
+1808
+808
+881
+2290
+538
+974
+921
+2520
+1010
+2084
+2497
+765
+3096
+1396
+1210
+1946
+2160
+2277
+1029
+1941
+1754
+555
+643
+1217
+2412
+2653
+3085
+1636
+2931
+2404
+3223
+2597
+775
+2961
+609
+1877
+180
+2105
+1614
+382
+1718
+690
+243
+674
+2312
+2441
+99
+2470
+1939
+2918
+3091
+1788
+624
+1361
+2763
+2265
+1014
+2468
+1398
+1588
+2259
+369
+2728
+2971
+3211
+866
+2686
+671
+2258
+1123
+407
+646
+2170
+2212
+2962
+1768
+1789
+713
+1504
+1694
+2276
+456
+2328
+3134
+1301
+2353
+3094
+148
+784
+2196
+2232
+2540
+564
+1376
+950
+410
+2186
+455
+790
+970
+813
+2426
+2652
+2951
+1748
+3003
+985
+1148
+2003
+3224
+275
+2469
+1983
+1086
+2123
+2224
+2648
+1248
+516
+1468
+3196
+2527
+1323
+465
+1390
+1653
+319
+183
+956
+1979
+1048
+1442
+123
+2179
+2534
+1869
+1051
+1899
+549
+2140
+2864
+2481
+2967
+744
+1116
+2013
+2462
+3014
+517
+1486
+1795
+2585
+321
+1530
+1925
+1767
+1608
+533
+1629
+3018
+392
+920
+1214
+2115
+981
+2316
+15
+112
+2256
+1131
+1002
+2581
+1058
+1631
+1676
+2009
+2563
+689
+1992
+1922
+348
+2478
+3016
+1989
+2936
+2024
+1715
+2214
+2783
+219
+1938
+1643
+917
+1910
+221
+669
+2629
+995
+1295
+1964
+280
+698
+1852
+3158
+1228
+978
+469
+2203
+3181
+987
+2545
+3105
+766
+1278
+80
+85
+318
+1670
+2335
+1977
+979
+1422
+2064
+2930
+223
+1690
+2422
+934
+2654
+2519
+2922
+3043
+33
+193
+583
+2340
+626
+3143
+1821
+2240
+1044
+1098
+1179
+38
+1553
+1501
+3102
+3140
+1716
+229
+457
+2900
+406
+3095
+435
+1416
+2420
+3210
+1137
+1535
+2884
+1207
+2878
+192
+1678
+1902
+2346
+537
+1753
+2452
+1199
+1728
+796
+1046
+443
+1581
+2574
+1065
+1522
+3012
+1164
+1867
+2727
+198
+477
+571
+361
+2503
+2704
+2975
+502
+3179
+1833
+1927
+2359
+1102
+2351
+2756
+423
+1146
+2764
+1532
+155
+2620
+1187
+1757
+1781
+914
+2771
+569
+1687
+464
+2994
+693
+1475
+2246
+2842
+171
+795
+1204
+1618
+176
+2712
+2799
+3074
+2500
+3108
+1252
+1930
+294
+1485
+1663
+2622
+1888
+1900
+2373
+453
+1702
+2317
+1133
+2051
+1417
+3035
+2925
+125
+2607
+1162
+1384
+3006
+1220
+748
+2567
+696
+1647
+1582
+1566
+1965
+2033
+44
+553
+756
+1337
+2692
+3207
+94
+1920
+878
+1559
+2378
+2632
+139
+2606
+2908
+1774
+2331
+236
+535
+2176
+2882
+904
+2238
+3045
+1969
+794
+661
+2114
+310
+473
+1352
+1691
+1875
+53
+1372
+1554
+2347
+554
+194
+2786
+387
+635
+2591
+546
+188
+2148
+2294
+2927
+1401
+2965
+2098
+1302
+3156
+442
+1987
+592
+1758
+2283
+2292
+887
+1874
+337
+830
+1491
+2133
+311
+451
+1528
+2480
+2223
+2403
+1739
+1232
+347
+366
+405
+1688
+3055
+1577
+2000
+758
+380
+752
+1459
+2714
+2555
+1211
+1856
+3001
+1508
+1600
+1926
+162
+738
+159
+2956
+688
+872
+2157
+615
+2082
+1109
+1731
+1238
+1182
+1733
+434
+2360
+670
+1936
+2721
+1462
+2550
+2578
+238
+1649
+3228
+1066
+2570
+2603
+2348
+1206
+438
+2635
+1447
+1607
+1967
+2444
+622
+1229
+2190
+2197
+702
+1008
+2628
+3217
+965
+1282
+2979
+1000
+2509
+266
+1075
+116
+177
+1251
+3041
+388
+883
+1267
+2395
+983
+284
+2434
+2755
+1592
+2840
+856
+1585
+1166
+2377
+1723
+1860
+2218
+1481
+1857
+620
+771
+1087
+3010
+1093
+2436
+364
+971
+2052
+1548
+2919
+306
+1879
+2319
+2605
+1124
+955
+967
+463
+2255
+3184
+2867
+2592
+2687
+325
+1830
+1091
+120
+1110
+1855
+3229
+186
+1213
+1892
+141
+2117
+2425
+2433
+2625
+21
+2644
+1973
+2797
+1409
+2227
+1322
+1404
+2313
+1775
+996
+2151
+2081
+633
+740
+1250
+1640
+2055
+2726
+1156
+1200
+1498
+1340
+1479
+659
+2666
+2754
+300
+408
+309
+638
+3141
+237
+903
+1379
+665
+1878
+2507
+2710
+3103
+528
+1510
+2414
+1017
+1263
+2909
+376
+151
+1183
+1755
+2959
+649
+259
+305
+1434
+2818
+2458
+1025
+1644
+1712
+727
+2883
+658
+724
+1978
+944
+1242
+1273
+2221
+356
+491
+1144
+1240
+801
+1745
+3015
+230
+2022
+3089
+817
+1594
+1730
+570
+1117
+1088
+1421
+2943
+534
+587
+743
+179
+1463
+601
+1013
+1389
+1047
+1108
+2854
+3114
+2442
+2598
+2824
+2912
+2034
+2643
+3047
+2231
+2551
+332
+1578
+181
+419
+1632
+201
+1994
+2016
+2137
+268
+1193
+1932
+1157
+2614
+820
+783
+1009
+2803
+170
+1408
+1993
+2876
+2280
+254
+976
+745
+994
+2851
+4
+1738
+1074
+2477
+1335
+1030
+1570
+2874
+1778
+1247
+1470
+1243
+913
+101
+61
+355
+1085
+1736
+368
+1995
+1662
+952
+3053
+1410
+448
+1669
+2058
+1423
+482
+3026
+1190
+2609
+1539
+374
+1111
+2147
+2415
+49
+628
+1181
+2460
+2525
+2881
+681
+2871
+2445
+1128
+1077
+1119
+1963
+2938
+2326
+650
+2793
+504
+2618
+160
+444
+708
+992
+2241
+3086
+901
+2612
+397
+471
+1824
+330
+1680
+1261
+2596
+299
+1513
+1792
+1001
+1439
+2917
+1310
+990
+1428
+379
+1651
+1842
+2887
+367
+2974
+3036
+1185
+461
+2095
+868
+585
+835
+2611
+1933
+2061
+352
+1502
+705
+1350
+1844
+1246
+1838
+2032
+1129
+1254
+3148
+2913
+432
+910
+1387
+1035
+2511
+1465
+1981
+2047
+2952
+1782
+1637
+2315
+3109
+173
+799
+876
+2180
+394
+1722
+1817
+1947
+1071
+2215
+2235
+1952
+22
+886
+1837
+2828
+2872
+2993
+2010
+1122
+3027
+634
+822
+777
+2408
+3017
+2863
+2996
+1155
+2388
+467
+104
+1189
+431
+2875
+3209
+1355
+3175
+1639
+354
+67
+686
+350
+2948
+3013
+2070
+1054
+1685
+1382
+2933
+108
+2827
+2892
+1351
+1452
+611
+1744
+1749
+126
+2903
+2467
+1811
+454
+2688
+2272
+2914
+459
+3121
+2037
+616
+962
+1318
+946
+2963
+1135
+1949
+2583
+3039
+69
+1202
+425
+1169
+1894
+2524
+1298
+127
+2866
+277
+829
+2984
+663
+1912
+828
+2495
+2531
+1143
+1039
+550
+1139
+2758
+2311
+2375
+168
+924
+1321
+1005
+1802
+2020
+858
+591
+2932
+175
+378
+1850
+2604
+1045
+1053
+1097
+1180
+1787
+1059
+1633
+2473
+2649
+2972
+2969
+1751
+2895
+540
+2076
+1249
+2200
+1052
+574
+1433
+2273
+943
+798
+3090
+733
+1483
+1290
+1948
+1942
+2906
+675
+3120
+242
+1511
+2355
+1707
+648
+484
+1319
+3028
+527
+3030
+2559
+3145
+2613
+726
+422
+2910
+66
+169
+1026
+1868
+2262
+1484
+1557
+1482
+2341
+1917
+95
+1654
+130
+2542
+1429
+1270
+428
+2920
+3151
+3186
+1061
+107
+825
+1159
+760
+1042
+1019
+1234
+1266
+3073
+2286
+437
+1907
+2812
+334
+1103
+847
+2977
+2113
+2719
+2174
+2175
+1574
+1793
+1839
+2579
+2210
+590
+1889
+480
+1022
+3234
+2400
+1138
+496
+507
+2839
+2250
+2587
+389
+695
+481
+2471
+2568
+1448
+2547
+1937
+1550
+373
+2448
+629
+1493
+2019
+3191
+1435
+1713
+2576
+365
+2044
+1285
+1327
+993
+687
+2641
+1188
+596
+1092
+1235
+24
+2877
+1740
+598
+1503
+2043
+832
+1543
+1772
+1876
+2616
+2303
+96
+483
+1225
+2219
+684
+3123
+989
+490
+518
+1106
+3139
+941
+2528
+2985
+1118
+1825
+1149
+1891
+1908
+980
+1488
+87
+511
+1244
+651
+1882
+2023
+2455
+1198
+608
+1037
+1896
+1145
+245
+984
+1172
+2116
+420
+593
+843
+2271
+2848
+2039
+1152
+2499
+226
+328
+386
+1397
+746
+958
+1366
+2306
+110
+1346
+2901
+1096
+704
+1858
+2530
+2929
+2619
+2132
+2831
+2995
+1881
+370
+470
+582
+2368
+1576
+536
+1163
+1810
+1859
+2898
+500
+785
+1134
+2490
+1458
+2325
+603
+1313
+2109
+488
+2374
+2894
+1984
+2599
+911
+1464
+2465
+449
+927
+2502
+963
+3202
+2860
+1300
+1221
+2387
+600
+3110
+1356
+312
+2060
+381
+2096
+2295
+114
+2893
+2861
+697
+2837
+531
+899
+2514
+1365
+2146
+1968
+1115
+458
+2035
+2668
+557
+1473
+272
+447
+418
+503
+2713
+1271
+1343
+2150
+861
+3203
+871
+1534
+2304
+2508
+3118
+401
+3136
+2703
+383
+1142
+1563
+1099
+2409
+865
+1374
+2233
+2538
+2889
+2565
+1184
+2430
+2389
+1140
+1215
+1901
+167
+430
+2790
+1771
+1828
+1524
+2830
+2833
+543
+2161
+1079
+2040
+2289
+2099
+115
+140
+761
+413
+2627
+3049
+2030
+2050
+3115
+1861
+2394
+918
+323
+157
+3236
+345
+1132
+31
+2890
+2094
+515
+1264
+1127
+1412
+816
+1406
+1835
+1741
+1883
+2479
+1806
+2832
+823
+1635
+2483
+1258
+191
+1141
+2474
+833
+1477
+1176
+3144
+1913
+1154
+489
+228
+2362
+1080
+2517
+2834
+2957
+682
+2506
+1840
+414
+2350
+163
+657
+1114
+1194
+2080
+2838
+1269
+1487
+3084
+877
+1226
+1195
+1591
+1975
+2463
+1413
+2014
+1655
+2243
+138
+1719
+2322
+1003
+2488
+2964
+2446
+678
+964
+2997
+109
+3068
+656
+2323
+1259
+113
+1809
+864
+526
+2492
+1171
+1255
+1955
+2915
+2862
+819
+1759
+403
+1276
+1173
+641
+2192
+575
+810
+807
+133
+1174
+875
+1568
+2031
+111
+2475
+1438
+2106
+472
+1177
+636
+667
+1537
+1816
+2858
+2421
+1790
+1549
+610
+2835
+2546
+2049
+2870
+852
+1597
+2880
+18
+2911
+2059
+1168
+602
+1161
+926
+1160
+2885
+3171
+846
+2222
+362
+1262
+2504
+440
+1786
+2675
+3218
+2028
+1186
+1752
+2610
+2268
+476
+2886
+2891
+909
+1826
+2859
+860
+1317
+97
+2586
+2021
+1057
+2124
+2327
+837
+2774
+2025
+850
+2847
+1813
+1721
+1344
+178
+642
+1178
+884
+1100
+2855
+1167
+524
+885
+736
+1391
+153
+1375
+417
+343
+298
+2330
+3107
+1104
+1311
+2907
+106
+1720
+372
+1191
+2427
+218
+1571
+2658
+132
+2449
+1192
+2383
+1349
+821
+1105
+898
+3060
+1546
+2845
+1970
+666
+2069
+867
+2747
+1689
+1049
+2868
+2821
+960
+1454
+1780
+1348
+1158
+2496
+2595
+803
+1956
+404
+156
+395
+2228
+100
+977
+640
+1339
+840
+680
+2079
+1769
+2027
+1230
+2158
+836
+2136
+1467
+1515
+2850
+2202
+2159
+3024
+508
+1492
+654
+897
+1284
+27
+2046
+3161
+2423
+2873
+1072
+2329
+1196
+3052
+896
+2905
+879
+2843
+1150
+1175
+2091
+497
+2017
+402
+2780
+2431
+2822
+1068
+390
+3106
+2823
+2846
+357
+3081
+3216
+433
+1934
+845
+2857
+915
+776
+988
+811
+2810
+1451
+547
+2172
+581
+2071
+2004
+314
+855
+2869
+2651
+1496
+863
+2418
+3176
+2216
+2491
+2852
+2888
+893
+2826
+2841
+1024
+1466
+315
+1328
+806
+812
+804
+1055
+891
+1345
+1411
+1381
+848
+814
+1216
+2916
+1765
+1921
+23
+859
+815
+2045
+3208
+2631
+895
+2849
+874
+809
+1474
+824
+2263
+2435
+2144
+805
+882
+2921
+873
+2086
+152
+2118
+1460
+892
+1960
+2825
+1803
+2291
+1601
+1675
+889
+1747
+1593
+1779
+2257
+3165
+673
+802
+1525
+870
+2904
+190

mse_deepeval_dataset.py

@@ -0,0 +1,197 @@
+import argparse
+import jsonlines
+import json
+# from deepeval.scorer import Scorer
+from deepeval.models import OllamaModel
+from deepeval.metrics import (
+  ContextualRelevancyMetric,
+  ContextualRecallMetric,
+  ContextualPrecisionMetric,
+  AnswerRelevancyMetric,
+  FaithfulnessMetric
+)
+
+# import docx
+
+
+from deepeval.test_case import LLMTestCase
+from deepeval.dataset import EvaluationDataset, Golden
+
+from deepeval import evaluate
+from deepeval.models import OllamaModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from Llemma_Finetuned import Llemma_Finetuned
+import ollama
+
+#ollama run Hudson/llemma:7b
+#deepeval set-ollama Hudson/llemma:7b
+
+if __name__=="__main__":
+    # Initialize parser
+    parser = argparse.ArgumentParser()
+
+    # Adding optional argument
+    parser.add_argument("-n", "--num", help = "Number of test cases to use")
+    parser.add_argument("-s", "--shot", help = "n-shot inference examples")
+    parser.add_argument("-d", "--dataset", help = "Path to test case dataset")
+
+    # Read arguments from command line
+    args = parser.parse_args()
+    test_case_num = int(args.num)
+    num_shot = int(args.shot)
+    dataset_name = str(args.dataset)
+
+    
+
+    # orig
+    # model = ollama.pull(model="Hudson/llemma:7b")
+    #OllamaModel(model="Hudson/llemma:7b")
+
+    # finetuned
+    # llemma_model = AutoModelForCausalLM.from_pretrained("./train_llemma/merged_models/llemma_lora_merged")
+    # tokenizer = AutoTokenizer.from_pretrained("./train_llemma/merged_models/llemma_lora_merged")
+    # model = Llemma_Finetuned(model=llemma_model, tokenizer=tokenizer)
+
+    sorted_rows = []
+    with open('dataset_row_stl.txt', 'r') as file:
+        sorted_rows = file.readlines()
+    # print(sorted_rows)
+    sorted_rows = sorted_rows[0:num_shot]
+    sorted_rows = [int(x) for x in sorted_rows]
+
+    print("Read in sorted rows.")
+
+    examples = "Here are " + str(num_shot) + " examples of math questions (Q) with given answers (A).\n"
+    with jsonlines.open("mse_text_img_QA_ds_test.jsonl", mode='r') as fp:
+        #with open("mse_text_img_QA_ds_test.jsonl", mode='r') as fp:
+        n = 0
+        for j, data in enumerate(fp):
+            if j + 1 in sorted_rows:
+                print("Num shot row " + str(j + 1))
+                # data = json.loads(line)
+                examples += "Q: " + data["body"] + "\n\n"
+                is_accepted = False
+                best_score = float('-inf')
+                output_text = ""
+                for i in range(len(data["answers"])):
+                    if bool(data["answers"][i]["accepted"]) == True:
+                        if is_accepted == False:
+                            is_accepted = True
+                            best_score = int(data["answers"][i]["score"])
+                            output_text = data["answers"][i]["body"]
+                        elif int(data["answers"][i]["score"]) > best_score:
+                            best_score = int(data["answers"][i]["score"])
+                            output_text = data["answers"][i]["body"]
+                    elif int(data["answers"][i]["score"]) > best_score:
+                        best_score = int(data["answers"][i]["score"])
+                        output_text = data["answers"][i]["body"]
+                examples += "A: " + output_text + "\n\n"
+                if n == (num_shot - 1):
+                    examples += "Provide an answer (A) to the following math question (Q) in a similar manner to the previous example(s) given.\n\nQ: "
+                # 26th line
+                n += 1
+            elif n >= num_shot:
+                break
+            else:
+                continue
+    
+    print("Generated examples for", str(num_shot), "shot.")
+
+    mse_dataset = []
+    with jsonlines.open("mse_text_img_QA_ds_test.jsonl", mode='r') as reader:
+
+        count = 0
+        
+        curr_row = 0
+        for row in reader.iter(type=dict, skip_invalid=True):
+            curr_row += 1
+            if curr_row == 33 or curr_row == 36 or curr_row == 69 \
+                or curr_row == 24 or curr_row == 76 \
+                or curr_row == 66 or curr_row == 9 \
+                or curr_row == 26 or curr_row == 27 \
+                or curr_row == 37 or curr_row == 55 \
+                or curr_row == 54 or curr_row == 138 \
+                or curr_row == 77 or curr_row == 84 or curr_row == 87 \
+                or curr_row == 80 or curr_row == 81 or curr_row == 97 \
+                or curr_row == 115 or curr_row == 106:
+                print("Skipped row " + str(curr_row))
+                continue
+            elif curr_row in sorted_rows:
+                print("Skipped row " + str(curr_row) + " because it is a shorter example")
+                continue
+        # question_path = "output/" + row["id"]
+            # if count ual<= 0:
+            #     print(obj)
+            if count >= test_case_num:
+                break
+            else:
+                input_text = row["body"]
+                # response = ollama.generate(model='Hudson/llemma:7b', prompt=input_text)
+                # actual_response = response['response']
+                is_accepted = False
+                best_score = float('-inf')
+                output_text = ""
+                # context = []
+                next_best_answer = ""
+                for i in range(len(row["answers"])):
+                    if bool(row["answers"][i]["accepted"]) == True:
+                        if is_accepted == False:
+                            is_accepted = True
+                            next_best_answer = output_text
+                            best_score = int(row["answers"][i]["score"])
+                            output_text = row["answers"][i]["body"]
+                        elif int(row["answers"][i]["score"]) > best_score:
+                            next_best_answer = output_text
+                            best_score = int(row["answers"][i]["score"])
+                            output_text = row["answers"][i]["body"]
+                        # else:
+                            # context.append(row["answers"][i]["body"])
+                    elif int(row["answers"][i]["score"]) > best_score:
+                        next_best_answer = output_text
+                        best_score = int(row["answers"][i]["score"])
+                        output_text = row["answers"][i]["body"]
+                    # else:
+                    #     context.append(row["answers"][i]["body"])
+                if next_best_answer == "" or next_best_answer is None:
+                    next_best_answer = row["title"]
+                # test_case_dataset.append(LLMTestCase(input=input_text, actual_output=actual_response, expected_output=output_text, retrieval_context=None))
+                # test_case_dataset.append(LLMTestCase(input=input_text, actual_output=model.generate(input_text), expected_output=output_text, retrieval_context=context))
+                if num_shot == 0:
+                    i_text = json.dumps(input_text)
+                    e_output = json.dumps(output_text)
+                    r_context = json.dumps(next_best_answer)
+                    gen_answer = ollama.generate(model="Hudson/llemma:7b", prompt=i_text)
+                    a_output = json.dumps(gen_answer.response)
+                    # print("i_text = ", i_text)
+                    # print("a_output = ", a_output)
+                    # print("e_output = ", e_output)
+                    # print("r_context = ", r_context)
+                    # r_context = gen_answer.context
+                    # if is_invalid_length(i_text) or is_invalid_length(e_output) or is_invalid_length(r_context):
+                    #     continue
+                    mse_dataset.append(LLMTestCase(input=i_text, actual_output=a_output, expected_output=e_output, retrieval_context=[r_context]))
+                else:
+                    i_text = json.dumps(examples + input_text)
+                    e_output = json.dumps(output_text)
+                    r_context = json.dumps(next_best_answer)
+                    gen_answer = ollama.generate(model="Hudson/llemma:7b", prompt=i_text)
+                    a_output = json.dumps(gen_answer.response)
+                    # r_context = gen_answer.context
+                    # print("i_text = ", i_text)
+                    # print("a_output = ", a_output)
+                    # print("e_output = ", e_output)
+                    # print("r_context = ", r_context)
+                    # if is_invalid_length(i_text) or is_invalid_length(e_output) or is_invalid_length(r_context):
+                    #     continue
+                    mse_dataset.append(LLMTestCase(input=i_text, actual_output=a_output, expected_output=e_output, retrieval_context=[r_context]))
+                count = count + 1
+                # if curr_row % 1 == 0:
+                print("At", str(count), "out of", str(test_case_num), " current row =", str(curr_row))
+
+    # first_test_case = LLMTestCase(input="...", actual_output="...", context=["..."])
+    # second_test_case = LLMTestCase(input="...", actual_output="...", context=["..."])
+
+
+    dataset = EvaluationDataset(test_cases=mse_dataset)
+    dataset.save_as(file_type="json", directory="./deepeval-test-dataset", file_name=dataset_name, include_test_cases=True)
+    

mse_ollama_run.py

@@ -0,0 +1,268 @@
+import argparse
+import jsonlines
+import json
+# from deepeval.scorer import Scorer
+from deepeval.models import OllamaModel
+from deepeval.metrics import (
+  ContextualRelevancyMetric,
+  ContextualRecallMetric,
+  ContextualPrecisionMetric,
+  AnswerRelevancyMetric,
+  FaithfulnessMetric
+)
+
+# import docx
+
+
+from deepeval.test_case import LLMTestCase
+from deepeval.dataset import EvaluationDataset, Golden
+
+from deepeval import evaluate
+from deepeval.models import OllamaModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from Llemma_Finetuned import Llemma_Finetuned
+import ollama
+
+#ollama run Hudson/llemma:7b
+#deepeval set-ollama Hudson/llemma:7b
+
+def is_invalid_length(text, length=4096):
+    if len(text) <= length:
+        return False
+    else:
+        return True
+
+if __name__=="__main__":
+    # Initialize parser
+    parser = argparse.ArgumentParser()
+
+    # Adding optional argument
+    parser.add_argument("-t", "--test", help = "Test to run (ar, cp, crec, crel, f)")
+    parser.add_argument("-d", "--dataset", help = "Path to test case dataset")
+
+    # Read arguments from command line
+    args = parser.parse_args()
+    test_type = str(args.test)
+    test_data = str(args.dataset)
+    dataset = EvaluationDataset()
+
+    # Add as test cases
+    dataset.add_test_cases_from_json_file(
+        # file_path is the absolute path to you .json file
+        file_path=test_data,
+        input_key_name="input",
+        actual_output_key_name="actual_output",
+        expected_output_key_name="expected_output",
+        context_key_name="context",
+        retrieval_context_key_name="retrieval_context",
+    )
+    dataset.test_cases = dataset.test_cases[0:100]
+
+    # orig
+    # model = ollama.pull(model="Hudson/llemma:7b")
+    #OllamaModel(model="Hudson/llemma:7b")
+    
+    ollama_model = OllamaModel(model="Hudson/llemma:7b")
+
+    # finetuned
+    # llemma_model = AutoModelForCausalLM.from_pretrained("./train_llemma/merged_models/llemma_lora_merged")
+    # tokenizer = AutoTokenizer.from_pretrained("./train_llemma/merged_models/llemma_lora_merged")
+    # model = Llemma_Finetuned(model=llemma_model, tokenizer=tokenizer)
+
+    # sorted_rows = []
+    # with open('dataset_row_stl.txt', 'r') as file:
+    #     sorted_rows = file.readlines()
+    # # print(sorted_rows)
+    # sorted_rows = sorted_rows[0:num_shot]
+    # sorted_rows = [int(x) for x in sorted_rows]
+
+    # print("Read in sorted rows.")
+
+    # examples = "Here are " + str(num_shot) + " examples of math questions (Q) with given answers (A).\n"
+    # with jsonlines.open("mse_text_img_QA_ds_test.jsonl", mode='r') as fp:
+    #     #with open("mse_text_img_QA_ds_test.jsonl", mode='r') as fp:
+    #     n = 0
+    #     for j, data in enumerate(fp):
+    #         if j + 1 in sorted_rows:
+    #             print("Num shot row " + str(j + 1))
+    #             # data = json.loads(line)
+    #             examples += "Q: " + data["body"] + "\n\n"
+    #             is_accepted = False
+    #             best_score = float('-inf')
+    #             output_text = ""
+    #             for i in range(len(data["answers"])):
+    #                 if bool(data["answers"][i]["accepted"]) == True:
+    #                     if is_accepted == False:
+    #                         is_accepted = True
+    #                         best_score = int(data["answers"][i]["score"])
+    #                         output_text = data["answers"][i]["body"]
+    #                     elif int(data["answers"][i]["score"]) > best_score:
+    #                         best_score = int(data["answers"][i]["score"])
+    #                         output_text = data["answers"][i]["body"]
+    #                 elif int(data["answers"][i]["score"]) > best_score:
+    #                     best_score = int(data["answers"][i]["score"])
+    #                     output_text = data["answers"][i]["body"]
+    #             examples += "A: " + output_text + "\n\n"
+    #             if n == (num_shot - 1):
+    #                 examples += "Provide an answer (A) to the following math question (Q) in a similar manner to the previous example(s) given.\n\nQ: "
+    #             # 26th line
+    #             n += 1
+    #         elif n >= num_shot:
+    #             break
+    #         else:
+    #             continue
+    
+    # print("Generated examples for", str(num_shot), "shot.")
+
+    # mse_dataset = []
+    # with jsonlines.open("mse_text_img_QA_ds_test.jsonl", mode='r') as reader:
+
+    #     count = 0
+        
+    #     curr_row = 0
+    #     for row in reader.iter(type=dict, skip_invalid=True):
+    #         curr_row += 1
+    #         if curr_row <= skip_to:
+    #             continue
+    #         elif curr_row == 33 or curr_row == 36 or curr_row == 69 \
+    #             or curr_row == 24 or curr_row == 76 \
+    #             or curr_row == 66 or curr_row == 9 \
+    #             or curr_row == 26 or curr_row == 27 \
+    #             or curr_row == 37 or curr_row == 55 \
+    #             or curr_row == 54 or curr_row == 138 \
+    #             or curr_row == 77 or curr_row == 84 or curr_row == 87 \
+    #             or curr_row == 80 or curr_row == 81 or curr_row == 97 \
+    #             or curr_row == 115 or curr_row == 106:
+    #             print("Skipped row " + str(curr_row))
+    #             continue
+    #         elif curr_row in sorted_rows:
+    #             print("Skipped row " + str(curr_row) + " because it is a shorter example")
+    #             continue
+    #     # question_path = "output/" + row["id"]
+    #         # if count ual<= 0:
+    #         #     print(obj)
+    #         if count >= test_case_num:
+    #             break
+    #         else:
+    #             input_text = row["body"]
+    #             # response = ollama.generate(model='Hudson/llemma:7b', prompt=input_text)
+    #             # actual_response = response['response']
+    #             is_accepted = False
+    #             best_score = float('-inf')
+    #             output_text = ""
+    #             # context = []
+    #             next_best_answer = ""
+    #             for i in range(len(row["answers"])):
+    #                 if bool(row["answers"][i]["accepted"]) == True:
+    #                     if is_accepted == False:
+    #                         is_accepted = True
+    #                         next_best_answer = output_text
+    #                         best_score = int(row["answers"][i]["score"])
+    #                         output_text = row["answers"][i]["body"]
+    #                     elif int(row["answers"][i]["score"]) > best_score:
+    #                         next_best_answer = output_text
+    #                         best_score = int(row["answers"][i]["score"])
+    #                         output_text = row["answers"][i]["body"]
+    #                     # else:
+    #                         # context.append(row["answers"][i]["body"])
+    #                 elif int(row["answers"][i]["score"]) > best_score:
+    #                     next_best_answer = output_text
+    #                     best_score = int(row["answers"][i]["score"])
+    #                     output_text = row["answers"][i]["body"]
+    #                 # else:
+    #                 #     context.append(row["answers"][i]["body"])
+    #             if next_best_answer == "" or next_best_answer is None:
+    #                 next_best_answer = row["title"]
+    #             # test_case_dataset.append(LLMTestCase(input=input_text, actual_output=actual_response, expected_output=output_text, retrieval_context=None))
+    #             # test_case_dataset.append(LLMTestCase(input=input_text, actual_output=model.generate(input_text), expected_output=output_text, retrieval_context=context))
+    #             if num_shot == 0:
+    #                 i_text = json.dumps(input_text)
+    #                 e_output = json.dumps(output_text)
+    #                 r_context = json.dumps(next_best_answer)
+    #                 gen_answer = ollama.generate(model="Hudson/llemma:7b", prompt=i_text)
+    #                 a_output = json.dumps(gen_answer.response)
+    #                 # print("i_text = ", i_text)
+    #                 # print("a_output = ", a_output)
+    #                 # print("e_output = ", e_output)
+    #                 # print("r_context = ", r_context)
+    #                 # r_context = gen_answer.context
+    #                 # if is_invalid_length(i_text) or is_invalid_length(e_output) or is_invalid_length(r_context):
+    #                 #     continue
+    #                 mse_dataset.append(LLMTestCase(input=i_text, actual_output=a_output, expected_output=e_output, retrieval_context=[r_context]))
+    #             else:
+    #                 i_text = json.dumps(examples + input_text)
+    #                 e_output = json.dumps(output_text)
+    #                 r_context = json.dumps(next_best_answer)
+    #                 gen_answer = ollama.generate(model="Hudson/llemma:7b", prompt=i_text)
+    #                 a_output = json.dumps(gen_answer.response)
+    #                 # r_context = gen_answer.context
+    #                 # print("i_text = ", i_text)
+    #                 # print("a_output = ", a_output)
+    #                 # print("e_output = ", e_output)
+    #                 # print("r_context = ", r_context)
+    #                 # if is_invalid_length(i_text) or is_invalid_length(e_output) or is_invalid_length(r_context):
+    #                 #     continue
+    #                 mse_dataset.append(LLMTestCase(input=i_text, actual_output=a_output, expected_output=e_output, retrieval_context=[r_context]))
+    #             count = count + 1
+    #             # if curr_row % 1 == 0:
+    #             print("At", str(count), "out of", str(test_case_num), " current row =", str(curr_row))
+
+    # first_test_case = LLMTestCase(input="...", actual_output="...", context=["..."])
+    # second_test_case = LLMTestCase(input="...", actual_output="...", context=["..."])
+
+
+    # dataset = EvaluationDataset(test_cases=mse_dataset)
+    pass_threshold = 0.7
+
+    # eval_output = ""
+
+    if test_type == "ar":
+        # answer_relevancy = AnswerRelevancyMetric(model=model, threshold=pass_threshold, async_mode=False)
+        answer_relevancy = AnswerRelevancyMetric(model=ollama_model, threshold=pass_threshold, async_mode=False)
+        # evaluate(dataset, metrics=[answer_relevancy], out_file=out_path, run_async=True)
+        # with open(out_path, "a") as f:
+        #     # f.write(dataset.evaluate([answer_relevancy]))
+        # eval_output = dataset.evaluate([answer_relevancy])
+        # evaluate(goldens=dataset.goldens, metrics=[answer_relevancy])
+        evaluate(dataset, metrics=[answer_relevancy])
+    elif test_type == "cp":
+        contextual_precision = ContextualPrecisionMetric(model=ollama_model, threshold=pass_threshold, async_mode=False)
+        # evaluate(dataset, metrics=[contextual_precision], out_file=out_path, run_async=True)
+        # evaluate(dataset, metrics=[contextual_precision])
+        # eval_output = dataset.evaluate([contextual_precision])
+        # evaluate(goldens=dataset.goldens, metrics=[contextual_precision])
+        evaluate(dataset, metrics=[contextual_precision])
+    elif test_type == "crec":
+        contextual_recall = ContextualRecallMetric(model=ollama_model, threshold=pass_threshold, async_mode=False)
+        # evaluate(dataset, metrics=[contextual_recall], out_file=out_path, run_async=True)
+        # evaluate(dataset, metrics=[contextual_recall])
+        # eval_output = dataset.evaluate([contextual_recall])
+        # evaluate(goldens=dataset.goldens, metrics=[contextual_recall])
+        evaluate(dataset, metrics=[contextual_recall])
+    else:
+        print("Test case (" + test_type + ") not covered")
+    
+    # with open(out_path, "a") as f:
+    #     f.write(str(eval_output))
+
+    # Create a document
+    # doc = docx.Document()
+
+    # # Add a paragraph to the document
+    # p = doc.add_paragraph()
+
+    # # Add some formatting to the paragraph
+    # p.paragraph_format.line_spacing = 1
+    # p.paragraph_format.space_after = 0
+
+    # # Add a run to the paragraph
+    # run = p.add_run(eval_output)
+
+    # # Add some formatting to the run
+    # run.bold = False
+    # run.italic = False
+    # run.font.name = 'Arial'
+    # run.font.size = docx.shared.Pt(12)
+
+    # # Save the document
+    # doc.save(out_path)

mse_ollama_run_ft.py

@@ -0,0 +1,315 @@
+import argparse
+import jsonlines
+import json
+# from deepeval.scorer import Scorer
+from deepeval.models import OllamaModel
+from deepeval.metrics import (
+  ContextualRelevancyMetric,
+  ContextualRecallMetric,
+  ContextualPrecisionMetric,
+  AnswerRelevancyMetric,
+  FaithfulnessMetric
+)
+
+# import docx
+
+
+from deepeval.test_case import LLMTestCase
+from deepeval.dataset import EvaluationDataset, Golden
+
+from deepeval import evaluate
+from deepeval.models import OllamaModel
+import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from Llemma_Finetuned import Llemma_Finetuned
+import ollama
+import torch
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from deepeval.models import DeepEvalBaseLLM
+
+
+class CustomLlemma(DeepEvalBaseLLM):
+    def __init__(self):
+        self.torch_device = "cuda" if torch.cuda.is_available() else "cpu"
+
+        # finetuned
+        model = AutoModelForCausalLM.from_pretrained("./merged_models/llemma_lora_merged").to(self.torch_device)
+        tokenizer = AutoTokenizer.from_pretrained("./merged_models/llemma_lora_merged")
+        
+        self.model = model
+        self.tokenizer = tokenizer
+
+    def load_model(self):
+        return self.model
+
+    def generate(self, prompt: str) -> str:
+        model = self.load_model()
+        pipeline = transformers.pipeline(
+            "text-generation",
+            model=model,
+            tokenizer=self.tokenizer,
+            framework="pt",
+            device=0,
+            max_length=4096,
+            eos_token_id=self.tokenizer.eos_token_id,
+            pad_token_id=self.tokenizer.eos_token_id,
+        )
+
+        return pipeline(prompt)
+        
+
+        # inputs = self.tokenizer(prompt, return_tensors='pt').to(self.torch_device)
+        # output = self.model.generate(**inputs)
+        # a_output = self.tokenizer.decode(output[0])
+
+        # return json.dumps(a_output)
+
+    async def a_generate(self, prompt: str) -> str:
+        return self.generate(prompt)
+
+    def get_model_name(self):
+        return "Llemma Fine-tuned"
+
+#ollama run Hudson/llemma:7b
+#deepeval set-ollama Hudson/llemma:7b
+
+def is_invalid_length(text, length=4096):
+    if len(text) <= length:
+        return False
+    else:
+        return True
+
+if __name__=="__main__":
+    # Initialize parser
+    parser = argparse.ArgumentParser()
+
+    # Adding optional argument
+    parser.add_argument("-t", "--test", help = "Test to run (ar, cp, crec, crel, f)")
+    parser.add_argument("-d", "--dataset", help = "Path to test case dataset")
+
+    # Read arguments from command line
+    args = parser.parse_args()
+    test_type = str(args.test)
+    test_data = str(args.dataset)
+    dataset = EvaluationDataset()
+
+    # Add as test cases
+    dataset.add_test_cases_from_json_file(
+        # file_path is the absolute path to you .json file
+        file_path=test_data,
+        input_key_name="input",
+        actual_output_key_name="actual_output",
+        expected_output_key_name="expected_output",
+        context_key_name="context",
+        retrieval_context_key_name="retrieval_context",
+    )
+
+    # orig
+    # model = ollama.pull(model="Hudson/llemma:7b")
+    #OllamaModel(model="Hudson/llemma:7b")
+    custom_llm = CustomLlemma()
+
+    # finetuned
+    # llemma_model = AutoModelForCausalLM.from_pretrained("./train_llemma/merged_models/llemma_lora_merged")
+    # tokenizer = AutoTokenizer.from_pretrained("./train_llemma/merged_models/llemma_lora_merged")
+    # model = Llemma_Finetuned(model=llemma_model, tokenizer=tokenizer)
+
+    # sorted_rows = []
+    # with open('dataset_row_stl.txt', 'r') as file:
+    #     sorted_rows = file.readlines()
+    # # print(sorted_rows)
+    # sorted_rows = sorted_rows[0:num_shot]
+    # sorted_rows = [int(x) for x in sorted_rows]
+
+    # print("Read in sorted rows.")
+
+    # examples = "Here are " + str(num_shot) + " examples of math questions (Q) with given answers (A).\n"
+    # with jsonlines.open("mse_text_img_QA_ds_test.jsonl", mode='r') as fp:
+    #     #with open("mse_text_img_QA_ds_test.jsonl", mode='r') as fp:
+    #     n = 0
+    #     for j, data in enumerate(fp):
+    #         if j + 1 in sorted_rows:
+    #             print("Num shot row " + str(j + 1))
+    #             # data = json.loads(line)
+    #             examples += "Q: " + data["body"] + "\n\n"
+    #             is_accepted = False
+    #             best_score = float('-inf')
+    #             output_text = ""
+    #             for i in range(len(data["answers"])):
+    #                 if bool(data["answers"][i]["accepted"]) == True:
+    #                     if is_accepted == False:
+    #                         is_accepted = True
+    #                         best_score = int(data["answers"][i]["score"])
+    #                         output_text = data["answers"][i]["body"]
+    #                     elif int(data["answers"][i]["score"]) > best_score:
+    #                         best_score = int(data["answers"][i]["score"])
+    #                         output_text = data["answers"][i]["body"]
+    #                 elif int(data["answers"][i]["score"]) > best_score:
+    #                     best_score = int(data["answers"][i]["score"])
+    #                     output_text = data["answers"][i]["body"]
+    #             examples += "A: " + output_text + "\n\n"
+    #             if n == (num_shot - 1):
+    #                 examples += "Provide an answer (A) to the following math question (Q) in a similar manner to the previous example(s) given.\n\nQ: "
+    #             # 26th line
+    #             n += 1
+    #         elif n >= num_shot:
+    #             break
+    #         else:
+    #             continue
+    
+    # print("Generated examples for", str(num_shot), "shot.")
+
+    # mse_dataset = []
+    # with jsonlines.open("mse_text_img_QA_ds_test.jsonl", mode='r') as reader:
+
+    #     count = 0
+        
+    #     curr_row = 0
+    #     for row in reader.iter(type=dict, skip_invalid=True):
+    #         curr_row += 1
+    #         if curr_row <= skip_to:
+    #             continue
+    #         elif curr_row == 33 or curr_row == 36 or curr_row == 69 \
+    #             or curr_row == 24 or curr_row == 76 \
+    #             or curr_row == 66 or curr_row == 9 \
+    #             or curr_row == 26 or curr_row == 27 \
+    #             or curr_row == 37 or curr_row == 55 \
+    #             or curr_row == 54 or curr_row == 138 \
+    #             or curr_row == 77 or curr_row == 84 or curr_row == 87 \
+    #             or curr_row == 80 or curr_row == 81 or curr_row == 97 \
+    #             or curr_row == 115 or curr_row == 106:
+    #             print("Skipped row " + str(curr_row))
+    #             continue
+    #         elif curr_row in sorted_rows:
+    #             print("Skipped row " + str(curr_row) + " because it is a shorter example")
+    #             continue
+    #     # question_path = "output/" + row["id"]
+    #         # if count ual<= 0:
+    #         #     print(obj)
+    #         if count >= test_case_num:
+    #             break
+    #         else:
+    #             input_text = row["body"]
+    #             # response = ollama.generate(model='Hudson/llemma:7b', prompt=input_text)
+    #             # actual_response = response['response']
+    #             is_accepted = False
+    #             best_score = float('-inf')
+    #             output_text = ""
+    #             # context = []
+    #             next_best_answer = ""
+    #             for i in range(len(row["answers"])):
+    #                 if bool(row["answers"][i]["accepted"]) == True:
+    #                     if is_accepted == False:
+    #                         is_accepted = True
+    #                         next_best_answer = output_text
+    #                         best_score = int(row["answers"][i]["score"])
+    #                         output_text = row["answers"][i]["body"]
+    #                     elif int(row["answers"][i]["score"]) > best_score:
+    #                         next_best_answer = output_text
+    #                         best_score = int(row["answers"][i]["score"])
+    #                         output_text = row["answers"][i]["body"]
+    #                     # else:
+    #                         # context.append(row["answers"][i]["body"])
+    #                 elif int(row["answers"][i]["score"]) > best_score:
+    #                     next_best_answer = output_text
+    #                     best_score = int(row["answers"][i]["score"])
+    #                     output_text = row["answers"][i]["body"]
+    #                 # else:
+    #                 #     context.append(row["answers"][i]["body"])
+    #             if next_best_answer == "" or next_best_answer is None:
+    #                 next_best_answer = row["title"]
+    #             # test_case_dataset.append(LLMTestCase(input=input_text, actual_output=actual_response, expected_output=output_text, retrieval_context=None))
+    #             # test_case_dataset.append(LLMTestCase(input=input_text, actual_output=model.generate(input_text), expected_output=output_text, retrieval_context=context))
+    #             if num_shot == 0:
+    #                 i_text = json.dumps(input_text)
+    #                 e_output = json.dumps(output_text)
+    #                 r_context = json.dumps(next_best_answer)
+    #                 gen_answer = ollama.generate(model="Hudson/llemma:7b", prompt=i_text)
+    #                 a_output = json.dumps(gen_answer.response)
+    #                 # print("i_text = ", i_text)
+    #                 # print("a_output = ", a_output)
+    #                 # print("e_output = ", e_output)
+    #                 # print("r_context = ", r_context)
+    #                 # r_context = gen_answer.context
+    #                 # if is_invalid_length(i_text) or is_invalid_length(e_output) or is_invalid_length(r_context):
+    #                 #     continue
+    #                 mse_dataset.append(LLMTestCase(input=i_text, actual_output=a_output, expected_output=e_output, retrieval_context=[r_context]))
+    #             else:
+    #                 i_text = json.dumps(examples + input_text)
+    #                 e_output = json.dumps(output_text)
+    #                 r_context = json.dumps(next_best_answer)
+    #                 gen_answer = ollama.generate(model="Hudson/llemma:7b", prompt=i_text)
+    #                 a_output = json.dumps(gen_answer.response)
+    #                 # r_context = gen_answer.context
+    #                 # print("i_text = ", i_text)
+    #                 # print("a_output = ", a_output)
+    #                 # print("e_output = ", e_output)
+    #                 # print("r_context = ", r_context)
+    #                 # if is_invalid_length(i_text) or is_invalid_length(e_output) or is_invalid_length(r_context):
+    #                 #     continue
+    #                 mse_dataset.append(LLMTestCase(input=i_text, actual_output=a_output, expected_output=e_output, retrieval_context=[r_context]))
+    #             count = count + 1
+    #             # if curr_row % 1 == 0:
+    #             print("At", str(count), "out of", str(test_case_num), " current row =", str(curr_row))
+
+    # first_test_case = LLMTestCase(input="...", actual_output="...", context=["..."])
+    # second_test_case = LLMTestCase(input="...", actual_output="...", context=["..."])
+
+
+    # dataset = EvaluationDataset(test_cases=mse_dataset)
+    pass_threshold = 0.7
+
+    # eval_output = ""
+
+    if test_type == "ar":
+        # answer_relevancy = AnswerRelevancyMetric(model=model, threshold=pass_threshold, async_mode=False)
+        answer_relevancy = AnswerRelevancyMetric(model=custom_llm, threshold=pass_threshold)
+        # evaluate(dataset, metrics=[answer_relevancy], out_file=out_path, run_async=True)
+        # with open(out_path, "a") as f:
+        #     # f.write(dataset.evaluate([answer_relevancy]))
+        # eval_output = dataset.evaluate([answer_relevancy])
+        # evaluate(goldens=dataset.goldens, metrics=[answer_relevancy])
+        evaluate(dataset, metrics=[answer_relevancy])
+    elif test_type == "cp":
+        contextual_precision = ContextualPrecisionMetric(model=custom_llm, threshold=pass_threshold)
+        # evaluate(dataset, metrics=[contextual_precision], out_file=out_path, run_async=True)
+        # evaluate(dataset, metrics=[contextual_precision])
+        # eval_output = dataset.evaluate([contextual_precision])
+        # evaluate(goldens=dataset.goldens, metrics=[contextual_precision])
+        evaluate(dataset, metrics=[contextual_precision])
+    elif test_type == "crec":
+        contextual_recall = ContextualRecallMetric(model=custom_llm, threshold=pass_threshold)
+        # evaluate(dataset, metrics=[contextual_recall], out_file=out_path, run_async=True)
+        # evaluate(dataset, metrics=[contextual_recall])
+        # eval_output = dataset.evaluate([contextual_recall])
+        # evaluate(goldens=dataset.goldens, metrics=[contextual_recall])
+        evaluate(dataset, metrics=[contextual_recall])
+    else:
+        print("Test case (" + test_type + ") not covered")
+    
+    # with open(out_path, "a") as f:
+    #     f.write(str(eval_output))
+
+    # Create a document
+    # doc = docx.Document()
+
+    # # Add a paragraph to the document
+    # p = doc.add_paragraph()
+
+    # # Add some formatting to the paragraph
+    # p.paragraph_format.line_spacing = 1
+    # p.paragraph_format.space_after = 0
+
+    # # Add a run to the paragraph
+    # run = p.add_run(eval_output)
+
+    # # Add some formatting to the run
+    # run.bold = False
+    # run.italic = False
+    # run.font.name = 'Arial'
+    # run.font.size = docx.shared.Pt(12)
+
+    # # Save the document
+    # doc.save(out_path)

mse_ollama_timer.py

@@ -0,0 +1,17 @@
+import jsonlines
+import json
+# import docx
+# from deepeval.models import OllamaModel
+import ollama
+import time
+#ollama run Hudson/llemma:7b
+#deepeval set-ollama Hudson/llemma:7b
+
+if __name__=="__main__":
+    # Initialize parser
+
+    tic = time.perf_counter()
+    ollama.generate(model="Hudson/llemma:7b", prompt="I need to determine the radius of convergence of the series $\\sum_{n=1}^\\infty a_nx^n$, where $a_n=a^n+b^n$ and $a,b$ are real numbers.")
+    # print(response)
+    toc = time.perf_counter()
+    print(f"Ran Llemma in {toc - tic:0.4f} seconds")

process_mse_data.sh

@@ -0,0 +1,137 @@
+#!/bin/bash
+
+#SBATCH --time=1:00:00   # walltime.  hours:minutes:seconds
+#SBATCH --ntasks=8   # number of processor cores (i.e. tasks)
+#SBATCH --nodes=1   # number of nodes
+#SBATCH --gpus=1
+#SBATCH --mem=80G   # 164G memory per CPU core
+#SBATCH --mail-user=aw742@byu.edu   # email address
+#SBATCH --mail-type=BEGIN
+#SBATCH --mail-type=END
+#SBATCH --mail-type=FAIL
+#SBATCH --qos=cs
+#SBATCH --partition=cs
+
+# some helpful debugging options
+set -e
+set -u
+
+# LOAD MODULES, INSERT CODE, AND RUN YOUR PROGRAMS HERE
+# module load python/3.11
+
+source ./mse_env/Scripts/activate
+
+# json config = "max_samples": 500,
+
+# python mse_text_img_process.py
+# python convert_mse.py
+
+# pip install jsonlines
+# pip install deepeval
+
+NUM_TEST_CASES=100
+
+# python mse_ollama_run.py --num $NUM_TEST_CASES --test f --shot 0 --out_file metric_test_orig_100_f.txt
+# echo "Test case faithfulness finished"
+
+NUM_SHOT=0
+
+# set DEEPEVAL_RESULTS_FOLDER=.\data
+
+python mse_ollama_timer.py
+echo "Test time calculated"
+
+# deepeval set-local-model --model-name Hudson/llemma:7b
+# ollama pull Hudson/llemma:7b
+# deepeval set-ollama Hudson/llemma:7b
+
+# export DEEPEVAL_RESULTS_FOLDER="./metric_test_0_shot_100_ar"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test ar --shot $NUM_SHOT #--out_file metric_test_0_shot_100_ar.txt 
+# echo "Test case answer relevancy finished"
+# export DEEPEVAL_RESULTS_FOLDER="./metric_test_0_shot_100_crec"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test crec --shot $NUM_SHOT #--out_file metric_test_0_shot_100_crec.txt
+# echo "Test case contexual recall finished"
+# export DEEPEVAL_RESULTS_FOLDER="./metric_test_0_shot_100_cp"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test cp --shot $NUM_SHOT #--out_file metric_test_0_shot_100_cp.txt
+# echo "Test case contextual precision finished"
+
+NUM_SHOT=1
+
+# export DEEPEVAL_RESULTS_FOLDER="./metric_test_1_shot_100_ar"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test ar --shot $NUM_SHOT #--out_file metric_test_1_shot_100_ar.txt 
+# echo "Test case answer relevancy finished"
+# export DEEPEVAL_RESULTS_FOLDER="./metric_test_1_shot_100_crec"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test crec --shot $NUM_SHOT #--out_file metric_test_1_shot_100_crec.txt
+# echo "Test case contexual recall finished"
+# export DEEPEVAL_RESULTS_FOLDER="./metric_test_1_shot_100_cp"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test cp --shot $NUM_SHOT #--out_file metric_test_1_shot_100_cp.txt
+# echo "Test case contextual precision finished"
+
+
+
+NUM_SHOT=5
+# export DEEPEVAL_RESULTS_FOLDER="./metric_test_5_shot_100_ar"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test ar --shot $NUM_SHOT #--out_file metric_test_5_shot_100_ar.txt 
+# echo "Test case answer relevancy finished"
+# export DEEPEVAL_RESULTS_FOLDER="./metric_test_5_shot_100_crec"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test crec --shot $NUM_SHOT #--out_file metric_test_5_shot_100_crec.txt
+# echo "Test case contexual recall finished"
+# export DEEPEVAL_RESULTS_FOLDER="./metric_test_5_shot_100_cp"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test cp --shot $NUM_SHOT #--out_file metric_test_5_shot_100_cp.txt
+# echo "Test case contextual precision finished"
+
+# python mse_ollama_run.py --num 25 --begin 0 --test cp --shot $NUM_SHOT --out_file metric_test_5_shot_25_cp.txt
+# echo "Test case contextual precision finished"
+
+# python mse_ollama_run.py --num 25 --begin 25 --test cp --shot $NUM_SHOT --out_file metric_test_5_shot_25_b25_cp.txt
+# echo "Test case contextual precision finished (start 25)"
+# python mse_ollama_run.py --num 25 --begin 50 --test cp --shot $NUM_SHOT --out_file metric_test_5_shot_25_b50_cp.txt
+# echo "Test case contextual precision finished (start 50)"
+# python mse_ollama_run.py --num 25 --begin 75 --test cp --shot $NUM_SHOT --out_file metric_test_5_shot_25_b75_cp.txt
+# echo "Test case contextual precision finished (start 75)"
+
+
+NUM_SHOT=10
+# export DEEPEVAL_RESULTS_FOLDER="./metric_test_10_shot_100_ar"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test ar --shot $NUM_SHOT -out_file metric_test_10_shot_100_ar.txt 
+# echo "Test case answer relevancy finished"
+# export DEEPEVAL_RESULTS_FOLDER="./metric_test_10_shot_100_crec"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test crec --shot $NUM_SHOT -out_file metric_test_10_shot_100_crec.txt
+# echo "Test case contexual recall finished"
+# export DEEPEVAL_RESULTS_FOLDER="./metric_test_10_shot_100_cp"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test cp --shot $NUM_SHOT -out_file metric_test_10_shot_100_cp.txt
+# echo "Test case contextual precision finished"
+
+# finetuned
+NUM_SHOT=0
+# export DEEPEVAL_RESULTS_FOLDER="metric_test_ft_100_ar"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test ar --shot $NUM_SHOT #> metric_test_ft_100_ar.txt 
+# echo "Test case answer relevancy finished"
+# export DEEPEVAL_RESULTS_FOLDER="metric_test_ft_100_crec"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test crec --shot $NUM_SHOT #> metric_test_ft_100_crec.txt
+# echo "Test case contexual recall finished"
+# export DEEPEVAL_RESULTS_FOLDER="metric_test_ft_100_cp"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test cp --shot $NUM_SHOT > metric_test_ft_100_cp.txt
+# echo "Test case contextual precision finished"
+
+
+# python mse_ollama_run.py --num $NUM_TEST_CASES --test crel --out_file metric_test_orig_100_crel.txt
+# echo "Test case contextual relevancy finished"
+
+
+# python mse_ollama_run.py --num $NUM_TEST_CASES --test f --out_file metric_test_orig_100_f.txt
+# echo "Test case faithfulness finished"
+
+# python mse_jsonl_resize.py
+
+# python finetune.py
+
+# echo "Original Llemma Model"
+# echo "Processing 0 shot 100 test cases"
+# CUDA_VISIBLE_DEVICES=0 python mse_deepeval_dataset.py --num 100 --shot 0 --dataset mse_llemma_orig_100_case_0_shot
+# echo "Processing 1 shot 100 test cases"
+# CUDA_VISIBLE_DEVICES=0 python mse_deepeval_dataset.py --num 100 --shot 1 --dataset mse_llemma_orig_100_case_1_shot
+# echo "Processing 5 shot 100 test cases"
+# CUDA_VISIBLE_DEVICES=0 python mse_deepeval_dataset.py --num 100 --shot 5 --dataset mse_llemma_orig_100_case_5_shot
+# echo "Processing 10 shot 100 test cases"
+# CUDA_VISIBLE_DEVICES=0 python mse_deepeval_dataset.py --num 100 --shot 10 --dataset mse_llemma_orig_100_case_10_shot

process_mse_data_2.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+
+#SBATCH --time=1:00:00   # walltime.  hours:minutes:seconds
+#SBATCH --ntasks=8   # number of processor cores (i.e. tasks)
+#SBATCH --nodes=1   # number of nodes
+#SBATCH --gpus=1
+#SBATCH --mem=80G   # 164G memory per CPU core
+#SBATCH --mail-user=aw742@byu.edu   # email address
+#SBATCH --mail-type=BEGIN
+#SBATCH --mail-type=END
+#SBATCH --mail-type=FAIL
+#SBATCH --qos=cs
+#SBATCH --partition=cs
+
+# some helpful debugging options
+set -e
+set -u
+
+# LOAD MODULES, INSERT CODE, AND RUN YOUR PROGRAMS HERE
+# module load python/3.11
+
+source ./mse_env/Scripts/activate
+
+# json config = "max_samples": 500,
+
+# python mse_text_img_process.py
+# python convert_mse.py
+
+# pip install jsonlines
+# pip install deepeval
+
+NUM_TEST_CASES=100
+
+# python mse_ollama_run.py --num $NUM_TEST_CASES --test f --shot 0 --out_file metric_test_orig_100_f.txt
+# echo "Test case faithfulness finished"
+
+NUM_SHOT=0
+
+# set DEEPEVAL_RESULTS_FOLDER=.\data
+
+# deepeval set-local-model --model-name Hudson/llemma:7b
+# ollama pull Hudson/llemma:7b
+# deepeval set-ollama Hudson/llemma:7b
+
+# python mse_ollama_run.py --test ar --dataset ./deepeval-test-dataset/mse_llemma_orig_100_case_0_shot.json > metric_test_0_shot_100_ar.txt 
+# echo "Test case answer relevancy finished"
+# python mse_ollama_run.py --test crec --dataset ./deepeval-test-dataset/mse_llemma_orig_100_case_0_shot.json #> metric_test_0_shot_100_crec.txt 
+# echo "Test case contexual recall finished"
+# python mse_ollama_run.py --test cp --dataset ./deepeval-test-dataset/mse_llemma_orig_100_case_0_shot.json > metric_test_0_shot_100_cp.txt
+# echo "Test case contextual precision finished"
+
+NUM_SHOT=1
+
+# python mse_ollama_run.py --test ar --dataset ./deepeval-test-dataset/mse_llemma_orig_100_case_1_shot.json > metric_test_1_shot_100_ar.txt
+# echo "Test case answer relevancy finished"
+# python mse_ollama_run.py --test crec --dataset ./deepeval-test-dataset/mse_llemma_orig_100_case_1_shot.json #> metric_test_1_shot_100_crec.txt
+# echo "Test case contexual recall finished"
+# python mse_ollama_run.py --test cp --dataset ./deepeval-test-dataset/mse_llemma_orig_100_case_1_shot.json #> metric_test_1_shot_100_cp.txt
+# echo "Test case contextual precision finished"
+
+
+
+NUM_SHOT=5
+# python mse_ollama_run.py --test ar --dataset ./deepeval-test-dataset/mse_llemma_orig_100_case_5_shot.json > metric_test_5_shot_100_ar.txt 
+# echo "Test case answer relevancy finished"
+# python mse_ollama_run.py --test crec --dataset ./deepeval-test-dataset/mse_llemma_orig_100_case_5_shot.json #> metric_test_5_shot_100_crec.txt
+# echo "Test case contexual recall finished"
+# python mse_ollama_run.py --test cp --dataset ./deepeval-test-dataset/mse_llemma_orig_100_case_5_shot.json #> metric_test_5_shot_100_cp.txt
+# echo "Test case contextual precision finished"
+
+# # python mse_ollama_run.py --num 25 --begin 0 --test cp --shot $NUM_SHOT --out_file metric_test_5_shot_25_cp.txt
+# # echo "Test case contextual precision finished"
+
+# # python mse_ollama_run.py --num 25 --begin 25 --test cp --shot $NUM_SHOT --out_file metric_test_5_shot_25_b25_cp.txt
+# # echo "Test case contextual precision finished (start 25)"
+# # python mse_ollama_run.py --num 25 --begin 50 --test cp --shot $NUM_SHOT --out_file metric_test_5_shot_25_b50_cp.txt
+# # echo "Test case contextual precision finished (start 50)"
+# # python mse_ollama_run.py --num 25 --begin 75 --test cp --shot $NUM_SHOT --out_file metric_test_5_shot_25_b75_cp.txt
+# # echo "Test case contextual precision finished (start 75)"
+
+
+# NUM_SHOT=10
+# python mse_ollama_run.py --test ar --dataset ./deepeval-test-dataset/mse_llemma_orig_100_case_10_shot.json > metric_test_10_shot_100_ar.txt
+# echo "Test case answer relevancy finished"
+# python mse_ollama_run.py --test crec --dataset ./deepeval-test-dataset/mse_llemma_orig_100_case_10_shot.json #> metric_test_10_shot_100_crec.txt
+# echo "Test case contexual recall finished"
+# python mse_ollama_run.py --test cp --dataset ./deepeval-test-dataset/mse_llemma_orig_100_case_10_shot.json #> metric_test_10_shot_100_cp.txt
+# echo "Test case contextual precision finished"
+
+# finetuned
+NUM_SHOT=0
+# export DEEPEVAL_RESULTS_FOLDER="metric_test_ft_100_ar"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test ar --shot $NUM_SHOT --out_file metric_test_ft_100_ar.docx
+# echo "Test case answer relevancy finished"
+# export DEEPEVAL_RESULTS_FOLDER="metric_test_ft_100_crec"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test crec --shot $NUM_SHOT --out_file metric_test_ft_100_crec.docx
+# echo "Test case contexual recall finished"
+# export DEEPEVAL_RESULTS_FOLDER="metric_test_ft_100_cp"
+# python mse_ollama_run.py --num $NUM_TEST_CASES --begin 0 --test cp --shot $NUM_SHOT --out_file metric_test_ft_100_cp.docx
+# echo "Test case contextual precision finished"
+
+python mse_ollama_run_ft.py --test ar --dataset ./deepeval-test-dataset/mse_llemma_ft_100_case_0_shot.json #> metric_test_ft_100_ar.txt
+echo "Test case answer relevancy finished"
+python mse_ollama_run_ft.py --test crec --dataset ./deepeval-test-dataset/mse_llemma_ft_100_case_0_shot.json > metric_test_ft_100_crec.txt
+echo "Test case contexual recall finished"
+python mse_ollama_run_ft.py --test cp --dataset ./deepeval-test-dataset/mse_llemma_ft_100_case_0_shot.json > metric_test_ft_100_cp.txt
+echo "Test case contextual precision finished"
+
+# python mse_ollama_run.py --num $NUM_TEST_CASES --test crel --out_file metric_test_orig_100_crel.txt
+# echo "Test case contextual relevancy finished"
+
+
+# python mse_ollama_run.py --num $NUM_TEST_CASES --test f --out_file metric_test_orig_100_f.txt
+# echo "Test case faithfulness finished"
+
+# python mse_jsonl_resize.py
+
+# python finetune.py

requirements.txt

@@ -0,0 +1,20 @@
+numpy
+matplotlib
+scipy
+sentencepiece
+protobuf
+torch
+gradio
+torchvision
+opencv-python-headless
+tensorboardX
+transformers
+datasets
+pylatex
+pnglatex
+zstandard
+jsonlines
+pyramid==1.5
+deepeval
+ollama
+jsonlines

run.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+#SBATCH --time=1:00:00   # walltime.  hours:minutes:seconds
+#SBATCH --ntasks=8   # number of processor cores (i.e. tasks)
+#SBATCH --nodes=1   # number of nodes
+#SBATCH --gpus=1
+#SBATCH --mem=80G   # 164G memory per CPU core
+#SBATCH --mail-user=aw742@byu.edu   # email address
+#SBATCH --mail-type=BEGIN
+#SBATCH --mail-type=END
+#SBATCH --mail-type=FAIL
+#SBATCH --qos=cs
+#SBATCH --partition=cs
+
+# some helpful debugging options
+set -e
+set -u
+
+# LOAD MODULES, INSERT CODE, AND RUN YOUR PROGRAMS HERE
+# module load python/3.11
+
+# pip install virtualenv
+
+# python -m venv mse_env
+
+source ./mse_env/Scripts/activate
+
+pip uninstall transformers
+pip uninstall torch torchvision torchaudio
+
+cd math-evaluation-harness
+pip install -r requirements.txt
+cd ..
+
+pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128
+# pip install tqdm
+# pip install accelerate

timer_test_llemma.txt

@@ -0,0 +1,17 @@
+Ran Llemma in 3.5200 seconds
+Test time calculated
+
+Ran Llemma in 6.0093 seconds
+Test time calculated
+
+Ran Llemma in 2.3846 seconds
+Test time calculated
+
+Ran Llemma in 2.6123 seconds
+Test time calculated
+
+Ran Llemma in 2.0650 seconds
+Test time calculated
+
+Avg:
+3.32004 seconds