Update results

evaluate.py

@@ -8,7 +8,7 @@ from huggingface_hub import list_repo_refs
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 device = "cuda"
-test_indices_length = 1000
+test_indices_length = 10
 
 models = ["distributed/optimized-gpt2-250m", "distributed/optimized-gpt2-250m-v0.1.1", "distributed/gpt2-94m"]
 
@@ -28,9 +28,9 @@ for model_name in models:
     refs = list_repo_refs(model_name, repo_type="model")
     global_epoch = max([int(tag.name) for tag in refs.tags]) if refs.tags else None
 
-    for epoch in range(0,global_epoch, 5):
+    for epoch in range(0,global_epoch):
 
-        if str(epoch) in results[model_name].keys():
+        if str(epoch) in results[model_name]['main-net'].keys():
             continue
 
         model = AutoModelForCausalLM.from_pretrained(model_name, revision=str(epoch), trust_remote_code=True)
@@ -80,7 +80,7 @@ for model_name in models:
             model.zero_grad()
 
         average_loss = total_loss / (index+1)
-        results[model_name][str(epoch)] = [average_loss]
+        results[model_name]['main-net'][str(epoch)] = [average_loss]
         print(f"Epoch: {epoch}  Average Loss: {average_loss:.2f}")
 
         with open("results.json", "w") as outfile:

results.json

@@ -3285,9 +3285,306 @@
             ],
             "1094": [
                 5.5514771938323975
+            ],
+            "1095": [
+                5.654173533121745
+            ],
+            "1096": [
+                5.783674240112305
+            ],
+            "1097": [
+                5.732811212539673
+            ],
+            "1098": [
+                5.725842118263245
+            ],
+            "1099": [
+                6.016797780990601
+            ],
+            "1100": [
+                5.492693265279134
+            ],
+            "1101": [
+                5.746817111968994
+            ],
+            "1102": [
+                5.732641816139221
+            ],
+            "1103": [
+                5.6667522430419925
+            ],
+            "1104": [
+                6.042284965515137
+            ],
+            "1105": [
+                5.957233905792236
+            ],
+            "1106": [
+                6.250933527946472
+            ],
+            "1107": [
+                6.189672231674194
+            ],
+            "1108": [
+                5.723267146519253
+            ],
+            "1109": [
+                5.82790470123291
+            ],
+            "1110": [
+                5.603849093119304
+            ],
+            "1111": [
+                5.782259106636047
+            ],
+            "1112": [
+                5.6029471556345625
+            ],
+            "1113": [
+                5.546664714813232
+            ],
+            "1114": [
+                5.836468458175659
+            ],
+            "1115": [
+                5.762608170509338
+            ],
+            "1116": [
+                6.046388339996338
+            ],
+            "1117": [
+                6.0027131080627445
+            ],
+            "1118": [
+                6.388125038146972
+            ],
+            "1119": [
+                6.11626410484314
+            ],
+            "1120": [
+                6.112424373626709
+            ],
+            "1121": [
+                6.001961326599121
+            ],
+            "1122": [
+                5.683912754058838
+            ],
+            "1123": [
+                5.743198204040527
+            ],
+            "1124": [
+                5.722175757090251
+            ],
+            "1125": [
+                5.8825499534606935
+            ],
+            "1126": [
+                6.028543186187744
+            ],
+            "1127": [
+                5.9720460891723635
+            ],
+            "1128": [
+                6.058712800343831
+            ],
+            "1129": [
+                5.475493907928467
+            ],
+            "1130": [
+                5.970499634742737
+            ],
+            "1131": [
+                5.9493067264556885
+            ],
+            "1132": [
+                5.458620548248291
+            ],
+            "1133": [
+                5.992060820261638
+            ],
+            "1134": [
+                5.951226472854614
+            ],
+            "1135": [
+                5.877881646156311
+            ],
+            "1136": [
+                5.603206443786621
+            ],
+            "1137": [
+                5.8340943336486815
+            ],
+            "1138": [
+                5.788412570953369
+            ],
+            "1139": [
+                5.737103462219238
+            ],
+            "1140": [
+                5.636613210042317
+            ],
+            "1141": [
+                5.949309587478638
+            ],
+            "1142": [
+                5.854878067970276
+            ],
+            "1143": [
+                5.924749374389648
+            ],
+            "1144": [
+                6.321739387512207
+            ],
+            "1145": [
+                5.9811422030131025
+            ],
+            "1146": [
+                5.701364517211914
+            ],
+            "1147": [
+                5.503353691101074
+            ],
+            "1148": [
+                5.773120641708374
+            ],
+            "1149": [
+                6.042929470539093
+            ],
+            "1150": [
+                5.8076521555582685
+            ],
+            "1151": [
+                5.682760079701741
+            ],
+            "1152": [
+                5.757667303085327
+            ],
+            "1153": [
+                5.896499156951904
+            ],
+            "1154": [
+                6.025218367576599
+            ],
+            "1155": [
+                5.879011154174805
+            ],
+            "1156": [
+                5.868439674377441
+            ],
+            "1157": [
+                6.418252754211426
+            ],
+            "1158": [
+                6.2828675508499146
+            ],
+            "1159": [
+                6.36786642074585
+            ],
+            "1160": [
+                6.58310022354126
+            ],
+            "1161": [
+                6.19826873143514
+            ],
+            "1162": [
+                6.289691209793091
+            ],
+            "1163": [
+                5.9907801151275635
+            ],
+            "1164": [
+                6.041745066642761
+            ],
+            "1165": [
+                6.02010326385498
+            ],
+            "1166": [
+                5.7515941460927325
+            ],
+            "1167": [
+                5.48467755317688
+            ],
+            "1168": [
+                6.096215724945068
+            ],
+            "1169": [
+                5.959380865097046
+            ],
+            "1170": [
+                5.851028124491374
+            ],
+            "1171": [
+                5.8480740785598755
+            ],
+            "1172": [
+                5.9064167737960815
+            ],
+            "1173": [
+                5.956684430440267
+            ],
+            "1174": [
+                6.00377357006073
+            ],
+            "1175": [
+                6.077920118967692
+            ],
+            "1176": [
+                5.967975934346517
+            ],
+            "1177": [
+                6.253712558746338
+            ],
+            "1178": [
+                5.780354785919189
+            ],
+            "1179": [
+                5.4884843826293945
+            ],
+            "1180": [
+                5.482951482137044
+            ],
+            "1181": [
+                5.966793219248454
+            ],
+            "1182": [
+                5.51493239402771
+            ],
+            "1183": [
+                5.4840850830078125
+            ],
+            "1184": [
+                5.834247946739197
+            ],
+            "1185": [
+                5.770521521568298
+            ],
+            "1186": [
+                5.671548962593079
+            ],
+            "1187": [
+                5.491109371185303
+            ],
+            "1188": [
+                5.561888694763184
+            ],
+            "1189": [
+                5.711345076560974
+            ],
+            "1190": [
+                5.628474712371826
+            ],
+            "1191": [
+                5.514147567749023
+            ],
+            "1192": [
+                5.556583046913147
+            ],
+            "1193": [
+                5.653698126475017
             ]
         },
-        "baseline":{
+        "baseline": {
             "0": [
                 10.93135
             ],
@@ -3795,7 +4092,7 @@
         }
     },
     "distributed/optimized-gpt2-250m-v0.1.1": {
-        "main-net":{
+        "main-net": {
             "0": [
                 11.042416954040528
             ],
@@ -4727,7 +5024,7 @@
                 6.409368515014648
             ]
         },
-        "baseline":{
+        "baseline": {
             "0": [
                 10.93135
             ],
@@ -5235,7 +5532,7 @@
         }
     },
     "distributed/gpt2-94m": {
-        "main-net":{
+        "main-net": {
             "0": [
                 10.942681312561035
             ],