Auotmate evalaution

evaluate.py

@@ -2,15 +2,18 @@ import json
 import random
 
 import torch
+import time
 import os
 from distributed_training.data.dataset import DataLoader
 from huggingface_hub import list_repo_refs
 from transformers import AutoModelForCausalLM, AutoTokenizer
+from huggingface_hub import create_tag, list_repo_refs, scan_cache_dir
 
 device = "cuda"
 test_indices_length = 1000
+AUTOMATE = True
 
-models = ["distributed/optimized-gpt2-500m", "distributed/optimized-gpt2-250m", "distributed/optimized-gpt2-250m-v0.1.3", "distributed/optimized-gpt2-250m-v0.1.1", "distributed/gpt2-94m"]
+models = ["distributed/optimized-gpt2-1b", "distributed/optimized-gpt2-500m", "distributed/optimized-gpt2-250m", "distributed/optimized-gpt2-250m-v0.1.3", "distributed/optimized-gpt2-250m-v0.1.1", "distributed/gpt2-94m"]
 
 if os.path.exists("results.json"):
     with open('results.json', 'r') as file:
@@ -18,70 +21,102 @@ if os.path.exists("results.json"):
 else:
     results = {}
 
-for model_name in models:
+while True:
+    for model_name in [models[0]]:
 
-    if (model_name not in results.keys()):
-        results[model_name] = {}
+        if (model_name not in results.keys()):
+            results[model_name] = {}
 
-    tokenizer = AutoTokenizer.from_pretrained("distributed/optimized-gpt2-250m", trust_remote_code=True)
+        tokenizer = AutoTokenizer.from_pretrained("distributed/optimized-gpt2-250m", trust_remote_code=True)
 
-    refs = list_repo_refs(model_name, repo_type="model")
-    global_epoch = max([int(tag.name) for tag in refs.tags]) if refs.tags else None
+        refs = list_repo_refs(model_name, repo_type="model")
+        global_epoch = max([int(tag.name) for tag in refs.tags]) if refs.tags else None
 
-    for epoch in range(0,global_epoch, 1):
+        if global_epoch in results[model_name]['main-net'].keys():
+            print(f"Results for epoch {global_epoch} already calcualted")
+            time.sleep(30*60)
 
-        if str(epoch) in results[model_name]['main-net'].keys():
-            continue
+        for epoch in range(0,global_epoch, 1):
 
-        model = AutoModelForCausalLM.from_pretrained(model_name, revision=str(epoch), trust_remote_code=True)
-        model = model.to(device)
+            if str(epoch) in results[model_name]['main-net'].keys():
+                continue
 
-        search_start = random.choice(
-            range(
-                DataLoader.max_pages
-                - test_indices_length
-                + 1
+            model = AutoModelForCausalLM.from_pretrained(model_name, revision=str(epoch), trust_remote_code=True)
+            model = model.to(device)
+
+            search_start = random.choice(
+                range(
+                    DataLoader.max_pages
+                    - test_indices_length
+                    + 1
+                )
             )
-        )
-        group = [
-            i
-            for i in range(
-                search_start, search_start + test_indices_length
+            group = [
+                i
+                for i in range(
+                    search_start, search_start + test_indices_length
+                )
+            ]
+
+            dataloader = DataLoader(
+                batch_size=1,
+                sequence_length=1024,
+                rows=group,
             )
-        ]
-
-        dataloader = DataLoader(
-            batch_size=1,
-            sequence_length=1024,
-            rows=group,
-        )
-
-        total_loss = 0
-        index = 0
-        # Train data for one epoch
-        for index, batch in enumerate(dataloader):
-            inputs = batch[0].to(device)
-            labels = batch[1].to(device)
-
-            if (len(inputs[0]) != len(labels[0])):
-                breakpoint()
-                
-            if "optimized" in model_name:
-                outputs = model(input_ids=inputs, labels=labels)
-                loss = outputs[1]
-            else:
-                outputs = model(input_ids=inputs, labels=inputs)
-                loss = outputs.loss
-
-            # Accumulate Total Loss
-            total_loss += loss.detach().item()
-
-            # Backward Pass
-            model.zero_grad()
-
-        average_loss = total_loss / (index+1)
-        results[model_name]['main-net'][str(epoch)] = [average_loss]
-        print(f"Epoch: {epoch}  Average Loss: {average_loss:.2f}")
-
-        with open("results.json", "w") as outfile:
-            json.dump(results, outfile, indent = 4)
+
+            total_loss = 0
+            index = 0
+            # Train data for one epoch
+            for index, batch in enumerate(dataloader):
+                inputs = batch[0].to(device)
+                labels = batch[1].to(device)
+
+                if (len(inputs[0]) != len(labels[0])):
+                    breakpoint()
+                if "optimized" in model_name:
+                    outputs = model(input_ids=inputs, labels=labels)
+                    loss = outputs[1]
+                else:
+                    outputs = model(input_ids=inputs, labels=inputs)
+                    loss = outputs.loss
+
+                # Accumulate Total Loss
+                total_loss += loss.detach().item()
+
+                # Backward Pass
+                model.zero_grad()
+
+            average_loss = total_loss / (index+1)
+            results[model_name]['main-net'][str(epoch)] = [average_loss]
+            print(f"Epoch: {epoch}  Average Loss: {average_loss:.2f}")
+
+            with open("results.json", "w") as outfile:
+                json.dump(results, outfile, indent = 4)
+
+        current_revision = model.config._commit_hash
+        keep_recent=1
+        try:
+            cache_info = scan_cache_dir()
+            for repo in cache_info.repos:
+                if repo.repo_id == model_name:
+                    revisions = sorted(
+                        repo.revisions, key=lambda r: r.last_modified, reverse=True
+                    )
+                    current_index = next(
+                        (
+                            i
+                            for i, r in enumerate(revisions)
+                            if r.commit_hash == current_revision
+                        ),
+                        None,
+                    )
+                    if current_index is not None:
+                        for revision in revisions[
+                            max(current_index + 1, keep_recent) :
+                        ]:
+                            cache_info.delete_revisions(revision.commit_hash).execute()
+                    break
+        except:
+            print(
+                "Failed to delete previous model version from cache. This might lead to 100% disk space utlisation in the future."
+            )

results.json

@@ -1635,6 +1635,12 @@
             ],
             "544": [
                 5.487078181581001
+            ],
+            "545": [
+                5.599780409645654
+            ],
+            "546": [
+                5.532580448878751
             ]
         },
         "baseline": {