Adding upload script

.gitignore

@@ -0,0 +1 @@
+.env

Pipfile

@@ -11,6 +11,7 @@ numpy = "*"
 transformers = "*"
 torch = "*"
 diffusers = "*"
+peft = "*"
 
 [dev-packages]
 

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "c187f0d79da53236f1ef7cf6e37c48bd750594b5870a901518759b09f45830f0"
+            "sha256": "c4efec9481152ce6f9599932fcd2ea1a68f264c46a099862473a4cbc56ae87ab"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -17,6 +17,14 @@
         ]
     },
     "default": {
+        "accelerate": {
+            "hashes": [
+                "sha256:c6aa0c7b8a797cb150471e90e3ca36ac41f5d4b40512cdd6f058b8bf25589467",
+                "sha256:e8f95fc2db14915dc0a9182edfcf3068e5ddb2fa310b583717ad44e5c442399c"
+            ],
+            "markers": "python_full_version >= '3.8.0'",
+            "version": "==1.0.1"
+        },
         "certifi": {
             "hashes": [
                 "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8",
@@ -392,6 +400,15 @@
             "markers": "python_version >= '3.9'",
             "version": "==2.2.3"
         },
+        "peft": {
+            "hashes": [
+                "sha256:0e0cbd40ebdf5fe4ea79f255880d02f96712d18899509369a2cc5768ad46d672",
+                "sha256:d4e0951ec78eac11c45a051801c569913436888c578d48e5ce86996b715bc6ef"
+            ],
+            "index": "pypi",
+            "markers": "python_full_version >= '3.8.0'",
+            "version": "==0.13.2"
+        },
         "pillow": {
             "hashes": [
                 "sha256:00177a63030d612148e659b55ba99527803288cea7c75fb05766ab7981a8c1b7",
@@ -474,6 +491,29 @@
             "markers": "python_version >= '3.9'",
             "version": "==11.0.0"
         },
+        "psutil": {
+            "hashes": [
+                "sha256:000d1d1ebd634b4efb383f4034437384e44a6d455260aaee2eca1e9c1b55f047",
+                "sha256:045f00a43c737f960d273a83973b2511430d61f283a44c96bf13a6e829ba8fdc",
+                "sha256:0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e",
+                "sha256:1209036fbd0421afde505a4879dee3b2fd7b1e14fee81c0069807adcbbcca747",
+                "sha256:1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e",
+                "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a",
+                "sha256:498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b",
+                "sha256:5cd2bcdc75b452ba2e10f0e8ecc0b57b827dd5d7aaffbc6821b2a9a242823a76",
+                "sha256:6d3fbbc8d23fcdcb500d2c9f94e07b1342df8ed71b948a2649b5cb060a7c94ca",
+                "sha256:6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688",
+                "sha256:9118f27452b70bb1d9ab3198c1f626c2499384935aaf55388211ad982611407e",
+                "sha256:9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38",
+                "sha256:a8506f6119cff7015678e2bce904a4da21025cc70ad283a53b099e7620061d85",
+                "sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be",
+                "sha256:c0e0c00aa18ca2d3b2b991643b799a15fc8f0563d2ebb6040f64ce8dc027b942",
+                "sha256:d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a",
+                "sha256:ff34df86226c0227c52f38b919213157588a678d049688eded74c76c8ba4a5d0"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "version": "==6.1.0"
+        },
         "python-dateutil": {
             "hashes": [
                 "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3",
@@ -773,14 +813,6 @@
             "markers": "python_version >= '3.7'",
             "version": "==0.4.5"
         },
-        "setuptools": {
-            "hashes": [
-                "sha256:753bb6ebf1f465a1912e19ed1d41f403a79173a9acf66a42e7e6aec45c3c16ec",
-                "sha256:a7fcb66f68b4d9e8e66b42f9876150a3371558f98fa32222ffaa5bced76406f8"
-            ],
-            "markers": "python_version >= '3.12'",
-            "version": "==75.2.0"
-        },
         "six": {
             "hashes": [
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",

models/ESD-X/esd-picasso_from_picasso-xattn_1-epochs_200.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f52f71a02de7c24b6191ec4385a6a5b08a7b2177458a7998dfffed28a6b49f50
-size 175883766

upload.py

@@ -0,0 +1,55 @@
+from huggingface_hub import HfApi, HfFolder, hf_hub_upload
+import os
+import subprocess
+
+hf_access_token = os.getenv("HUGGINGFACE_API_KEY")
+HfFolder.save_token(hf_access_token)
+
+MODEL_PATH = '/raid/aag/scasper/models/'
+
+def run_cleanup():
+    try:
+        subprocess.run(["python", "clean_folder.py"], check=True)
+        print("Cleanup completed successfully.")
+    except subprocess.CalledProcessError as e:
+        print(f"Error running cleanup script: {e}")
+    except FileNotFoundError:
+        print("clean_folder.py not found in the current directory.")
+
+def is_repo_empty(repo_id):
+    api = HfApi()
+    try:
+        files = api.list_repo_files(repo_id)
+        return len(files) <= 2
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        return True
+
+def upload_pt_file(subdir, model_name, ckpt):
+    file_path = f"{MODEL_PATH}{subdir}/{model_name}{ckpt}.pt"
+    repo_id = f"LLM-GAT/{model_name}{ckpt}"
+    if is_repo_empty(repo_id):
+        hf_hub_upload(repo_id=repo_id, path_or_fileobj=file_path, token=hf_access_token)
+        print(f"Uploaded {file_path} to {repo_id}")
+    else:
+        print(f"Repository {repo_id} is not empty. Skipping upload.")
+
+subdirs = ['gd']
+model_names = ['llama3_gd_lora-256-128_beta-14_bs-32_lr-1e-04_checkpoint-']
+ckpts = list(range(1, 9))
+
+def main():
+    missing_models = []
+    for subdir, model in zip(subdirs, model_names):
+        for ckpt in ckpts:
+            file_path = f"{MODEL_PATH}{subdir}/{model}{ckpt}.pt"
+            if os.path.isfile(file_path):
+                print(f"Uploading {file_path}...")
+                upload_pt_file(subdir, model, ckpt)
+            else:
+                missing_models.append(file_path)
+            run_cleanup()
+    print('MISSING MODELS:', missing_models)
+
+if __name__ == "__main__":
+    main()