Testing upload script

Pipfile

@@ -12,6 +12,8 @@ transformers = "*"
 torch = "*"
 diffusers = "*"
 peft = "*"
+huggingface-hub = "*"
+python-dotenv = "*"
 
 [dev-packages]
 

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "c4efec9481152ce6f9599932fcd2ea1a68f264c46a099862473a4cbc56ae87ab"
+            "sha256": "57e3bd5688f1054cbc1ce950e6dcc4d8ed73eee2d2c72c93c6c2bcd83fd1ac09"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -174,6 +174,7 @@
                 "sha256:414c0d9b769eecc86c70f9d939d0f48bb28e8461dd1130021542eff0212db890",
                 "sha256:5927a8fc64ae68859cd954b7cc29d1c8390a5e15caba6d3d349c973be8fdacf3"
             ],
+            "index": "pypi",
             "markers": "python_full_version >= '3.8.0'",
             "version": "==0.26.1"
         },
@@ -522,6 +523,15 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.9.0.post0"
         },
+        "python-dotenv": {
+            "hashes": [
+                "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca",
+                "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==1.0.1"
+        },
         "pytz": {
             "hashes": [
                 "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a",
@@ -813,6 +823,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==0.4.5"
         },
+        "setuptools": {
+            "hashes": [
+                "sha256:753bb6ebf1f465a1912e19ed1d41f403a79173a9acf66a42e7e6aec45c3c16ec",
+                "sha256:a7fcb66f68b4d9e8e66b42f9876150a3371558f98fa32222ffaa5bced76406f8"
+            ],
+            "markers": "python_version >= '3.12'",
+            "version": "==75.2.0"
+        },
         "six": {
             "hashes": [
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",

models/{ESD-U/esd-vangogh_from_vangogh-noxattn_1-epochs_200.pt → ESD-X/esd-picasso_from_picasso-xattn_1-epochs_200.pt}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dcdd21da7574a33e4fa2690e59c7b8695862eec92a3186b32f181af87b0b75c1
-size 3261580826
+oid sha256:f52f71a02de7c24b6191ec4385a6a5b08a7b2177458a7998dfffed28a6b49f50
+size 175883766

upload.py

@@ -1,55 +1,49 @@
-from huggingface_hub import HfApi, HfFolder, hf_hub_upload
+from huggingface_hub import HfApi
 import os
-import subprocess
+from dotenv import load_dotenv
 
+# Load environment variables from .env file
+load_dotenv()
+
+# Access Hugging Face API token
 hf_access_token = os.getenv("HUGGINGFACE_API_KEY")
-HfFolder.save_token(hf_access_token)
+print(hf_access_token)  # Print to confirm loading, remove in production
 
-MODEL_PATH = '/raid/aag/scasper/models/'
+# Base path to your models directory
+MODEL_PATH = "/Users/klu/ErasingDiffusionModels/models/"
+hf_username = "kevinlu4588"  # replace with your Hugging Face username or org name
 
-def run_cleanup():
-    try:
-        subprocess.run(["python", "clean_folder.py"], check=True)
-        print("Cleanup completed successfully.")
-    except subprocess.CalledProcessError as e:
-        print(f"Error running cleanup script: {e}")
-    except FileNotFoundError:
-        print("clean_folder.py not found in the current directory.")
+api = HfApi()
 
 def is_repo_empty(repo_id):
-    api = HfApi()
     try:
         files = api.list_repo_files(repo_id)
         return len(files) <= 2
     except Exception as e:
-        print(f"An error occurred: {e}")
+        print(f"Error checking repository {repo_id}: {e}")
         return True
 
-def upload_pt_file(subdir, model_name, ckpt):
-    file_path = f"{MODEL_PATH}{subdir}/{model_name}{ckpt}.pt"
-    repo_id = f"LLM-GAT/{model_name}{ckpt}"
+def upload_model_folder(subfolder_path):
+    model_name = os.path.basename(subfolder_path)
+    repo_id = f"{hf_username}/{model_name}"
+    
+    # Create the repo if it doesn't exist, or skip if it has files
+    api.create_repo(repo_id, exist_ok=True, token=hf_access_token)
     if is_repo_empty(repo_id):
-        hf_hub_upload(repo_id=repo_id, path_or_fileobj=file_path, token=hf_access_token)
-        print(f"Uploaded {file_path} to {repo_id}")
+        print(f"Uploading {subfolder_path} to {repo_id}...")
+        api.upload_folder(folder_path=subfolder_path,
+                          repo_id=repo_id,
+                          repo_type="model",
+                          token=hf_access_token)
+        print(f"Successfully uploaded {subfolder_path} to {repo_id}.")
     else:
-        print(f"Repository {repo_id} is not empty. Skipping upload.")
-
-subdirs = ['gd']
-model_names = ['llama3_gd_lora-256-128_beta-14_bs-32_lr-1e-04_checkpoint-']
-ckpts = list(range(1, 9))
+        print(f"Repository {repo_id} already contains files. Skipping upload.")
 
 def main():
-    missing_models = []
-    for subdir, model in zip(subdirs, model_names):
-        for ckpt in ckpts:
-            file_path = f"{MODEL_PATH}{subdir}/{model}{ckpt}.pt"
-            if os.path.isfile(file_path):
-                print(f"Uploading {file_path}...")
-                upload_pt_file(subdir, model, ckpt)
-            else:
-                missing_models.append(file_path)
-            run_cleanup()
-    print('MISSING MODELS:', missing_models)
+    for subfolder in os.listdir(MODEL_PATH):
+        subfolder_path = os.path.join(MODEL_PATH, subfolder)
+        if os.path.isdir(subfolder_path):
+            upload_model_folder(subfolder_path)
 
 if __name__ == "__main__":
-    main()
+    main()