SYNC: Uploading latest local repository state

sync_hf_repo.py

@@ -0,0 +1,101 @@
+from huggingface_hub import HfApi, CommitOperationDelete
+import os
+from pathlib import Path
+
+REPO_ID = "NickMystic/DeepDream-MLX"
+LOCAL_REPO_PATH = Path(".")
+
+def get_local_files(path: Path):
+    """Recursively list all files in the local repository respecting .gitignore."""
+    # Use git to get the list of files that *should* be in the repo
+    # This is more robust than walking the directory and parsing .gitignore manually.
+    import subprocess
+    result = subprocess.run(
+        ["git", "ls-files", "--full-name"],
+        cwd=path,
+        capture_output=True,
+        text=True,
+        check=True
+    )
+    return set(result.stdout.splitlines())
+
+def main():
+    api = HfApi()
+    
+    print(f"--- Synchronizing Hugging Face Repo: {REPO_ID} ---")
+
+    # 1. Get comprehensive list of files on the remote
+    print("Fetching remote file list...")
+    remote_files = set(api.list_repo_files(repo_id=REPO_ID))
+    print(f"Found {len(remote_files)} files on remote.")
+
+    # 2. Get comprehensive list of files locally (respecting .gitignore)
+    print("Fetching local file list (respecting .gitignore)...")
+    local_files = get_local_files(LOCAL_REPO_PATH)
+    print(f"Found {len(local_files)} files locally (tracked by git).")
+
+    # 3. Identify files to delete from remote (present on remote, not locally)
+    files_to_delete = remote_files - local_files
+    # Also explicitly add the `toConvert` folder if it exists on remote but not locally anymore.
+    # We explicitly deleted `toConvert` locally, so ensure it's gone from remote.
+    if "toConvert/" in remote_files or any(f.startswith("toConvert/") for f in remote_files):
+        print("`toConvert` folder found on remote, adding to deletion list.")
+        # Mark all files within toConvert/ for deletion
+        files_to_delete.update([f for f in remote_files if f.startswith("toConvert/")])
+        
+    if "comparisons/" in remote_files or any(f.startswith("comparisons/") for f in remote_files):
+        print("`comparisons` folder found on remote, ensuring it's synced (may have changed contents).")
+        # For comparisons, we handle with upload_folder, so don't delete entire contents unless absolutely sure.
+        # But we want `comparisons/torch_dream.py` to be the only thing. If there are other files, they should be deleted.
+        pass # Handle below with upload_folder
+
+
+    # 4. Identify files to upload/update (present locally, or different checksum)
+    # This is handled by `upload_folder` below for the entire directory, making it simpler.
+
+    operations = []
+    if files_to_delete:
+        print(f"\nIdentified {len(files_to_delete)} files for deletion from remote:")
+        for file_path in sorted(list(files_to_delete)):
+            print(f"  - {file_path}")
+            operations.append(CommitOperationDelete(path_in_repo=file_path))
+    else:
+        print("\nNo files identified for deletion from remote.")
+
+    if operations:
+        try:
+            api.create_commit(
+                repo_id=REPO_ID,
+                operations=operations,
+                commit_message="CLEANUP: Removing obsolete/untracked files from remote"
+            )
+            print("✅ Successfully executed deletions on Hugging Face.")
+        except Exception as e:
+            print(f"❌ Error during remote deletion commit: {e}")
+            # If commit failed, maybe it was due to non-existent files?
+            # Re-list repo files to debug.
+            try:
+                remote_files_after_attempt = set(api.list_repo_files(repo_id=REPO_ID, recursive=True))
+                # Identify which files *actually* remained after the delete attempt
+                remaining_unwanted_files = files_to_delete.intersection(remote_files_after_attempt)
+                if remaining_unwanted_files:
+                    print(f"Warning: Some files intended for deletion are still on remote: {remaining_unwanted_files}")
+            except Exception as inner_e:
+                print(f"Error re-listing files after failed delete attempt: {inner_e}")
+
+
+    # 5. Upload/Update all local files (this handles assets, new files, and updates existing ones)
+    print(f"\nUploading/updating all local tracked files to {REPO_ID}...")
+    try:
+        api.upload_folder(
+            folder_path=LOCAL_REPO_PATH,
+            repo_id=REPO_ID,
+            commit_message="SYNC: Uploading latest local repository state",
+            repo_type="model" # Explicitly specifying type
+        )
+        print("✅ Successfully synced local repository state to Hugging Face.")
+    except Exception as e:
+        print(f"❌ Error during final upload/sync: {e}")
+
+if __name__ == "__main__":
+    main()