Upload Hugging_Face_Upload_Download_V3.ipynb

Hugging_Face_Upload_Download_V3.ipynb

@@ -0,0 +1,412 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "0abc7dd7",
+   "metadata": {},
+   "source": [
+    "Master Notebook for Hugging Face repos Upload and Download V3\n",
+    "\n",
+    "Latest version on : https://www.patreon.com/posts/104672510"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "caa74bef",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install huggingface_hub --upgrade\n",
+    "\n",
+    "!pip install ipywidgets --upgrade"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "502249bf",
+   "metadata": {},
+   "source": [
+    "Use below cell to paste your Hugging Face token key. \n",
+    "\n",
+    "Access Tokens are here : https://huggingface.co/settings/tokens"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0d8027c8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Use this cell to enter your Hugging Face token and authenticate \n",
+    "\n",
+    "hugging_face_token = 'your_HF_Token'\n",
+    "\n",
+    "!export HUGGING_FACE_HUB_TOKEN=hugging_face_token\n",
+    "\n",
+    "import subprocess\n",
+    "\n",
+    "# Command to log in using the token\n",
+    "command = ['huggingface-cli', 'login', '--token', hugging_face_token]\n",
+    "\n",
+    "# Execute the command\n",
+    "subprocess.run(command, check=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0502703b-8b44-4e58-af2c-82f67d713548",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# This cell is used to upload single file into a repo with certain name\n",
+    "\n",
+    "from huggingface_hub import HfApi\n",
+    "api = HfApi()\n",
+    "api.upload_file(\n",
+    "    path_or_fileobj=r\"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion/model_name.safetensors\",\n",
+    "    path_in_repo=\"model_name.safetensors\",\n",
+    "    repo_id=\"YourUserName/reponame\",\n",
+    "    repo_type=\"model\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3be9367c-a04d-4d05-83ad-c8504e0383b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# This cell is used to upload a folder into a repo with single commit\n",
+    "\n",
+    "from huggingface_hub import HfApi\n",
+    "api = HfApi()\n",
+    "api.upload_folder(\n",
+    "    folder_path=r\"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\",\n",
+    "    repo_id=\"YourUserName/reponame\",\n",
+    "    repo_type=\"model\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8c5ff0b4",
+   "metadata": {},
+   "source": [
+    "To upload all files in given folder to the target Hugging Face repository use below"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6fc96104",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# This cell uploads a folder into remote repo with multi commit\n",
+    "# Supports continue feature so if gets interrupted you can run again to continue / resume\n",
+    "\n",
+    "from huggingface_hub import HfApi\n",
+    "from huggingface_hub import get_collection, delete_collection_item\n",
+    "from huggingface_hub import upload_file\n",
+    "from huggingface_hub import (\n",
+    "    HfFolder,\n",
+    "    ModelCard,\n",
+    "    ModelCardData,\n",
+    "    create_repo,\n",
+    "    hf_hub_download,\n",
+    "    upload_folder,\n",
+    "    whoami,\n",
+    ")\n",
+    "api = HfApi()\n",
+    "upload_folder(\n",
+    "    folder_path=r\"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\",\n",
+    "    repo_id=\"YourUserName/reponame\",\n",
+    "    repo_type=\"model\",\n",
+    "    multi_commits=True,\n",
+    "    multi_commits_verbose=True,\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7381726f",
+   "metadata": {},
+   "source": [
+    "To download all files in given Hugging Face repository use below"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "90f6c9b1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# This cell downloads all files from given repo 1 by 1 \n",
+    "# This is not same as clone and it is faster and more convenient to use \n",
+    "\n",
+    "!pip install tqdm --upgrade\n",
+    "\n",
+    "import os\n",
+    "import requests\n",
+    "from huggingface_hub import list_repo_files, hf_hub_url\n",
+    "from huggingface_hub.utils import HfFolder\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "# Define the repository and target folder\n",
+    "repo_id = \"YourUserName/reponame\"\n",
+    "target_folder = \"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\"\n",
+    "\n",
+    "# Retrieve the token from the .huggingface folder or set it manually\n",
+    "token = HfFolder.get_token()\n",
+    "# or set your token directly\n",
+    "# token = \"your_huggingface_token\"\n",
+    "\n",
+    "if not token:\n",
+    "    raise ValueError(\"Hugging Face token not found. Please log in using `huggingface-cli login` or set the token manually.\")\n",
+    "\n",
+    "headers = {\n",
+    "    \"Authorization\": f\"Bearer {token}\"\n",
+    "}\n",
+    "\n",
+    "# List all files in the repository\n",
+    "files = list_repo_files(repo_id)\n",
+    "\n",
+    "# Ensure the target folder exists\n",
+    "os.makedirs(target_folder, exist_ok=True)\n",
+    "\n",
+    "# Download each file directly to the target folder\n",
+    "for file in files:\n",
+    "    try:\n",
+    "        # Define the target path for the file\n",
+    "        target_path = os.path.join(target_folder, file)\n",
+    "        \n",
+    "        # Check if the file already exists\n",
+    "        if os.path.exists(target_path):\n",
+    "            print(f\"File {file} already exists. Skipping download.\")\n",
+    "            continue\n",
+    "        \n",
+    "        # Get the URL for the file\n",
+    "        file_url = hf_hub_url(repo_id, filename=file, repo_type='model')\n",
+    "        \n",
+    "        # Ensure subdirectories exist\n",
+    "        os.makedirs(os.path.dirname(target_path), exist_ok=True)\n",
+    "        \n",
+    "        # Download the file with authentication\n",
+    "        response = requests.get(file_url, headers=headers, stream=True)\n",
+    "        response.raise_for_status()  # Raise an error for bad status codes\n",
+    "        \n",
+    "        # Get the total file size from the response headers\n",
+    "        total_size = int(response.headers.get('content-length', 0))\n",
+    "        \n",
+    "        # Progress bar setup\n",
+    "        with tqdm(total=total_size, unit='B', unit_scale=True, desc=file, initial=0, ascii=True) as pbar:\n",
+    "            # Write the file to the target path\n",
+    "            with open(target_path, 'wb') as f:\n",
+    "                for chunk in response.iter_content(chunk_size=8192):\n",
+    "                    if chunk:\n",
+    "                        f.write(chunk)\n",
+    "                        pbar.update(len(chunk))\n",
+    "        \n",
+    "        # Set the correct permissions for the downloaded file\n",
+    "        os.chmod(target_path, 0o644)  # Read and write for owner, read for group and others\n",
+    "    \n",
+    "    except Exception as e:\n",
+    "        print(f\"An error occurred while processing file {file}: {e}\")\n",
+    "\n",
+    "print(f\"All files have been downloaded to {target_folder}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "download_subdirectory",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# This cell downloads a specific subdirectory from a given repo\n",
+    "\n",
+    "!pip install tqdm --upgrade\n",
+    "\n",
+    "import os\n",
+    "import requests\n",
+    "from huggingface_hub import list_repo_files, hf_hub_url\n",
+    "from huggingface_hub.utils import HfFolder\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "# Define the repository, subdirectory, and target folder\n",
+    "repo_id = \"YourUserName/reponame\" # example: MonsterMMORPG/kohya_new_test\n",
+    "subdirectory = \"path/to/subdirectory\" # example: 6e_6\n",
+    "target_folder = \"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\"\n",
+    "\n",
+    "# Retrieve the token from the .huggingface folder or set it manually\n",
+    "token = HfFolder.get_token()\n",
+    "# or set your token directly\n",
+    "# token = \"your_huggingface_token\"\n",
+    "\n",
+    "if not token:\n",
+    "    raise ValueError(\"Hugging Face token not found. Please log in using `huggingface-cli login` or set the token manually.\")\n",
+    "\n",
+    "headers = {\n",
+    "    \"Authorization\": f\"Bearer {token}\"\n",
+    "}\n",
+    "\n",
+    "# List all files in the repository\n",
+    "files = list_repo_files(repo_id)\n",
+    "\n",
+    "# Ensure the target folder exists\n",
+    "os.makedirs(target_folder, exist_ok=True)\n",
+    "\n",
+    "# Filter files to only include those in the specified subdirectory\n",
+    "subdirectory_files = [file for file in files if file.startswith(subdirectory + '/')]\n",
+    "\n",
+    "# Download each file directly to the target folder\n",
+    "for file in subdirectory_files:\n",
+    "    try:\n",
+    "        # Define the target path for the file\n",
+    "        target_path = os.path.join(target_folder, os.path.relpath(file, subdirectory))\n",
+    "        \n",
+    "        # Check if the file already exists\n",
+    "        if os.path.exists(target_path):\n",
+    "            print(f\"File {file} already exists. Skipping download.\")\n",
+    "            continue\n",
+    "        \n",
+    "        # Get the URL for the file\n",
+    "        file_url = hf_hub_url(repo_id, filename=file, repo_type='model')\n",
+    "        \n",
+    "        # Ensure subdirectories exist\n",
+    "        os.makedirs(os.path.dirname(target_path), exist_ok=True)\n",
+    "        \n",
+    "        # Download the file with authentication\n",
+    "        response = requests.get(file_url, headers=headers, stream=True)\n",
+    "        response.raise_for_status()  # Raise an error for bad status codes\n",
+    "        \n",
+    "        # Get the total file size from the response headers\n",
+    "        total_size = int(response.headers.get('content-length', 0))\n",
+    "        \n",
+    "        # Progress bar setup\n",
+    "        with tqdm(total=total_size, unit='B', unit_scale=True, desc=file, initial=0, ascii=True) as pbar:\n",
+    "            # Write the file to the target path\n",
+    "            with open(target_path, 'wb') as f:\n",
+    "                for chunk in response.iter_content(chunk_size=8192):\n",
+    "                    if chunk:\n",
+    "                        f.write(chunk)\n",
+    "                        pbar.update(len(chunk))\n",
+    "        \n",
+    "        # Set the correct permissions for the downloaded file\n",
+    "        os.chmod(target_path, 0o644)  # Read and write for owner, read for group and others\n",
+    "    \n",
+    "    except Exception as e:\n",
+    "        print(f\"An error occurred while processing file {file}: {e}\")\n",
+    "\n",
+    "print(f\"All files from the subdirectory '{subdirectory}' have been downloaded to {target_folder}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "download_specific_file",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# This cell downloads a specific file from a given repo\n",
+    "\n",
+    "!pip install tqdm --upgrade\n",
+    "\n",
+    "import os\n",
+    "import requests\n",
+    "from huggingface_hub import hf_hub_url\n",
+    "from huggingface_hub.utils import HfFolder\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "# Define the repository, file path, and target folder\n",
+    "repo_id = \"YourUserName/reponame\" # example: MonsterMMORPG/kohya_new_test\n",
+    "file_path = \"path/to/your/file.ext\" # example: 6e_6/9e_6_TE_1e_6.safetensors download model file inside 6e_6 folder\n",
+    "target_folder = \"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\"\n",
+    "\n",
+    "# Retrieve the token from the .huggingface folder or set it manually\n",
+    "token = HfFolder.get_token()\n",
+    "# or set your token directly\n",
+    "# token = \"your_huggingface_token\"\n",
+    "\n",
+    "if not token:\n",
+    "    raise ValueError(\"Hugging Face token not found. Please log in using `huggingface-cli login` or set the token manually.\")\n",
+    "\n",
+    "headers = {\n",
+    "    \"Authorization\": f\"Bearer {token}\"\n",
+    "}\n",
+    "\n",
+    "# Ensure the target folder exists\n",
+    "os.makedirs(target_folder, exist_ok=True)\n",
+    "\n",
+    "# Define the target path for the file\n",
+    "target_path = os.path.join(target_folder, os.path.basename(file_path))\n",
+    "\n",
+    "# Check if the file already exists\n",
+    "if os.path.exists(target_path):\n",
+    "    print(f\"File {file_path} already exists. Skipping download.\")\n",
+    "else:\n",
+    "    try:\n",
+    "        # Get the URL for the file\n",
+    "        file_url = hf_hub_url(repo_id, filename=file_path, repo_type='model')\n",
+    "        \n",
+    "        # Download the file with authentication\n",
+    "        response = requests.get(file_url, headers=headers, stream=True)\n",
+    "        response.raise_for_status()  # Raise an error for bad status codes\n",
+    "        \n",
+    "        # Get the total file size from the response headers\n",
+    "        total_size = int(response.headers.get('content-length', 0))\n",
+    "        \n",
+    "        # Progress bar setup\n",
+    "        with tqdm(total=total_size, unit='B', unit_scale=True, desc=os.path.basename(file_path), initial=0, ascii=True) as pbar:\n",
+    "            # Write the file to the target path\n",
+    "            with open(target_path, 'wb') as f:\n",
+    "                for chunk in response.iter_content(chunk_size=8192):\n",
+    "                    if chunk:\n",
+    "                        f.write(chunk)\n",
+    "                        pbar.update(len(chunk))\n",
+    "        \n",
+    "        # Set the correct permissions for the downloaded file\n",
+    "        os.chmod(target_path, 0o644)  # Read and write for owner, read for group and others\n",
+    "    \n",
+    "    except Exception as e:\n",
+    "        print(f\"An error occurred while processing file {file_path}: {e}\")\n",
+    "\n",
+    "print(f\"The file '{file_path}' has been downloaded to {target_folder}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4fd5f2fb",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}