{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0abc7dd7",
   "metadata": {},
   "source": [
    "Master Notebook for Hugging Face repos Upload and Download V3\n",
    "\n",
    "Latest version on : https://www.patreon.com/posts/104672510"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caa74bef",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install huggingface_hub --upgrade\n",
    "\n",
    "!pip install ipywidgets --upgrade"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "502249bf",
   "metadata": {},
   "source": [
    "Use below cell to paste your Hugging Face token key. \n",
    "\n",
    "Access Tokens are here : https://huggingface.co/settings/tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d8027c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use this cell to enter your Hugging Face token and authenticate \n",
    "\n",
    "hugging_face_token = 'your_HF_Token'\n",
    "\n",
    "!export HUGGING_FACE_HUB_TOKEN=hugging_face_token\n",
    "\n",
    "import subprocess\n",
    "\n",
    "# Command to log in using the token\n",
    "command = ['huggingface-cli', 'login', '--token', hugging_face_token]\n",
    "\n",
    "# Execute the command\n",
    "subprocess.run(command, check=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0502703b-8b44-4e58-af2c-82f67d713548",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This cell is used to upload single file into a repo with certain name\n",
    "\n",
    "from huggingface_hub import HfApi\n",
    "api = HfApi()\n",
    "api.upload_file(\n",
    "    path_or_fileobj=r\"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion/model_name.safetensors\",\n",
    "    path_in_repo=\"model_name.safetensors\",\n",
    "    repo_id=\"YourUserName/reponame\",\n",
    "    repo_type=\"model\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3be9367c-a04d-4d05-83ad-c8504e0383b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This cell is used to upload a folder into a repo with single commit\n",
    "\n",
    "from huggingface_hub import HfApi\n",
    "api = HfApi()\n",
    "api.upload_folder(\n",
    "    folder_path=r\"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\",\n",
    "    repo_id=\"YourUserName/reponame\",\n",
    "    repo_type=\"model\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c5ff0b4",
   "metadata": {},
   "source": [
    "To upload all files in given folder to the target Hugging Face repository use below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fc96104",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This cell uploads a folder into remote repo with multi commit\n",
    "# Supports continue feature so if gets interrupted you can run again to continue / resume\n",
    "\n",
    "from huggingface_hub import HfApi\n",
    "from huggingface_hub import get_collection, delete_collection_item\n",
    "from huggingface_hub import upload_file\n",
    "from huggingface_hub import (\n",
    "    HfFolder,\n",
    "    ModelCard,\n",
    "    ModelCardData,\n",
    "    create_repo,\n",
    "    hf_hub_download,\n",
    "    upload_folder,\n",
    "    whoami,\n",
    ")\n",
    "api = HfApi()\n",
    "upload_folder(\n",
    "    folder_path=r\"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\",\n",
    "    repo_id=\"YourUserName/reponame\",\n",
    "    repo_type=\"model\",\n",
    "    multi_commits=True,\n",
    "    multi_commits_verbose=True,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7381726f",
   "metadata": {},
   "source": [
    "To download all files in given Hugging Face repository use below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90f6c9b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This cell downloads all files from given repo 1 by 1 \n",
    "# This is not same as clone and it is faster and more convenient to use \n",
    "\n",
    "!pip install tqdm --upgrade\n",
    "\n",
    "import os\n",
    "import requests\n",
    "from huggingface_hub import list_repo_files, hf_hub_url\n",
    "from huggingface_hub.utils import HfFolder\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Define the repository and target folder\n",
    "repo_id = \"YourUserName/reponame\"\n",
    "target_folder = \"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\"\n",
    "\n",
    "# Retrieve the token from the .huggingface folder or set it manually\n",
    "token = HfFolder.get_token()\n",
    "# or set your token directly\n",
    "# token = \"your_huggingface_token\"\n",
    "\n",
    "if not token:\n",
    "    raise ValueError(\"Hugging Face token not found. Please log in using `huggingface-cli login` or set the token manually.\")\n",
    "\n",
    "headers = {\n",
    "    \"Authorization\": f\"Bearer {token}\"\n",
    "}\n",
    "\n",
    "# List all files in the repository\n",
    "files = list_repo_files(repo_id)\n",
    "\n",
    "# Ensure the target folder exists\n",
    "os.makedirs(target_folder, exist_ok=True)\n",
    "\n",
    "# Download each file directly to the target folder\n",
    "for file in files:\n",
    "    try:\n",
    "        # Define the target path for the file\n",
    "        target_path = os.path.join(target_folder, file)\n",
    "        \n",
    "        # Check if the file already exists\n",
    "        if os.path.exists(target_path):\n",
    "            print(f\"File {file} already exists. Skipping download.\")\n",
    "            continue\n",
    "        \n",
    "        # Get the URL for the file\n",
    "        file_url = hf_hub_url(repo_id, filename=file, repo_type='model')\n",
    "        \n",
    "        # Ensure subdirectories exist\n",
    "        os.makedirs(os.path.dirname(target_path), exist_ok=True)\n",
    "        \n",
    "        # Download the file with authentication\n",
    "        response = requests.get(file_url, headers=headers, stream=True)\n",
    "        response.raise_for_status()  # Raise an error for bad status codes\n",
    "        \n",
    "        # Get the total file size from the response headers\n",
    "        total_size = int(response.headers.get('content-length', 0))\n",
    "        \n",
    "        # Progress bar setup\n",
    "        with tqdm(total=total_size, unit='B', unit_scale=True, desc=file, initial=0, ascii=True) as pbar:\n",
    "            # Write the file to the target path\n",
    "            with open(target_path, 'wb') as f:\n",
    "                for chunk in response.iter_content(chunk_size=8192):\n",
    "                    if chunk:\n",
    "                        f.write(chunk)\n",
    "                        pbar.update(len(chunk))\n",
    "        \n",
    "        # Set the correct permissions for the downloaded file\n",
    "        os.chmod(target_path, 0o644)  # Read and write for owner, read for group and others\n",
    "    \n",
    "    except Exception as e:\n",
    "        print(f\"An error occurred while processing file {file}: {e}\")\n",
    "\n",
    "print(f\"All files have been downloaded to {target_folder}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "download_subdirectory",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This cell downloads a specific subdirectory from a given repo\n",
    "\n",
    "!pip install tqdm --upgrade\n",
    "\n",
    "import os\n",
    "import requests\n",
    "from huggingface_hub import list_repo_files, hf_hub_url\n",
    "from huggingface_hub.utils import HfFolder\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Define the repository, subdirectory, and target folder\n",
    "repo_id = \"YourUserName/reponame\" # example: MonsterMMORPG/kohya_new_test\n",
    "subdirectory = \"path/to/subdirectory\" # example: 6e_6\n",
    "target_folder = \"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\"\n",
    "\n",
    "# Retrieve the token from the .huggingface folder or set it manually\n",
    "token = HfFolder.get_token()\n",
    "# or set your token directly\n",
    "# token = \"your_huggingface_token\"\n",
    "\n",
    "if not token:\n",
    "    raise ValueError(\"Hugging Face token not found. Please log in using `huggingface-cli login` or set the token manually.\")\n",
    "\n",
    "headers = {\n",
    "    \"Authorization\": f\"Bearer {token}\"\n",
    "}\n",
    "\n",
    "# List all files in the repository\n",
    "files = list_repo_files(repo_id)\n",
    "\n",
    "# Ensure the target folder exists\n",
    "os.makedirs(target_folder, exist_ok=True)\n",
    "\n",
    "# Filter files to only include those in the specified subdirectory\n",
    "subdirectory_files = [file for file in files if file.startswith(subdirectory + '/')]\n",
    "\n",
    "# Download each file directly to the target folder\n",
    "for file in subdirectory_files:\n",
    "    try:\n",
    "        # Define the target path for the file\n",
    "        target_path = os.path.join(target_folder, os.path.relpath(file, subdirectory))\n",
    "        \n",
    "        # Check if the file already exists\n",
    "        if os.path.exists(target_path):\n",
    "            print(f\"File {file} already exists. Skipping download.\")\n",
    "            continue\n",
    "        \n",
    "        # Get the URL for the file\n",
    "        file_url = hf_hub_url(repo_id, filename=file, repo_type='model')\n",
    "        \n",
    "        # Ensure subdirectories exist\n",
    "        os.makedirs(os.path.dirname(target_path), exist_ok=True)\n",
    "        \n",
    "        # Download the file with authentication\n",
    "        response = requests.get(file_url, headers=headers, stream=True)\n",
    "        response.raise_for_status()  # Raise an error for bad status codes\n",
    "        \n",
    "        # Get the total file size from the response headers\n",
    "        total_size = int(response.headers.get('content-length', 0))\n",
    "        \n",
    "        # Progress bar setup\n",
    "        with tqdm(total=total_size, unit='B', unit_scale=True, desc=file, initial=0, ascii=True) as pbar:\n",
    "            # Write the file to the target path\n",
    "            with open(target_path, 'wb') as f:\n",
    "                for chunk in response.iter_content(chunk_size=8192):\n",
    "                    if chunk:\n",
    "                        f.write(chunk)\n",
    "                        pbar.update(len(chunk))\n",
    "        \n",
    "        # Set the correct permissions for the downloaded file\n",
    "        os.chmod(target_path, 0o644)  # Read and write for owner, read for group and others\n",
    "    \n",
    "    except Exception as e:\n",
    "        print(f\"An error occurred while processing file {file}: {e}\")\n",
    "\n",
    "print(f\"All files from the subdirectory '{subdirectory}' have been downloaded to {target_folder}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "download_specific_file",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This cell downloads a specific file from a given repo\n",
    "\n",
    "!pip install tqdm --upgrade\n",
    "\n",
    "import os\n",
    "import requests\n",
    "from huggingface_hub import hf_hub_url\n",
    "from huggingface_hub.utils import HfFolder\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Define the repository, file path, and target folder\n",
    "repo_id = \"YourUserName/reponame\" # example: MonsterMMORPG/kohya_new_test\n",
    "file_path = \"path/to/your/file.ext\" # example: 6e_6/9e_6_TE_1e_6.safetensors download model file inside 6e_6 folder\n",
    "target_folder = \"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\"\n",
    "\n",
    "# Retrieve the token from the .huggingface folder or set it manually\n",
    "token = HfFolder.get_token()\n",
    "# or set your token directly\n",
    "# token = \"your_huggingface_token\"\n",
    "\n",
    "if not token:\n",
    "    raise ValueError(\"Hugging Face token not found. Please log in using `huggingface-cli login` or set the token manually.\")\n",
    "\n",
    "headers = {\n",
    "    \"Authorization\": f\"Bearer {token}\"\n",
    "}\n",
    "\n",
    "# Ensure the target folder exists\n",
    "os.makedirs(target_folder, exist_ok=True)\n",
    "\n",
    "# Define the target path for the file\n",
    "target_path = os.path.join(target_folder, os.path.basename(file_path))\n",
    "\n",
    "# Check if the file already exists\n",
    "if os.path.exists(target_path):\n",
    "    print(f\"File {file_path} already exists. Skipping download.\")\n",
    "else:\n",
    "    try:\n",
    "        # Get the URL for the file\n",
    "        file_url = hf_hub_url(repo_id, filename=file_path, repo_type='model')\n",
    "        \n",
    "        # Download the file with authentication\n",
    "        response = requests.get(file_url, headers=headers, stream=True)\n",
    "        response.raise_for_status()  # Raise an error for bad status codes\n",
    "        \n",
    "        # Get the total file size from the response headers\n",
    "        total_size = int(response.headers.get('content-length', 0))\n",
    "        \n",
    "        # Progress bar setup\n",
    "        with tqdm(total=total_size, unit='B', unit_scale=True, desc=os.path.basename(file_path), initial=0, ascii=True) as pbar:\n",
    "            # Write the file to the target path\n",
    "            with open(target_path, 'wb') as f:\n",
    "                for chunk in response.iter_content(chunk_size=8192):\n",
    "                    if chunk:\n",
    "                        f.write(chunk)\n",
    "                        pbar.update(len(chunk))\n",
    "        \n",
    "        # Set the correct permissions for the downloaded file\n",
    "        os.chmod(target_path, 0o644)  # Read and write for owner, read for group and others\n",
    "    \n",
    "    except Exception as e:\n",
    "        print(f\"An error occurred while processing file {file_path}: {e}\")\n",
    "\n",
    "print(f\"The file '{file_path}' has been downloaded to {target_folder}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fd5f2fb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}