{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "360f3e9b-c5b6-4f03-aa1d-afab0f137247",
   "metadata": {},
   "outputs": [],
   "source": [
    "from huggingface_hub import HfApi\n",
    "import huggingface_hub as hfh\n",
    "api = HfApi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "542ff8be-3ef7-4788-805f-101484eaef29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c98c55f0c01044f0afdd73b603cc7002",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "hfh.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a268481-e3d3-4af6-824c-e6782ee5a9da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa7a9526865547ffa039d1f007589596",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Recovering from metadata files:   0%|          | 0/17447 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "---------- 2024-12-18 14:38:19 (0:00:00) ----------\n",
      "Files:   hashed 17447/17447 (32.2G/32.2G) | pre-uploaded: 5605/5605 (29.7G/32.2G) (+4 unsure) | committed: 13020/17447 (18.5G/32.2G) | ignored: 0\n",
      "Workers: hashing: 0 | get upload mode: 4 | pre-uploading: 0 | committing: 1 | waiting: 185\n",
      "---------------------------------------------------\n",
      "\u001b[K\u001b[F\n",
      "---------- 2024-12-18 14:39:19 (0:01:00) ----------\n",
      "Files:   hashed 17447/17447 (32.2G/32.2G) | pre-uploaded: 5605/5605 (29.7G/32.2G) | committed: 13489/17447 (20.1G/32.2G) | ignored: 0\n",
      "Workers: hashing: 0 | get upload mode: 0 | pre-uploading: 0 | committing: 1 | waiting: 189\n",
      "---------------------------------------------------\n",
      "\u001b[K\u001b[F                       \n",
      "---------- 2024-12-18 14:40:19 (0:02:00) ----------\n",
      "Files:   hashed 17447/17447 (32.2G/32.2G) | pre-uploaded: 5605/5605 (29.7G/32.2G) | committed: 14389/17447 (20.2G/32.2G) | ignored: 0\n",
      "Workers: hashing: 0 | get upload mode: 0 | pre-uploading: 0 | committing: 1 | waiting: 189\n",
      "---------------------------------------------------\n",
      "\u001b[K\u001b[F                       \n",
      "---------- 2024-12-18 14:41:19 (0:03:00) ----------\n",
      "Files:   hashed 17447/17447 (32.2G/32.2G) | pre-uploaded: 5605/5605 (29.7G/32.2G) | committed: 15261/17447 (25.1G/32.2G) | ignored: 0\n",
      "Workers: hashing: 0 | get upload mode: 0 | pre-uploading: 0 | committing: 1 | waiting: 189\n",
      "---------------------------------------------------\n",
      "\u001b[K\u001b[F                       \n",
      "---------- 2024-12-18 14:42:19 (0:04:00) ----------\n",
      "Files:   hashed 17447/17447 (32.2G/32.2G) | pre-uploaded: 5605/5605 (29.7G/32.2G) | committed: 16012/17447 (27.5G/32.2G) | ignored: 0\n",
      "Workers: hashing: 0 | get upload mode: 0 | pre-uploading: 0 | committing: 1 | waiting: 189\n",
      "---------------------------------------------------\n",
      "                             "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/admin/home-ckadirt/mindeye/lib/python3.11/site-packages/huggingface_hub/hf_api.py:3981: UserWarning: It seems that you are about to commit a data file (fMRI-foundation-model/src/unique_features.parquet) to a model repository. You are sure this is intended? If you are trying to upload a dataset, please set `repo_type='dataset'` or `--repo-type=dataset` in a CLI.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[K\u001b[F\n",
      "---------- 2024-12-18 14:43:19 (0:05:00) ----------\n",
      "Files:   hashed 17447/17447 (32.2G/32.2G) | pre-uploaded: 5605/5605 (29.7G/32.2G) | committed: 16764/17447 (30.7G/32.2G) | ignored: 0\n",
      "Workers: hashing: 0 | get upload mode: 0 | pre-uploading: 0 | committing: 1 | waiting: 189\n",
      "---------------------------------------------------\n",
      "                             "
     ]
    }
   ],
   "source": [
    "api.upload_large_folder(\n",
    "    folder_path=\"/weka/proj-fmri/ckadirt\",\n",
    "    # folder_path = \"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/slurms\",\n",
    "    # path_in_repo=\"ckadirt/ctr\",\n",
    "    repo_id=\"ckadirt/backup_s\",\n",
    "    repo_type=\"model\",\n",
    "    ignore_patterns=[\n",
    "        \"fMRI-foundation-model/src/checkpoints/**\",\n",
    "        \"*.hdf5\"\n",
    "    ]\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}