diff --git "a/project/tmft_experiment.ipynb" "b/project/tmft_experiment.ipynb" new file mode 100644--- /dev/null +++ "b/project/tmft_experiment.ipynb" @@ -0,0 +1,2493 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "78c489b6", + "metadata": {}, + "source": [ + "# TMFT End-to-End Experiment\n", + "\n", + "Run this notebook from the uploaded `tmft_project/` directory. It prepares real PII-containing Enron splits, trains all five methods, and computes TER, SER, perplexity, MDP, and two MIA AUC metrics." + ] + }, + { + "cell_type": "markdown", + "id": "305ebc86", + "metadata": {}, + "source": [ + "## 0. Verify Project Structure" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "ce28e817", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Project: /home/TMFT\n", + "Python: 3.10.12 (main, Jul 29 2024, 16:56:48) [GCC 11.4.0]\n", + "Platform: Linux-5.4.0-193-generic-x86_64-with-glibc2.35\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os, sys, platform\n", + "\n", + "PROJECT_ROOT = Path.cwd()\n", + "required = [\n", + " 'configs/config.yaml', 'src/data_prep.py', 'src/train.py',\n", + " 'src/masking.py', 'src/evaluate_pii.py', 'src/evaluate_ppl.py',\n", + " 'src/evaluate_mia.py', 'src/plot_results.py', 'main.py', 'requirements.txt'\n", + "]\n", + "missing = [path for path in required if not (PROJECT_ROOT / path).exists()]\n", + "if missing:\n", + " raise FileNotFoundError('Open the notebook inside the full tmft_project directory. Missing: ' + ', '.join(missing))\n", + "print('Project:', PROJECT_ROOT)\n", + "print('Python:', sys.version)\n", + "print('Platform:', platform.platform())" + ] + }, + { + "cell_type": "markdown", + "id": "2e0f5f9e", + "metadata": {}, + "source": [ + "## 1. Install Dependencies\n", + "Run once in a fresh Vessel workspace, then restart the kernel." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "7a72e18d", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: pip in /usr/local/lib/python3.10/dist-packages (26.1.2)\n", + "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n", + "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n", + "Found existing installation: transformers 4.41.2\n", + "Uninstalling transformers-4.41.2:\n", + " Successfully uninstalled transformers-4.41.2\n", + "Found existing installation: peft 0.11.1\n", + "Uninstalling peft-0.11.1:\n", + " Successfully uninstalled peft-0.11.1\n", + "Found existing installation: accelerate 0.31.0\n", + "Uninstalling accelerate-0.31.0:\n", + " Successfully uninstalled accelerate-0.31.0\n", + "Found existing installation: tokenizers 0.19.1\n", + "Uninstalling tokenizers-0.19.1:\n", + " Successfully uninstalled tokenizers-0.19.1\n", + "Found existing installation: huggingface-hub 0.23.4\n", + "Uninstalling huggingface-hub-0.23.4:\n", + " Successfully uninstalled huggingface-hub-0.23.4\n", + "Found existing installation: datasets 2.20.0\n", + "Uninstalling datasets-2.20.0:\n", + " Successfully uninstalled datasets-2.20.0\n", + "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n", + "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n", + "Collecting accelerate==0.31.0 (from -r requirements.txt (line 1))\n", + " Using cached accelerate-0.31.0-py3-none-any.whl.metadata (19 kB)\n", + "Collecting datasets==2.20.0 (from -r requirements.txt (line 2))\n", + " Using cached datasets-2.20.0-py3-none-any.whl.metadata (19 kB)\n", + "Requirement already satisfied: evaluate in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 3)) (0.4.6)\n", + "Collecting huggingface_hub==0.23.4 (from -r requirements.txt (line 4))\n", + " Using cached huggingface_hub-0.23.4-py3-none-any.whl.metadata (12 kB)\n", + "Requirement already satisfied: ipykernel in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 5)) (6.29.5)\n", + "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 6)) (3.9.2)\n", + "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 7)) (3.9.4)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 8)) (1.24.4)\n", + "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 9)) (2.2.2)\n", + "Collecting peft==0.11.1 (from -r requirements.txt (line 10))\n", + " Using cached peft-0.11.1-py3-none-any.whl.metadata (13 kB)\n", + "Requirement already satisfied: presidio-analyzer in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 11)) (2.2.362)\n", + "Requirement already satisfied: presidio-anonymizer in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 12)) (2.2.362)\n", + "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 13)) (6.0.3)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 14)) (2026.5.9)\n", + "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 15)) (1.5.1)\n", + "Requirement already satisfied: seaborn in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 16)) (0.13.2)\n", + "Requirement already satisfied: spacy in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 17)) (3.8.14)\n", + "Collecting tokenizers==0.19.1 (from -r requirements.txt (line 18))\n", + " Using cached tokenizers-0.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 19)) (4.66.5)\n", + "Collecting transformers==4.41.2 (from -r requirements.txt (line 20))\n", + " Using cached transformers-4.41.2-py3-none-any.whl.metadata (43 kB)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate==0.31.0->-r requirements.txt (line 1)) (24.1)\n", + "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate==0.31.0->-r requirements.txt (line 1)) (5.9.8)\n", + "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate==0.31.0->-r requirements.txt (line 1)) (2.3.1+cu121)\n", + "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate==0.31.0->-r requirements.txt (line 1)) (0.8.0)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (3.13.1)\n", + "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (24.0.0)\n", + "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (0.7)\n", + "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (0.3.8)\n", + "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (2.32.3)\n", + "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (3.7.0)\n", + "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (0.70.16)\n", + "Requirement already satisfied: fsspec<=2024.5.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.5.0,>=2023.1.0->datasets==2.20.0->-r requirements.txt (line 2)) (2024.2.0)\n", + "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (3.14.1)\n", + "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub==0.23.4->-r requirements.txt (line 4)) (4.15.0)\n", + "Requirement already satisfied: comm>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (0.2.2)\n", + "Requirement already satisfied: debugpy>=1.6.5 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (1.8.5)\n", + "Requirement already satisfied: ipython>=7.23.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (8.26.0)\n", + "Requirement already satisfied: jupyter-client>=6.1.12 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (8.6.2)\n", + "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (5.7.2)\n", + "Requirement already satisfied: matplotlib-inline>=0.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (0.1.7)\n", + "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (1.6.0)\n", + "Requirement already satisfied: pyzmq>=24 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (26.1.0)\n", + "Requirement already satisfied: tornado>=6.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (6.4.1)\n", + "Requirement already satisfied: traitlets>=5.4.0 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (5.14.3)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 6)) (1.2.1)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 6)) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 6)) (4.53.1)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 6)) (1.4.5)\n", + "Requirement already satisfied: pillow>=8 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 6)) (10.4.0)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3/dist-packages (from matplotlib->-r requirements.txt (line 6)) (2.4.7)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 6)) (2.9.0.post0)\n", + "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->-r requirements.txt (line 7)) (8.4.1)\n", + "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->-r requirements.txt (line 7)) (1.4.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->-r requirements.txt (line 9)) (2024.1)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->-r requirements.txt (line 9)) (2024.1)\n", + "Requirement already satisfied: phonenumbers<10.0.0,>=8.12 in /usr/local/lib/python3.10/dist-packages (from presidio-analyzer->-r requirements.txt (line 11)) (9.0.32)\n", + "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from presidio-analyzer->-r requirements.txt (line 11)) (2.8.2)\n", + "Requirement already satisfied: tldextract in /usr/local/lib/python3.10/dist-packages (from presidio-analyzer->-r requirements.txt (line 11)) (5.3.1)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.0.0->presidio-analyzer->-r requirements.txt (line 11)) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.0.0->presidio-analyzer->-r requirements.txt (line 11)) (2.20.1)\n", + "Requirement already satisfied: cryptography>=46.0.4 in /usr/local/lib/python3.10/dist-packages (from presidio-anonymizer->-r requirements.txt (line 12)) (49.0.0)\n", + "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->-r requirements.txt (line 15)) (1.14.0)\n", + "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->-r requirements.txt (line 15)) (3.5.0)\n", + "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (3.0.12)\n", + "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (1.0.5)\n", + "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (1.0.15)\n", + "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (2.0.13)\n", + "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (3.0.13)\n", + "Requirement already satisfied: thinc<8.4.0,>=8.3.12 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (8.3.13)\n", + "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (1.1.3)\n", + "Requirement already satisfied: srsly<3.0.0,>=2.5.3 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (2.5.3)\n", + "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (2.0.10)\n", + "Requirement already satisfied: weasel<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (1.0.0)\n", + "Requirement already satisfied: confection<2.0.0,>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (1.3.3)\n", + "Requirement already satisfied: typer<1.0.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (0.25.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (3.1.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (72.1.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets==2.20.0->-r requirements.txt (line 2)) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets==2.20.0->-r requirements.txt (line 2)) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets==2.20.0->-r requirements.txt (line 2)) (2.2.2)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets==2.20.0->-r requirements.txt (line 2)) (2024.7.4)\n", + "Requirement already satisfied: blis<1.4.0,>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from thinc<8.4.0,>=8.3.12->spacy->-r requirements.txt (line 17)) (1.3.3)\n", + "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0.0,>=0.3.0->spacy->-r requirements.txt (line 17)) (1.5.4)\n", + "Requirement already satisfied: rich>=13.8.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0.0,>=0.3.0->spacy->-r requirements.txt (line 17)) (15.0.0)\n", + "Requirement already satisfied: annotated-doc>=0.0.2 in /usr/local/lib/python3.10/dist-packages (from typer<1.0.0,>=0.3.0->spacy->-r requirements.txt (line 17)) (0.0.4)\n", + "Requirement already satisfied: cloudpathlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (0.24.0)\n", + "Requirement already satisfied: smart-open>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (7.6.1)\n", + "Requirement already satisfied: httpx>=0.24.0 in /usr/local/lib/python3.10/dist-packages (from weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (0.27.0)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (2.6.2)\n", + "Requirement already satisfied: aiosignal>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (1.4.0)\n", + "Requirement already satisfied: async-timeout<6.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (5.0.1)\n", + "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (24.2.0)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (1.8.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (6.7.1)\n", + "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (0.5.2)\n", + "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (1.24.2)\n", + "Requirement already satisfied: cffi>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from cryptography>=46.0.4->presidio-anonymizer->-r requirements.txt (line 12)) (2.0.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=2.0.0->cryptography>=46.0.4->presidio-anonymizer->-r requirements.txt (line 12)) (2.22)\n", + "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.0->weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (3.7.1)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.0->weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (1.0.5)\n", + "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.0->weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (1.3.1)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx>=0.24.0->weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (0.14.0)\n", + "Requirement already satisfied: decorator in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (5.1.1)\n", + "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (0.19.1)\n", + "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (3.0.47)\n", + "Requirement already satisfied: pygments>=2.4.0 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (2.18.0)\n", + "Requirement already satisfied: stack-data in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (0.6.3)\n", + "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (1.2.2)\n", + "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (4.9.0)\n", + "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (0.2.13)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /usr/local/lib/python3.10/dist-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (0.8.4)\n", + "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.10/dist-packages (from jupyter-core!=5.0.*,>=4.12->ipykernel->-r requirements.txt (line 5)) (4.2.2)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/dist-packages (from pexpect>4.3->ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (0.7.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib->-r requirements.txt (line 6)) (1.16.0)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=13.8.0->typer<1.0.0,>=0.3.0->spacy->-r requirements.txt (line 17)) (4.2.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=13.8.0->typer<1.0.0,>=0.3.0->spacy->-r requirements.txt (line 17)) (0.1.2)\n", + "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from smart-open>=5.2.1->weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (2.2.1)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (1.12)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (3.2.1)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.105)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (8.9.2.26)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.3.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (11.0.2.54)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (10.3.2.106)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (11.4.5.107)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.0.106)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (2.20.5)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.105)\n", + "Requirement already satisfied: triton==2.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (2.3.1)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.105)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy->-r requirements.txt (line 17)) (2.1.5)\n", + "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from stack-data->ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (2.0.1)\n", + "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.10/dist-packages (from stack-data->ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (2.4.1)\n", + "Requirement already satisfied: pure-eval in /usr/local/lib/python3.10/dist-packages (from stack-data->ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (0.2.3)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (1.3.0)\n", + "Requirement already satisfied: requests-file>=1.4 in /usr/local/lib/python3.10/dist-packages (from tldextract->presidio-analyzer->-r requirements.txt (line 11)) (3.0.1)\n", + "Using cached accelerate-0.31.0-py3-none-any.whl (309 kB)\n", + "Using cached datasets-2.20.0-py3-none-any.whl (547 kB)\n", + "Using cached huggingface_hub-0.23.4-py3-none-any.whl (402 kB)\n", + "Using cached peft-0.11.1-py3-none-any.whl (251 kB)\n", + "Using cached tokenizers-0.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n", + "Using cached transformers-4.41.2-py3-none-any.whl (9.1 MB)\n", + "Installing collected packages: huggingface_hub, tokenizers, transformers, accelerate, peft, datasets\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6/6\u001b[0m [datasets]5/6\u001b[0m [datasets]e]s]\n", + "\u001b[1A\u001b[2KSuccessfully installed accelerate-0.31.0 datasets-2.20.0 huggingface_hub-0.23.4 peft-0.11.1 tokenizers-0.19.1 transformers-4.41.2\n", + "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n", + "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n", + "Collecting en-core-web-sm==3.8.0\n", + " Downloading en_core_web_sm-3.8.0-py3-none-any.whl (12.8 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.8/12.8 MB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m \u001b[33m0:00:01\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25h\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n", + "You can now load the package via spacy.load('en_core_web_sm')\n", + "Restart the kernel now, then continue from the next cell.\n" + ] + } + ], + "source": [ + "%pip install -U pip\n", + "%pip uninstall -y transformers peft accelerate tokenizers huggingface_hub datasets\n", + "%pip install -r requirements.txt\n", + "!python -m spacy download en_core_web_sm\n", + "print('Restart the kernel now, then continue from the next cell.')" + ] + }, + { + "cell_type": "markdown", + "id": "8e07ef34", + "metadata": {}, + "source": [ + "## 2. Imports and GPU Check" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f86a6c00", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Methods: ('baseline', 'rmft', 'tmft_ner', 'tmft_mia', 'tmft_combined')\n", + "CUDA: True\n", + "GPU: NVIDIA GeForce RTX 3090\n" + ] + } + ], + "source": [ + "import os\n", + "os.environ.setdefault('USE_TF', '0')\n", + "os.environ.setdefault('TRANSFORMERS_NO_TF', '1')\n", + "\n", + "import pandas as pd\n", + "import torch\n", + "from src.data_prep import prepare_experiment_data\n", + "from src.plot_results import plot_results\n", + "from src.train import METHODS, load_config, train_model, upload_to_huggingface\n", + "from main import run_eval\n", + "\n", + "print('Methods:', METHODS)\n", + "print('CUDA:', torch.cuda.is_available())\n", + "if torch.cuda.is_available():\n", + " print('GPU:', torch.cuda.get_device_name(0))" + ] + }, + { + "cell_type": "markdown", + "id": "61c80350", + "metadata": {}, + "source": [ + "## 3. Smoke-Test Data Preparation" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "ec559a58", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using text column: text\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a756688b48e046e8ac2d31789863dd42", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Saving the dataset (0/1 shards): 0%| | 0/63 [00:00\n", + " \n", + " \n", + " [2/2 00:01, Epoch 0/1]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation Loss
0No log2.483602

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved: results/smoke/tmft_ner\n" + ] + } + ], + "source": [ + "trainer, tokenizer, smoke_output = train_model(\n", + " smoke_config,\n", + " method='tmft_ner',\n", + " train_dataset=smoke_splits['train'],\n", + " eval_dataset=smoke_splits['validation'],\n", + ")\n", + "print('Saved:', smoke_output)" + ] + }, + { + "cell_type": "markdown", + "id": "3f595473", + "metadata": {}, + "source": [ + "## 5. Final Data Preparation\n", + "This rebuilds real PII-containing train/validation/test splits and automatically creates `data/pii_eval.json`." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a9748daa", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using text column: text\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ec4a2d9588a145b69848bd343f06ba7b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Saving the dataset (0/1 shards): 0%| | 0/4799 [00:00\n", + " \n", + " \n", + " [750/750 13:15, Epoch 5/5]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation Loss
12.6241002.631289
22.5859002.605364
32.6237002.592506
42.6085002.585280
52.5693002.582293

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== TRAIN: rmft =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", + " warnings.warn(\n", + "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [750/750 13:14, Epoch 5/5]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation Loss
12.6188002.633788
22.5877002.606359
32.6255002.593279
42.6120002.590851
52.5700002.581487

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== TRAIN: tmft_ner =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", + " warnings.warn(\n", + "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [750/750 46:08, Epoch 5/5]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation Loss
12.5930002.692902
22.6093002.665379
32.6374002.652046
42.6134002.644830
52.5772002.641771

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== TRAIN: tmft_mia =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", + " warnings.warn(\n", + "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [750/750 24:44, Epoch 5/5]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation Loss
12.3842002.414698
22.3913002.416788
32.4394002.424283
42.4499002.442038
52.4056002.446106

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== TRAIN: tmft_combined =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", + " warnings.warn(\n", + "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [750/750 56:47, Epoch 5/5]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation Loss
12.4105002.564637
22.4497002.540955
32.5086002.577318
42.4850002.583186
52.4265002.589659

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "text/plain": [ + "{'baseline': 'results/baseline',\n", + " 'rmft': 'results/rmft',\n", + " 'tmft_ner': 'results/tmft_ner',\n", + " 'tmft_mia': 'results/tmft_mia',\n", + " 'tmft_combined': 'results/tmft_combined'}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "methods_to_run = ['baseline', 'rmft', 'tmft_ner', 'tmft_mia', 'tmft_combined']\n", + "train_outputs = {}\n", + "for method in methods_to_run:\n", + " print(f'\\n===== TRAIN: {method} =====')\n", + " trainer, tokenizer, output_dir = train_model(\n", + " final_config,\n", + " method=method,\n", + " train_dataset=final_splits['train'],\n", + " eval_dataset=final_splits['validation'],\n", + " )\n", + " train_outputs[method] = str(output_dir)\n", + "train_outputs" + ] + }, + { + "cell_type": "markdown", + "id": "cf30e805", + "metadata": {}, + "source": [ + "## 7. Evaluate Privacy and Utility\n", + "Computes TER, SER, held-out PPL/MDP, Loss-MIA AUC, and Min-K MIA AUC." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "27da6e57", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== EVAL: baseline =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== EVAL: rmft =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== EVAL: tmft_ner =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== EVAL: tmft_mia =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== EVAL: tmft_combined =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved results: results/tables/main_results.csv\n" + ] + }, + { + "data": { + "text/html": [ + "

\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
methodterserpplloss_mia_aucmin_k_mia_aucmasked_token_ratioskipped_samplespii_eval_samplesmia_samples_per_classmdp
0baseline0.0480.05286313.2255960.5337280.5235200.00000005002500.000000
1rmft0.0460.05066113.2566920.5336000.5234560.14995005002500.031096
2tmft_ner0.0220.02422913.9695170.5304480.5163680.21613428235002500.743922
3tmft_mia0.0240.02643214.8399310.5286560.5135040.248655275002501.614335
4tmft_combined0.0220.02422914.8445370.5285280.5132160.40505567825002501.618942
\n", + "
" + ], + "text/plain": [ + " method ter ser ppl loss_mia_auc min_k_mia_auc \\\n", + "0 baseline 0.048 0.052863 13.225596 0.533728 0.523520 \n", + "1 rmft 0.046 0.050661 13.256692 0.533600 0.523456 \n", + "2 tmft_ner 0.022 0.024229 13.969517 0.530448 0.516368 \n", + "3 tmft_mia 0.024 0.026432 14.839931 0.528656 0.513504 \n", + "4 tmft_combined 0.022 0.024229 14.844537 0.528528 0.513216 \n", + "\n", + " masked_token_ratio skipped_samples pii_eval_samples \\\n", + "0 0.000000 0 500 \n", + "1 0.149950 0 500 \n", + "2 0.216134 2823 500 \n", + "3 0.248655 27 500 \n", + "4 0.405055 6782 500 \n", + "\n", + " mia_samples_per_class mdp \n", + "0 250 0.000000 \n", + "1 250 0.031096 \n", + "2 250 0.743922 \n", + "3 250 1.614335 \n", + "4 250 1.618942 " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results_df = run_eval(final_config, 'all', final_splits, eval_path)\n", + "results_df" + ] + }, + { + "cell_type": "markdown", + "id": "fd9bc206", + "metadata": {}, + "source": [ + "## 8. Generate Submission Figures" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "b6e47be3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Table: results/tables/main_results.csv\n", + "Figures: [PosixPath('results/figures/privacy_utility_tradeoff.png'), PosixPath('results/figures/mia_auc.png'), PosixPath('results/figures/masked_token_ratio.png')]\n" + ] + } + ], + "source": [ + "csv_path = Path(final_config['results_table_dir']) / 'main_results.csv'\n", + "figure_paths = plot_results(csv_path, 'results/figures')\n", + "print('Table:', csv_path)\n", + "print('Figures:', figure_paths)" + ] + }, + { + "cell_type": "markdown", + "id": "8a5d5b78-d4b6-44a9-a6d2-28e861b2b940", + "metadata": {}, + "source": [ + "## 9. Additional Controlled Ablation Experiments\n", + "These runs reuse the existing prepared split and evaluation set. Baseline perplexity is loaded dynamically from `results/tables/main_results.csv`; if that table is missing, the saved baseline checkpoint is evaluated once. Existing ablation adapters are reused so an interrupted run can resume without retraining completed conditions." + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "801de114-ebbb-4c2f-a753-b80f4ae191ea", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Baseline PPL loaded from table: 13.225596\n", + "\n", + "===== rmft_216 =====\n", + "Reusing existing adapter: results/ablations/rmft_216/rmft\n", + "\n", + "===== EVAL: rmft =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved results: results/ablations/rmft_216/tables/main_results.csv\n", + "\n", + "===== ner_strict =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", + " warnings.warn(\n", + "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " \n", + " [750/750 47:08, Epoch 5/5]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation Loss
12.6136002.701755
22.6306002.675479
32.6717002.662506
42.6152002.654978
52.5708002.651994

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== EVAL: tmft_ner =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved results: results/ablations/ner_strict/tables/main_results.csv\n", + "\n", + "===== ner_moderate =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", + " warnings.warn(\n", + "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [750/750 47:20, Epoch 5/5]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation Loss
12.6006002.697284
22.6182002.670427
32.6591002.657601
42.6039002.649750
52.5660002.646821

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== EVAL: tmft_ner =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved results: results/ablations/ner_moderate/tables/main_results.csv\n", + "\n", + "===== mia_p90 =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", + " warnings.warn(\n", + "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [750/750 24:50, Epoch 5/5]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation Loss
12.5216002.532280
22.4989002.521058
32.5367002.510205
42.5299002.511608
52.4969002.507357

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== EVAL: tmft_mia =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved results: results/ablations/mia_p90/tables/main_results.csv\n", + "\n", + "===== combined_p90 =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", + " warnings.warn(\n", + "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [750/750 58:10, Epoch 5/5]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation Loss
12.4922002.631297
22.5440002.610864
32.5887002.601267
42.5512002.602542
52.5276002.599417

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== EVAL: tmft_combined =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved results: results/ablations/combined_p90/tables/main_results.csv\n", + "\n", + "===== combined_p90_skip65 =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", + " warnings.warn(\n", + "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [750/750 58:41, Epoch 5/5]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation Loss
12.4432002.554324
22.4858002.536572
32.5208002.522551
42.4811002.524528
52.4509002.520383

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===== EVAL: tmft_combined =====\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved results: results/ablations/combined_p90_skip65/tables/main_results.csv\n", + "Baseline PPL used for every MDP: 13.225596\n", + "Saved: results/tables/ablation_results.csv\n", + "Saved: results/tables/expanded_results.csv\n" + ] + }, + { + "data": { + "text/html": [ + "

\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
methodterserpplloss_mia_aucmin_k_mia_aucmasked_token_ratioskipped_samplespii_eval_samplesmia_samples_per_classmdpexperiment
0rmft0.0460.05066113.2757520.5331040.5228320.21612005002500.050156rmft_216
1tmft_ner0.0300.03304013.5583140.5322240.5189920.12567517625002500.332719ner_strict
2tmft_ner0.0360.03964813.5915410.5321920.5192800.13660418305002500.365945ner_moderate
3tmft_mia0.0240.02643214.0782620.5321440.5187680.10008905002500.852667mia_p90
4tmft_combined0.0140.01541914.3181130.5310720.5155360.29259837835002501.092517combined_p90
5tmft_combined0.0160.01762114.3018010.5311360.5156000.31685615305002501.076205combined_p90_skip65
\n", + "
" + ], + "text/plain": [ + " method ter ser ppl loss_mia_auc min_k_mia_auc \\\n", + "0 rmft 0.046 0.050661 13.275752 0.533104 0.522832 \n", + "1 tmft_ner 0.030 0.033040 13.558314 0.532224 0.518992 \n", + "2 tmft_ner 0.036 0.039648 13.591541 0.532192 0.519280 \n", + "3 tmft_mia 0.024 0.026432 14.078262 0.532144 0.518768 \n", + "4 tmft_combined 0.014 0.015419 14.318113 0.531072 0.515536 \n", + "5 tmft_combined 0.016 0.017621 14.301801 0.531136 0.515600 \n", + "\n", + " masked_token_ratio skipped_samples pii_eval_samples \\\n", + "0 0.216120 0 500 \n", + "1 0.125675 1762 500 \n", + "2 0.136604 1830 500 \n", + "3 0.100089 0 500 \n", + "4 0.292598 3783 500 \n", + "5 0.316856 1530 500 \n", + "\n", + " mia_samples_per_class mdp experiment \n", + "0 250 0.050156 rmft_216 \n", + "1 250 0.332719 ner_strict \n", + "2 250 0.365945 ner_moderate \n", + "3 250 0.852667 mia_p90 \n", + "4 250 1.092517 combined_p90 \n", + "5 250 1.076205 combined_p90_skip65 " + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from copy import deepcopy\n", + "import gc\n", + "from pathlib import Path\n", + "\n", + "import pandas as pd\n", + "import torch\n", + "from datasets import load_from_disk\n", + "\n", + "from src.train import load_config, train_model\n", + "from main import run_eval\n", + "\n", + "# Restore state safely after a kernel restart.\n", + "if 'final_config' not in globals():\n", + " final_config = load_config('configs/config.yaml')\n", + "if 'final_splits' not in globals():\n", + " final_splits = load_from_disk(final_config.get('prepared_data_dir', 'data/processed'))\n", + "if 'eval_path' not in globals():\n", + " eval_path = Path(final_config.get('pii_eval_path', 'data/pii_eval.json'))\n", + "\n", + "final_config['text_column'] = 'text'\n", + "main_results_path = Path('results/tables/main_results.csv')\n", + "\n", + "# Dynamic baseline: read the original result table when available.\n", + "if main_results_path.exists():\n", + " original_results = pd.read_csv(main_results_path)\n", + " baseline_rows = original_results.loc[original_results['method'] == 'baseline', 'ppl']\n", + " if baseline_rows.empty:\n", + " raise ValueError(f'Baseline row is missing from {main_results_path}')\n", + " baseline_ppl = float(baseline_rows.iloc[0])\n", + " print(f'Baseline PPL loaded from table: {baseline_ppl:.6f}')\n", + "else:\n", + " # Re-evaluate the saved baseline with the same validation split.\n", + " baseline_cfg = deepcopy(final_config)\n", + " baseline_cfg['results_table_dir'] = './results/ablations/baseline_reference/tables'\n", + " baseline_model_dir = Path(final_config.get('output_dir', './results')) / 'baseline'\n", + " if not baseline_model_dir.exists():\n", + " raise FileNotFoundError(f'Baseline checkpoint not found: {baseline_model_dir}')\n", + " baseline_result = run_eval(\n", + " baseline_cfg,\n", + " method='baseline',\n", + " splits=final_splits,\n", + " eval_path=eval_path,\n", + " model_dir=str(baseline_model_dir),\n", + " )\n", + " baseline_ppl = float(baseline_result['ppl'].iloc[0])\n", + " original_results = baseline_result.copy()\n", + " print(f'Baseline PPL re-evaluated: {baseline_ppl:.6f}')\n", + "\n", + "experiments = [\n", + " {\n", + " 'name': 'rmft_216',\n", + " 'method': 'rmft',\n", + " 'params': {'rmft_probability': 0.2161},\n", + " },\n", + " {\n", + " 'name': 'ner_strict',\n", + " 'method': 'tmft_ner',\n", + " 'params': {'ner_labels': ['PERSON', 'EMAIL', 'PHONE']},\n", + " },\n", + " {\n", + " 'name': 'ner_moderate',\n", + " 'method': 'tmft_ner',\n", + " 'params': {'ner_labels': ['PERSON', 'EMAIL', 'PHONE', 'GPE', 'LOC']},\n", + " },\n", + " {\n", + " 'name': 'mia_p90',\n", + " 'method': 'tmft_mia',\n", + " 'params': {'threshold_percentile': 90},\n", + " },\n", + " {\n", + " 'name': 'combined_p90',\n", + " 'method': 'tmft_combined',\n", + " 'params': {'threshold_percentile': 90, 'max_mask_ratio': 0.50},\n", + " },\n", + " {\n", + " 'name': 'combined_p90_skip65',\n", + " 'method': 'tmft_combined',\n", + " 'params': {'threshold_percentile': 90, 'max_mask_ratio': 0.65},\n", + " },\n", + "]\n", + "\n", + "ablation_results = []\n", + "\n", + "for experiment in experiments:\n", + " name = experiment['name']\n", + " method = experiment['method']\n", + " cfg = deepcopy(final_config)\n", + " cfg.update(experiment['params'])\n", + " cfg['seed'] = 42\n", + " cfg['text_column'] = 'text'\n", + " cfg['output_dir'] = f'./results/ablations/{name}'\n", + " cfg['results_table_dir'] = f'./results/ablations/{name}/tables'\n", + "\n", + " expected_model_dir = Path(cfg['output_dir']) / method\n", + " print(f'\\n===== {name} =====')\n", + "\n", + " if (expected_model_dir / 'adapter_config.json').exists():\n", + " model_dir = expected_model_dir\n", + " print(f'Reusing existing adapter: {model_dir}')\n", + " else:\n", + " trainer, tokenizer, model_dir = train_model(\n", + " cfg,\n", + " method=method,\n", + " train_dataset=final_splits['train'],\n", + " eval_dataset=final_splits['validation'],\n", + " )\n", + " # Free the training model before loading a fresh evaluation model.\n", + " del trainer, tokenizer\n", + " gc.collect()\n", + " if torch.cuda.is_available():\n", + " torch.cuda.empty_cache()\n", + "\n", + " result = run_eval(\n", + " cfg,\n", + " method=method,\n", + " splits=final_splits,\n", + " eval_path=eval_path,\n", + " model_dir=str(model_dir),\n", + " )\n", + " result['experiment'] = name\n", + " result['mdp'] = result['ppl'] - baseline_ppl\n", + " ablation_results.append(result)\n", + "\n", + "ablation_df = pd.concat(ablation_results, ignore_index=True)\n", + "\n", + "output_dir = Path('results/tables')\n", + "output_dir.mkdir(parents=True, exist_ok=True)\n", + "ablation_path = output_dir / 'ablation_results.csv'\n", + "ablation_df.to_csv(ablation_path, index=False)\n", + "\n", + "# Also save a single comparison table containing the original five conditions.\n", + "original_comparison = original_results.copy()\n", + "original_comparison['experiment'] = original_comparison['method'].astype(str) + '_original'\n", + "expanded_results = pd.concat([original_comparison, ablation_df], ignore_index=True, sort=False)\n", + "expanded_path = output_dir / 'expanded_results.csv'\n", + "expanded_results.to_csv(expanded_path, index=False)\n", + "\n", + "print(f'Baseline PPL used for every MDP: {baseline_ppl:.6f}')\n", + "print(f'Saved: {ablation_path}')\n", + "print(f'Saved: {expanded_path}')\n", + "ablation_df" + ] + }, + { + "cell_type": "markdown", + "id": "cfba4323", + "metadata": {}, + "source": [ + "## 10. Hugging Face Upload\n", + "Log in with `huggingface-cli login` first. Upload one validated checkpoint at a time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b993fcba-5859-4151-96ff-9dcb5c654619", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Create token: https://huggingface.co/settings/tokens\n" + ] + } + ], + "source": [ + "# Hugging Face login (prompt, hidden input)\n", + "import os\n", + "from getpass import getpass\n", + "\n", + "try:\n", + " from huggingface_hub import login\n", + "except Exception:\n", + " %pip install -U huggingface_hub\n", + " from huggingface_hub import login\n", + "\n", + "print('Create token: https://huggingface.co/settings/tokens')\n", + "\n", + "env_token = (os.environ.get('HF_TOKEN') or '').strip()\n", + "if env_token:\n", + " use_env = input('HF_TOKEN is set. Use it? [Y/n]: ').strip().lower()\n", + " token = env_token if use_env in ('', 'y', 'yes') else getpass('Enter HF token: ').strip()\n", + "else:\n", + " token = getpass('Enter HF token: ').strip()\n", + "\n", + "if not token:\n", + " raise ValueError('No token provided.')\n", + "\n", + "login(token=token)\n", + "os.environ['HF_TOKEN'] = token\n", + "print('Hugging Face login OK.')" + ] + }, + { + "cell_type": "markdown", + "id": "91aece4a-0e82-419a-ba3f-cf07beca9cbe", + "metadata": {}, + "source": [ + "## 11.Upload Model, Code, and Results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a29ad15c-b5c1-439f-aa81-1aab0101dd9c", + "metadata": {}, + "outputs": [], + "source": [ + "from huggingface_hub import HfApi\n", + "from pathlib import Path\n", + "\n", + "HF_REPO_ID = input('Repository ID (username/repository): ').strip()\n", + "PRIVATE_REPO = True # Change to False only after reviewing every uploaded artifact.\n", + "\n", + "if '/' not in HF_REPO_ID:\n", + " raise ValueError('Use the form username/repository')\n", + "\n", + "project_root = Path.cwd()\n", + "trained_paths = globals().get('train_outputs') or {\n", + " method: str(project_root / 'results' / method)\n", + " for method in ['baseline', 'rmft', 'tmft_ner', 'tmft_mia', 'tmft_combined']\n", + "}\n", + "primary_model_dir = Path(trained_paths.get('tmft_combined', 'results/tmft_combined'))\n", + "if not (primary_model_dir / 'adapter_config.json').exists():\n", + " raise FileNotFoundError(f'LoRA adapter not found: {primary_model_dir}')\n", + "\n", + "api = HfApi(token=os.environ['HF_TOKEN'])\n", + "api.create_repo(\n", + " repo_id=HF_REPO_ID,\n", + " repo_type='model',\n", + " private=PRIVATE_REPO,\n", + " exist_ok=True,\n", + ")\n", + "\n", + "# Root remains directly loadable as a PEFT model repository.\n", + "api.upload_folder(\n", + " repo_id=HF_REPO_ID,\n", + " repo_type='model',\n", + " folder_path=str(primary_model_dir),\n", + " path_in_repo='',\n", + " ignore_patterns=['checkpoint-*', '*.pt', 'optimizer.pt', 'scheduler.pt', 'rng_state.pth'],\n", + ")\n", + "\n", + "readme_path = project_root / 'README.md'\n", + "if readme_path.exists():\n", + " api.upload_file(\n", + " repo_id=HF_REPO_ID, repo_type='model', path_or_fileobj=str(readme_path), path_in_repo='README.md'\n", + " )\n", + "\n", + "for filename in ['requirements.txt', 'main.py', 'tmft_experiment.ipynb']:\n", + " file_path = project_root / filename\n", + " if file_path.exists():\n", + " api.upload_file(\n", + " repo_id=HF_REPO_ID,\n", + " repo_type='model',\n", + " path_or_fileobj=str(file_path),\n", + " path_in_repo=f'project/{filename}',\n", + " )\n", + "\n", + "for local_dir, repo_dir in [\n", + " ('src', 'project/src'),\n", + " ('configs', 'project/configs'),\n", + " ('results/tables', 'experiment/tables'),\n", + " ('results/figures', 'experiment/figures'),\n", + "]:\n", + " folder = project_root / local_dir\n", + " if folder.exists():\n", + " api.upload_folder(\n", + " repo_id=HF_REPO_ID,\n", + " repo_type='model',\n", + " folder_path=str(folder),\n", + " path_in_repo=repo_dir,\n", + " ignore_patterns=['__pycache__', '*.pyc', '.ipynb_checkpoints'],\n", + " )\n", + "\n", + "# Small LoRA comparison adapters make the five-condition experiment reproducible.\n", + "for method, model_path in trained_paths.items():\n", + " if method == 'tmft_combined':\n", + " continue\n", + " model_path = Path(model_path)\n", + " if (model_path / 'adapter_config.json').exists():\n", + " api.upload_folder(\n", + " repo_id=HF_REPO_ID,\n", + " repo_type='model',\n", + " folder_path=str(model_path),\n", + " path_in_repo=f'experiment/adapters/{method}',\n", + " ignore_patterns=['checkpoint-*', '*.pt', 'optimizer.pt', 'scheduler.pt', 'rng_state.pth'],\n", + " )\n", + "\n", + "print(f'Upload complete: https://huggingface.co/{HF_REPO_ID}')\n", + "print('Raw data and PII evaluation records were not uploaded.')" + ] + }, + { + "cell_type": "markdown", + "id": "994ac451-8446-48ea-9715-ff37047dba7e", + "metadata": {}, + "source": [ + "## 12. Download the Complete Vessel Project" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3e0fc62-21d8-4f26-ad82-2ee1612851e6", + "metadata": {}, + "outputs": [], + "source": [ + "import shutil\n", + "from pathlib import Path\n", + "from IPython.display import FileLink, display\n", + "\n", + "project_root = Path.cwd().resolve()\n", + "archive_destination = project_root / f'{project_root.name}_complete.zip'\n", + "archive_destination.unlink(missing_ok=True)\n", + "\n", + "temporary_base = Path('/tmp') / f'{project_root.name}_complete'\n", + "temporary_zip = Path(shutil.make_archive(\n", + " str(temporary_base),\n", + " 'zip',\n", + " root_dir=str(project_root.parent),\n", + " base_dir=project_root.name,\n", + "))\n", + "shutil.move(str(temporary_zip), str(archive_destination))\n", + "\n", + "size_gb = archive_destination.stat().st_size / (1024 ** 3)\n", + "print(f'Archive ready: {archive_destination} ({size_gb:.2f} GB)')\n", + "print('Keep this local backup private; it may contain PII.')\n", + "display(FileLink(archive_destination.name, result_html_prefix='Download complete project: '))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}