{ "cells": [ { "cell_type": "markdown", "id": "78c489b6", "metadata": {}, "source": [ "# TMFT End-to-End Experiment\n", "\n", "Run this notebook from the uploaded `tmft_project/` directory. It prepares real PII-containing Enron splits, trains all five methods, and computes TER, SER, perplexity, MDP, and two MIA AUC metrics." ] }, { "cell_type": "markdown", "id": "305ebc86", "metadata": {}, "source": [ "## 0. Verify Project Structure" ] }, { "cell_type": "code", "execution_count": 1, "id": "ce28e817", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Project: /home/TMFT\n", "Python: 3.10.12 (main, Jul 29 2024, 16:56:48) [GCC 11.4.0]\n", "Platform: Linux-5.4.0-193-generic-x86_64-with-glibc2.35\n" ] } ], "source": [ "from pathlib import Path\n", "import os, sys, platform\n", "\n", "PROJECT_ROOT = Path.cwd()\n", "required = [\n", " 'configs/config.yaml', 'src/data_prep.py', 'src/train.py',\n", " 'src/masking.py', 'src/evaluate_pii.py', 'src/evaluate_ppl.py',\n", " 'src/evaluate_mia.py', 'src/plot_results.py', 'main.py', 'requirements.txt'\n", "]\n", "missing = [path for path in required if not (PROJECT_ROOT / path).exists()]\n", "if missing:\n", " raise FileNotFoundError('Open the notebook inside the full tmft_project directory. Missing: ' + ', '.join(missing))\n", "print('Project:', PROJECT_ROOT)\n", "print('Python:', sys.version)\n", "print('Platform:', platform.platform())" ] }, { "cell_type": "markdown", "id": "2e0f5f9e", "metadata": {}, "source": [ "## 1. Install Dependencies\n", "Run once in a fresh Vessel workspace, then restart the kernel." ] }, { "cell_type": "code", "execution_count": 2, "id": "7a72e18d", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: pip in /usr/local/lib/python3.10/dist-packages (26.1.2)\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n", "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n", "Found existing installation: transformers 4.41.2\n", "Uninstalling transformers-4.41.2:\n", " Successfully uninstalled transformers-4.41.2\n", "Found existing installation: peft 0.11.1\n", "Uninstalling peft-0.11.1:\n", " Successfully uninstalled peft-0.11.1\n", "Found existing installation: accelerate 0.31.0\n", "Uninstalling accelerate-0.31.0:\n", " Successfully uninstalled accelerate-0.31.0\n", "Found existing installation: tokenizers 0.19.1\n", "Uninstalling tokenizers-0.19.1:\n", " Successfully uninstalled tokenizers-0.19.1\n", "Found existing installation: huggingface-hub 0.23.4\n", "Uninstalling huggingface-hub-0.23.4:\n", " Successfully uninstalled huggingface-hub-0.23.4\n", "Found existing installation: datasets 2.20.0\n", "Uninstalling datasets-2.20.0:\n", " Successfully uninstalled datasets-2.20.0\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n", "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n", "Collecting accelerate==0.31.0 (from -r requirements.txt (line 1))\n", " Using cached accelerate-0.31.0-py3-none-any.whl.metadata (19 kB)\n", "Collecting datasets==2.20.0 (from -r requirements.txt (line 2))\n", " Using cached datasets-2.20.0-py3-none-any.whl.metadata (19 kB)\n", "Requirement already satisfied: evaluate in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 3)) (0.4.6)\n", "Collecting huggingface_hub==0.23.4 (from -r requirements.txt (line 4))\n", " Using cached huggingface_hub-0.23.4-py3-none-any.whl.metadata (12 kB)\n", "Requirement already satisfied: ipykernel in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 5)) (6.29.5)\n", "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 6)) (3.9.2)\n", "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 7)) (3.9.4)\n", "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 8)) (1.24.4)\n", "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 9)) (2.2.2)\n", "Collecting peft==0.11.1 (from -r requirements.txt (line 10))\n", " Using cached peft-0.11.1-py3-none-any.whl.metadata (13 kB)\n", "Requirement already satisfied: presidio-analyzer in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 11)) (2.2.362)\n", "Requirement already satisfied: presidio-anonymizer in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 12)) (2.2.362)\n", "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 13)) (6.0.3)\n", "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 14)) (2026.5.9)\n", "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 15)) (1.5.1)\n", "Requirement already satisfied: seaborn in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 16)) (0.13.2)\n", "Requirement already satisfied: spacy in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 17)) (3.8.14)\n", "Collecting tokenizers==0.19.1 (from -r requirements.txt (line 18))\n", " Using cached tokenizers-0.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n", "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 19)) (4.66.5)\n", "Collecting transformers==4.41.2 (from -r requirements.txt (line 20))\n", " Using cached transformers-4.41.2-py3-none-any.whl.metadata (43 kB)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate==0.31.0->-r requirements.txt (line 1)) (24.1)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate==0.31.0->-r requirements.txt (line 1)) (5.9.8)\n", "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate==0.31.0->-r requirements.txt (line 1)) (2.3.1+cu121)\n", "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate==0.31.0->-r requirements.txt (line 1)) (0.8.0)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (3.13.1)\n", "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (24.0.0)\n", "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (0.7)\n", "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (0.3.8)\n", "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (2.32.3)\n", "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (3.7.0)\n", "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (0.70.16)\n", "Requirement already satisfied: fsspec<=2024.5.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.5.0,>=2023.1.0->datasets==2.20.0->-r requirements.txt (line 2)) (2024.2.0)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets==2.20.0->-r requirements.txt (line 2)) (3.14.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub==0.23.4->-r requirements.txt (line 4)) (4.15.0)\n", "Requirement already satisfied: comm>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (0.2.2)\n", "Requirement already satisfied: debugpy>=1.6.5 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (1.8.5)\n", "Requirement already satisfied: ipython>=7.23.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (8.26.0)\n", "Requirement already satisfied: jupyter-client>=6.1.12 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (8.6.2)\n", "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (5.7.2)\n", "Requirement already satisfied: matplotlib-inline>=0.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (0.1.7)\n", "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (1.6.0)\n", "Requirement already satisfied: pyzmq>=24 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (26.1.0)\n", "Requirement already satisfied: tornado>=6.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (6.4.1)\n", "Requirement already satisfied: traitlets>=5.4.0 in /usr/local/lib/python3.10/dist-packages (from ipykernel->-r requirements.txt (line 5)) (5.14.3)\n", "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 6)) (1.2.1)\n", "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 6)) (0.12.1)\n", "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 6)) (4.53.1)\n", "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 6)) (1.4.5)\n", "Requirement already satisfied: pillow>=8 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 6)) (10.4.0)\n", "Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3/dist-packages (from matplotlib->-r requirements.txt (line 6)) (2.4.7)\n", "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r requirements.txt (line 6)) (2.9.0.post0)\n", "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->-r requirements.txt (line 7)) (8.4.1)\n", "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->-r requirements.txt (line 7)) (1.4.2)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->-r requirements.txt (line 9)) (2024.1)\n", "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->-r requirements.txt (line 9)) (2024.1)\n", "Requirement already satisfied: phonenumbers<10.0.0,>=8.12 in /usr/local/lib/python3.10/dist-packages (from presidio-analyzer->-r requirements.txt (line 11)) (9.0.32)\n", "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from presidio-analyzer->-r requirements.txt (line 11)) (2.8.2)\n", "Requirement already satisfied: tldextract in /usr/local/lib/python3.10/dist-packages (from presidio-analyzer->-r requirements.txt (line 11)) (5.3.1)\n", "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.0.0->presidio-analyzer->-r requirements.txt (line 11)) (0.7.0)\n", "Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.0.0->presidio-analyzer->-r requirements.txt (line 11)) (2.20.1)\n", "Requirement already satisfied: cryptography>=46.0.4 in /usr/local/lib/python3.10/dist-packages (from presidio-anonymizer->-r requirements.txt (line 12)) (49.0.0)\n", "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->-r requirements.txt (line 15)) (1.14.0)\n", "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->-r requirements.txt (line 15)) (3.5.0)\n", "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (3.0.12)\n", "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (1.0.5)\n", "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (1.0.15)\n", "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (2.0.13)\n", "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (3.0.13)\n", "Requirement already satisfied: thinc<8.4.0,>=8.3.12 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (8.3.13)\n", "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (1.1.3)\n", "Requirement already satisfied: srsly<3.0.0,>=2.5.3 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (2.5.3)\n", "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (2.0.10)\n", "Requirement already satisfied: weasel<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (1.0.0)\n", "Requirement already satisfied: confection<2.0.0,>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (1.3.3)\n", "Requirement already satisfied: typer<1.0.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (0.25.1)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (3.1.4)\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy->-r requirements.txt (line 17)) (72.1.0)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets==2.20.0->-r requirements.txt (line 2)) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets==2.20.0->-r requirements.txt (line 2)) (3.7)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets==2.20.0->-r requirements.txt (line 2)) (2.2.2)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets==2.20.0->-r requirements.txt (line 2)) (2024.7.4)\n", "Requirement already satisfied: blis<1.4.0,>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from thinc<8.4.0,>=8.3.12->spacy->-r requirements.txt (line 17)) (1.3.3)\n", "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0.0,>=0.3.0->spacy->-r requirements.txt (line 17)) (1.5.4)\n", "Requirement already satisfied: rich>=13.8.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0.0,>=0.3.0->spacy->-r requirements.txt (line 17)) (15.0.0)\n", "Requirement already satisfied: annotated-doc>=0.0.2 in /usr/local/lib/python3.10/dist-packages (from typer<1.0.0,>=0.3.0->spacy->-r requirements.txt (line 17)) (0.0.4)\n", "Requirement already satisfied: cloudpathlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (0.24.0)\n", "Requirement already satisfied: smart-open>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (7.6.1)\n", "Requirement already satisfied: httpx>=0.24.0 in /usr/local/lib/python3.10/dist-packages (from weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (0.27.0)\n", "Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (2.6.2)\n", "Requirement already satisfied: aiosignal>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (1.4.0)\n", "Requirement already satisfied: async-timeout<6.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (5.0.1)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (24.2.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (1.8.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (6.7.1)\n", "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (0.5.2)\n", "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.20.0->-r requirements.txt (line 2)) (1.24.2)\n", "Requirement already satisfied: cffi>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from cryptography>=46.0.4->presidio-anonymizer->-r requirements.txt (line 12)) (2.0.0)\n", "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=2.0.0->cryptography>=46.0.4->presidio-anonymizer->-r requirements.txt (line 12)) (2.22)\n", "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.0->weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (3.7.1)\n", "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.0->weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (1.0.5)\n", "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.0->weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (1.3.1)\n", "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx>=0.24.0->weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (0.14.0)\n", "Requirement already satisfied: decorator in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (5.1.1)\n", "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (0.19.1)\n", "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (3.0.47)\n", "Requirement already satisfied: pygments>=2.4.0 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (2.18.0)\n", "Requirement already satisfied: stack-data in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (0.6.3)\n", "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (1.2.2)\n", "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (4.9.0)\n", "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (0.2.13)\n", "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /usr/local/lib/python3.10/dist-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (0.8.4)\n", "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.10/dist-packages (from jupyter-core!=5.0.*,>=4.12->ipykernel->-r requirements.txt (line 5)) (4.2.2)\n", "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/dist-packages (from pexpect>4.3->ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (0.7.0)\n", "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib->-r requirements.txt (line 6)) (1.16.0)\n", "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=13.8.0->typer<1.0.0,>=0.3.0->spacy->-r requirements.txt (line 17)) (4.2.0)\n", "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=13.8.0->typer<1.0.0,>=0.3.0->spacy->-r requirements.txt (line 17)) (0.1.2)\n", "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from smart-open>=5.2.1->weasel<2.0.0,>=1.0.0->spacy->-r requirements.txt (line 17)) (2.2.1)\n", "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (1.12)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (3.2.1)\n", "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.105)\n", "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (8.9.2.26)\n", "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.3.1)\n", "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (11.0.2.54)\n", "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (10.3.2.106)\n", "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (11.4.5.107)\n", "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.0.106)\n", "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (2.20.5)\n", "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.105)\n", "Requirement already satisfied: triton==2.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (2.3.1)\n", "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (12.1.105)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy->-r requirements.txt (line 17)) (2.1.5)\n", "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from stack-data->ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (2.0.1)\n", "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.10/dist-packages (from stack-data->ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (2.4.1)\n", "Requirement already satisfied: pure-eval in /usr/local/lib/python3.10/dist-packages (from stack-data->ipython>=7.23.1->ipykernel->-r requirements.txt (line 5)) (0.2.3)\n", "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate==0.31.0->-r requirements.txt (line 1)) (1.3.0)\n", "Requirement already satisfied: requests-file>=1.4 in /usr/local/lib/python3.10/dist-packages (from tldextract->presidio-analyzer->-r requirements.txt (line 11)) (3.0.1)\n", "Using cached accelerate-0.31.0-py3-none-any.whl (309 kB)\n", "Using cached datasets-2.20.0-py3-none-any.whl (547 kB)\n", "Using cached huggingface_hub-0.23.4-py3-none-any.whl (402 kB)\n", "Using cached peft-0.11.1-py3-none-any.whl (251 kB)\n", "Using cached tokenizers-0.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n", "Using cached transformers-4.41.2-py3-none-any.whl (9.1 MB)\n", "Installing collected packages: huggingface_hub, tokenizers, transformers, accelerate, peft, datasets\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6/6\u001b[0m [datasets]5/6\u001b[0m [datasets]e]s]\n", "\u001b[1A\u001b[2KSuccessfully installed accelerate-0.31.0 datasets-2.20.0 huggingface_hub-0.23.4 peft-0.11.1 tokenizers-0.19.1 transformers-4.41.2\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n", "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n", "Collecting en-core-web-sm==3.8.0\n", " Downloading en_core_web_sm-3.8.0-py3-none-any.whl (12.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.8/12.8 MB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m \u001b[33m0:00:01\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25h\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n", "\u001b[0m\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n", "You can now load the package via spacy.load('en_core_web_sm')\n", "Restart the kernel now, then continue from the next cell.\n" ] } ], "source": [ "%pip install -U pip\n", "%pip uninstall -y transformers peft accelerate tokenizers huggingface_hub datasets\n", "%pip install -r requirements.txt\n", "!python -m spacy download en_core_web_sm\n", "print('Restart the kernel now, then continue from the next cell.')" ] }, { "cell_type": "markdown", "id": "8e07ef34", "metadata": {}, "source": [ "## 2. Imports and GPU Check" ] }, { "cell_type": "code", "execution_count": 3, "id": "f86a6c00", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Methods: ('baseline', 'rmft', 'tmft_ner', 'tmft_mia', 'tmft_combined')\n", "CUDA: True\n", "GPU: NVIDIA GeForce RTX 3090\n" ] } ], "source": [ "import os\n", "os.environ.setdefault('USE_TF', '0')\n", "os.environ.setdefault('TRANSFORMERS_NO_TF', '1')\n", "\n", "import pandas as pd\n", "import torch\n", "from src.data_prep import prepare_experiment_data\n", "from src.plot_results import plot_results\n", "from src.train import METHODS, load_config, train_model, upload_to_huggingface\n", "from main import run_eval\n", "\n", "print('Methods:', METHODS)\n", "print('CUDA:', torch.cuda.is_available())\n", "if torch.cuda.is_available():\n", " print('GPU:', torch.cuda.get_device_name(0))" ] }, { "cell_type": "markdown", "id": "61c80350", "metadata": {}, "source": [ "## 3. Smoke-Test Data Preparation" ] }, { "cell_type": "code", "execution_count": 4, "id": "ec559a58", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Using text column: text\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "a756688b48e046e8ac2d31789863dd42", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Saving the dataset (0/1 shards): 0%| | 0/63 [00:00\n", " \n", " \n", " [2/2 00:01, Epoch 0/1]\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
0No log2.483602

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Saved: results/smoke/tmft_ner\n" ] } ], "source": [ "trainer, tokenizer, smoke_output = train_model(\n", " smoke_config,\n", " method='tmft_ner',\n", " train_dataset=smoke_splits['train'],\n", " eval_dataset=smoke_splits['validation'],\n", ")\n", "print('Saved:', smoke_output)" ] }, { "cell_type": "markdown", "id": "3f595473", "metadata": {}, "source": [ "## 5. Final Data Preparation\n", "This rebuilds real PII-containing train/validation/test splits and automatically creates `data/pii_eval.json`." ] }, { "cell_type": "code", "execution_count": 6, "id": "a9748daa", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Using text column: text\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ec4a2d9588a145b69848bd343f06ba7b", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Saving the dataset (0/1 shards): 0%| | 0/4799 [00:00\n", " \n", " \n", " [750/750 13:15, Epoch 5/5]\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
12.6241002.631289
22.5859002.605364
32.6237002.592506
42.6085002.585280
52.5693002.582293

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== TRAIN: rmft =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", " warnings.warn(\n", "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [750/750 13:14, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
12.6188002.633788
22.5877002.606359
32.6255002.593279
42.6120002.590851
52.5700002.581487

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== TRAIN: tmft_ner =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", " warnings.warn(\n", "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [750/750 46:08, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
12.5930002.692902
22.6093002.665379
32.6374002.652046
42.6134002.644830
52.5772002.641771

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== TRAIN: tmft_mia =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", " warnings.warn(\n", "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [750/750 24:44, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
12.3842002.414698
22.3913002.416788
32.4394002.424283
42.4499002.442038
52.4056002.446106

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== TRAIN: tmft_combined =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", " warnings.warn(\n", "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [750/750 56:47, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
12.4105002.564637
22.4497002.540955
32.5086002.577318
42.4850002.583186
52.4265002.589659

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n" ] }, { "data": { "text/plain": [ "{'baseline': 'results/baseline',\n", " 'rmft': 'results/rmft',\n", " 'tmft_ner': 'results/tmft_ner',\n", " 'tmft_mia': 'results/tmft_mia',\n", " 'tmft_combined': 'results/tmft_combined'}" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "methods_to_run = ['baseline', 'rmft', 'tmft_ner', 'tmft_mia', 'tmft_combined']\n", "train_outputs = {}\n", "for method in methods_to_run:\n", " print(f'\\n===== TRAIN: {method} =====')\n", " trainer, tokenizer, output_dir = train_model(\n", " final_config,\n", " method=method,\n", " train_dataset=final_splits['train'],\n", " eval_dataset=final_splits['validation'],\n", " )\n", " train_outputs[method] = str(output_dir)\n", "train_outputs" ] }, { "cell_type": "markdown", "id": "cf30e805", "metadata": {}, "source": [ "## 7. Evaluate Privacy and Utility\n", "Computes TER, SER, held-out PPL/MDP, Loss-MIA AUC, and Min-K MIA AUC." ] }, { "cell_type": "code", "execution_count": 8, "id": "27da6e57", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== EVAL: baseline =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== EVAL: rmft =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== EVAL: tmft_ner =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== EVAL: tmft_mia =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== EVAL: tmft_combined =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Saved results: results/tables/main_results.csv\n" ] }, { "data": { "text/html": [ "

\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
methodterserpplloss_mia_aucmin_k_mia_aucmasked_token_ratioskipped_samplespii_eval_samplesmia_samples_per_classmdp
0baseline0.0480.05286313.2255960.5337280.5235200.00000005002500.000000
1rmft0.0460.05066113.2566920.5336000.5234560.14995005002500.031096
2tmft_ner0.0220.02422913.9695170.5304480.5163680.21613428235002500.743922
3tmft_mia0.0240.02643214.8399310.5286560.5135040.248655275002501.614335
4tmft_combined0.0220.02422914.8445370.5285280.5132160.40505567825002501.618942
\n", "
" ], "text/plain": [ " method ter ser ppl loss_mia_auc min_k_mia_auc \\\n", "0 baseline 0.048 0.052863 13.225596 0.533728 0.523520 \n", "1 rmft 0.046 0.050661 13.256692 0.533600 0.523456 \n", "2 tmft_ner 0.022 0.024229 13.969517 0.530448 0.516368 \n", "3 tmft_mia 0.024 0.026432 14.839931 0.528656 0.513504 \n", "4 tmft_combined 0.022 0.024229 14.844537 0.528528 0.513216 \n", "\n", " masked_token_ratio skipped_samples pii_eval_samples \\\n", "0 0.000000 0 500 \n", "1 0.149950 0 500 \n", "2 0.216134 2823 500 \n", "3 0.248655 27 500 \n", "4 0.405055 6782 500 \n", "\n", " mia_samples_per_class mdp \n", "0 250 0.000000 \n", "1 250 0.031096 \n", "2 250 0.743922 \n", "3 250 1.614335 \n", "4 250 1.618942 " ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results_df = run_eval(final_config, 'all', final_splits, eval_path)\n", "results_df" ] }, { "cell_type": "markdown", "id": "fd9bc206", "metadata": {}, "source": [ "## 8. Generate Submission Figures" ] }, { "cell_type": "code", "execution_count": 9, "id": "b6e47be3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Table: results/tables/main_results.csv\n", "Figures: [PosixPath('results/figures/privacy_utility_tradeoff.png'), PosixPath('results/figures/mia_auc.png'), PosixPath('results/figures/masked_token_ratio.png')]\n" ] } ], "source": [ "csv_path = Path(final_config['results_table_dir']) / 'main_results.csv'\n", "figure_paths = plot_results(csv_path, 'results/figures')\n", "print('Table:', csv_path)\n", "print('Figures:', figure_paths)" ] }, { "cell_type": "markdown", "id": "8a5d5b78-d4b6-44a9-a6d2-28e861b2b940", "metadata": {}, "source": [ "## 9. Additional Controlled Ablation Experiments\n", "These runs reuse the existing prepared split and evaluation set. Baseline perplexity is loaded dynamically from `results/tables/main_results.csv`; if that table is missing, the saved baseline checkpoint is evaluated once. Existing ablation adapters are reused so an interrupted run can resume without retraining completed conditions." ] }, { "cell_type": "code", "execution_count": 33, "id": "801de114-ebbb-4c2f-a753-b80f4ae191ea", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Baseline PPL loaded from table: 13.225596\n", "\n", "===== rmft_216 =====\n", "Reusing existing adapter: results/ablations/rmft_216/rmft\n", "\n", "===== EVAL: rmft =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Saved results: results/ablations/rmft_216/tables/main_results.csv\n", "\n", "===== ner_strict =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", " warnings.warn(\n", "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [750/750 47:08, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
12.6136002.701755
22.6306002.675479
32.6717002.662506
42.6152002.654978
52.5708002.651994

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== EVAL: tmft_ner =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Saved results: results/ablations/ner_strict/tables/main_results.csv\n", "\n", "===== ner_moderate =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", " warnings.warn(\n", "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [750/750 47:20, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
12.6006002.697284
22.6182002.670427
32.6591002.657601
42.6039002.649750
52.5660002.646821

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== EVAL: tmft_ner =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Saved results: results/ablations/ner_moderate/tables/main_results.csv\n", "\n", "===== mia_p90 =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", " warnings.warn(\n", "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [750/750 24:50, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
12.5216002.532280
22.4989002.521058
32.5367002.510205
42.5299002.511608
52.4969002.507357

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== EVAL: tmft_mia =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Saved results: results/ablations/mia_p90/tables/main_results.csv\n", "\n", "===== combined_p90 =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", " warnings.warn(\n", "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [750/750 58:10, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
12.4922002.631297
22.5440002.610864
32.5887002.601267
42.5512002.602542
52.5276002.599417

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== EVAL: tmft_combined =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Saved results: results/ablations/combined_p90/tables/main_results.csv\n", "\n", "===== combined_p90_skip65 =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", " warnings.warn(\n", "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [750/750 58:41, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
12.4432002.554324
22.4858002.536572
32.5208002.522551
42.4811002.524528
52.4509002.520383

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "===== EVAL: tmft_combined =====\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Saved results: results/ablations/combined_p90_skip65/tables/main_results.csv\n", "Baseline PPL used for every MDP: 13.225596\n", "Saved: results/tables/ablation_results.csv\n", "Saved: results/tables/expanded_results.csv\n" ] }, { "data": { "text/html": [ "

\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
methodterserpplloss_mia_aucmin_k_mia_aucmasked_token_ratioskipped_samplespii_eval_samplesmia_samples_per_classmdpexperiment
0rmft0.0460.05066113.2757520.5331040.5228320.21612005002500.050156rmft_216
1tmft_ner0.0300.03304013.5583140.5322240.5189920.12567517625002500.332719ner_strict
2tmft_ner0.0360.03964813.5915410.5321920.5192800.13660418305002500.365945ner_moderate
3tmft_mia0.0240.02643214.0782620.5321440.5187680.10008905002500.852667mia_p90
4tmft_combined0.0140.01541914.3181130.5310720.5155360.29259837835002501.092517combined_p90
5tmft_combined0.0160.01762114.3018010.5311360.5156000.31685615305002501.076205combined_p90_skip65
\n", "
" ], "text/plain": [ " method ter ser ppl loss_mia_auc min_k_mia_auc \\\n", "0 rmft 0.046 0.050661 13.275752 0.533104 0.522832 \n", "1 tmft_ner 0.030 0.033040 13.558314 0.532224 0.518992 \n", "2 tmft_ner 0.036 0.039648 13.591541 0.532192 0.519280 \n", "3 tmft_mia 0.024 0.026432 14.078262 0.532144 0.518768 \n", "4 tmft_combined 0.014 0.015419 14.318113 0.531072 0.515536 \n", "5 tmft_combined 0.016 0.017621 14.301801 0.531136 0.515600 \n", "\n", " masked_token_ratio skipped_samples pii_eval_samples \\\n", "0 0.216120 0 500 \n", "1 0.125675 1762 500 \n", "2 0.136604 1830 500 \n", "3 0.100089 0 500 \n", "4 0.292598 3783 500 \n", "5 0.316856 1530 500 \n", "\n", " mia_samples_per_class mdp experiment \n", "0 250 0.050156 rmft_216 \n", "1 250 0.332719 ner_strict \n", "2 250 0.365945 ner_moderate \n", "3 250 0.852667 mia_p90 \n", "4 250 1.092517 combined_p90 \n", "5 250 1.076205 combined_p90_skip65 " ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from copy import deepcopy\n", "import gc\n", "from pathlib import Path\n", "\n", "import pandas as pd\n", "import torch\n", "from datasets import load_from_disk\n", "\n", "from src.train import load_config, train_model\n", "from main import run_eval\n", "\n", "# Restore state safely after a kernel restart.\n", "if 'final_config' not in globals():\n", " final_config = load_config('configs/config.yaml')\n", "if 'final_splits' not in globals():\n", " final_splits = load_from_disk(final_config.get('prepared_data_dir', 'data/processed'))\n", "if 'eval_path' not in globals():\n", " eval_path = Path(final_config.get('pii_eval_path', 'data/pii_eval.json'))\n", "\n", "final_config['text_column'] = 'text'\n", "main_results_path = Path('results/tables/main_results.csv')\n", "\n", "# Dynamic baseline: read the original result table when available.\n", "if main_results_path.exists():\n", " original_results = pd.read_csv(main_results_path)\n", " baseline_rows = original_results.loc[original_results['method'] == 'baseline', 'ppl']\n", " if baseline_rows.empty:\n", " raise ValueError(f'Baseline row is missing from {main_results_path}')\n", " baseline_ppl = float(baseline_rows.iloc[0])\n", " print(f'Baseline PPL loaded from table: {baseline_ppl:.6f}')\n", "else:\n", " # Re-evaluate the saved baseline with the same validation split.\n", " baseline_cfg = deepcopy(final_config)\n", " baseline_cfg['results_table_dir'] = './results/ablations/baseline_reference/tables'\n", " baseline_model_dir = Path(final_config.get('output_dir', './results')) / 'baseline'\n", " if not baseline_model_dir.exists():\n", " raise FileNotFoundError(f'Baseline checkpoint not found: {baseline_model_dir}')\n", " baseline_result = run_eval(\n", " baseline_cfg,\n", " method='baseline',\n", " splits=final_splits,\n", " eval_path=eval_path,\n", " model_dir=str(baseline_model_dir),\n", " )\n", " baseline_ppl = float(baseline_result['ppl'].iloc[0])\n", " original_results = baseline_result.copy()\n", " print(f'Baseline PPL re-evaluated: {baseline_ppl:.6f}')\n", "\n", "experiments = [\n", " {\n", " 'name': 'rmft_216',\n", " 'method': 'rmft',\n", " 'params': {'rmft_probability': 0.2161},\n", " },\n", " {\n", " 'name': 'ner_strict',\n", " 'method': 'tmft_ner',\n", " 'params': {'ner_labels': ['PERSON', 'EMAIL', 'PHONE']},\n", " },\n", " {\n", " 'name': 'ner_moderate',\n", " 'method': 'tmft_ner',\n", " 'params': {'ner_labels': ['PERSON', 'EMAIL', 'PHONE', 'GPE', 'LOC']},\n", " },\n", " {\n", " 'name': 'mia_p90',\n", " 'method': 'tmft_mia',\n", " 'params': {'threshold_percentile': 90},\n", " },\n", " {\n", " 'name': 'combined_p90',\n", " 'method': 'tmft_combined',\n", " 'params': {'threshold_percentile': 90, 'max_mask_ratio': 0.50},\n", " },\n", " {\n", " 'name': 'combined_p90_skip65',\n", " 'method': 'tmft_combined',\n", " 'params': {'threshold_percentile': 90, 'max_mask_ratio': 0.65},\n", " },\n", "]\n", "\n", "ablation_results = []\n", "\n", "for experiment in experiments:\n", " name = experiment['name']\n", " method = experiment['method']\n", " cfg = deepcopy(final_config)\n", " cfg.update(experiment['params'])\n", " cfg['seed'] = 42\n", " cfg['text_column'] = 'text'\n", " cfg['output_dir'] = f'./results/ablations/{name}'\n", " cfg['results_table_dir'] = f'./results/ablations/{name}/tables'\n", "\n", " expected_model_dir = Path(cfg['output_dir']) / method\n", " print(f'\\n===== {name} =====')\n", "\n", " if (expected_model_dir / 'adapter_config.json').exists():\n", " model_dir = expected_model_dir\n", " print(f'Reusing existing adapter: {model_dir}')\n", " else:\n", " trainer, tokenizer, model_dir = train_model(\n", " cfg,\n", " method=method,\n", " train_dataset=final_splits['train'],\n", " eval_dataset=final_splits['validation'],\n", " )\n", " # Free the training model before loading a fresh evaluation model.\n", " del trainer, tokenizer\n", " gc.collect()\n", " if torch.cuda.is_available():\n", " torch.cuda.empty_cache()\n", "\n", " result = run_eval(\n", " cfg,\n", " method=method,\n", " splits=final_splits,\n", " eval_path=eval_path,\n", " model_dir=str(model_dir),\n", " )\n", " result['experiment'] = name\n", " result['mdp'] = result['ppl'] - baseline_ppl\n", " ablation_results.append(result)\n", "\n", "ablation_df = pd.concat(ablation_results, ignore_index=True)\n", "\n", "output_dir = Path('results/tables')\n", "output_dir.mkdir(parents=True, exist_ok=True)\n", "ablation_path = output_dir / 'ablation_results.csv'\n", "ablation_df.to_csv(ablation_path, index=False)\n", "\n", "# Also save a single comparison table containing the original five conditions.\n", "original_comparison = original_results.copy()\n", "original_comparison['experiment'] = original_comparison['method'].astype(str) + '_original'\n", "expanded_results = pd.concat([original_comparison, ablation_df], ignore_index=True, sort=False)\n", "expanded_path = output_dir / 'expanded_results.csv'\n", "expanded_results.to_csv(expanded_path, index=False)\n", "\n", "print(f'Baseline PPL used for every MDP: {baseline_ppl:.6f}')\n", "print(f'Saved: {ablation_path}')\n", "print(f'Saved: {expanded_path}')\n", "ablation_df" ] }, { "cell_type": "markdown", "id": "cfba4323", "metadata": {}, "source": [ "## 10. Hugging Face Upload\n", "Log in with `huggingface-cli login` first. Upload one validated checkpoint at a time." ] }, { "cell_type": "code", "execution_count": null, "id": "b993fcba-5859-4151-96ff-9dcb5c654619", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Create token: https://huggingface.co/settings/tokens\n" ] } ], "source": [ "# Hugging Face login (prompt, hidden input)\n", "import os\n", "from getpass import getpass\n", "\n", "try:\n", " from huggingface_hub import login\n", "except Exception:\n", " %pip install -U huggingface_hub\n", " from huggingface_hub import login\n", "\n", "print('Create token: https://huggingface.co/settings/tokens')\n", "\n", "env_token = (os.environ.get('HF_TOKEN') or '').strip()\n", "if env_token:\n", " use_env = input('HF_TOKEN is set. Use it? [Y/n]: ').strip().lower()\n", " token = env_token if use_env in ('', 'y', 'yes') else getpass('Enter HF token: ').strip()\n", "else:\n", " token = getpass('Enter HF token: ').strip()\n", "\n", "if not token:\n", " raise ValueError('No token provided.')\n", "\n", "login(token=token)\n", "os.environ['HF_TOKEN'] = token\n", "print('Hugging Face login OK.')" ] }, { "cell_type": "markdown", "id": "91aece4a-0e82-419a-ba3f-cf07beca9cbe", "metadata": {}, "source": [ "## 11.Upload Model, Code, and Results" ] }, { "cell_type": "code", "execution_count": null, "id": "a29ad15c-b5c1-439f-aa81-1aab0101dd9c", "metadata": {}, "outputs": [], "source": [ "from huggingface_hub import HfApi\n", "from pathlib import Path\n", "\n", "HF_REPO_ID = input('Repository ID (username/repository): ').strip()\n", "PRIVATE_REPO = True # Change to False only after reviewing every uploaded artifact.\n", "\n", "if '/' not in HF_REPO_ID:\n", " raise ValueError('Use the form username/repository')\n", "\n", "project_root = Path.cwd()\n", "trained_paths = globals().get('train_outputs') or {\n", " method: str(project_root / 'results' / method)\n", " for method in ['baseline', 'rmft', 'tmft_ner', 'tmft_mia', 'tmft_combined']\n", "}\n", "primary_model_dir = Path(trained_paths.get('tmft_combined', 'results/tmft_combined'))\n", "if not (primary_model_dir / 'adapter_config.json').exists():\n", " raise FileNotFoundError(f'LoRA adapter not found: {primary_model_dir}')\n", "\n", "api = HfApi(token=os.environ['HF_TOKEN'])\n", "api.create_repo(\n", " repo_id=HF_REPO_ID,\n", " repo_type='model',\n", " private=PRIVATE_REPO,\n", " exist_ok=True,\n", ")\n", "\n", "# Root remains directly loadable as a PEFT model repository.\n", "api.upload_folder(\n", " repo_id=HF_REPO_ID,\n", " repo_type='model',\n", " folder_path=str(primary_model_dir),\n", " path_in_repo='',\n", " ignore_patterns=['checkpoint-*', '*.pt', 'optimizer.pt', 'scheduler.pt', 'rng_state.pth'],\n", ")\n", "\n", "readme_path = project_root / 'README.md'\n", "if readme_path.exists():\n", " api.upload_file(\n", " repo_id=HF_REPO_ID, repo_type='model', path_or_fileobj=str(readme_path), path_in_repo='README.md'\n", " )\n", "\n", "for filename in ['requirements.txt', 'main.py', 'tmft_experiment.ipynb']:\n", " file_path = project_root / filename\n", " if file_path.exists():\n", " api.upload_file(\n", " repo_id=HF_REPO_ID,\n", " repo_type='model',\n", " path_or_fileobj=str(file_path),\n", " path_in_repo=f'project/{filename}',\n", " )\n", "\n", "for local_dir, repo_dir in [\n", " ('src', 'project/src'),\n", " ('configs', 'project/configs'),\n", " ('results/tables', 'experiment/tables'),\n", " ('results/figures', 'experiment/figures'),\n", "]:\n", " folder = project_root / local_dir\n", " if folder.exists():\n", " api.upload_folder(\n", " repo_id=HF_REPO_ID,\n", " repo_type='model',\n", " folder_path=str(folder),\n", " path_in_repo=repo_dir,\n", " ignore_patterns=['__pycache__', '*.pyc', '.ipynb_checkpoints'],\n", " )\n", "\n", "# Small LoRA comparison adapters make the five-condition experiment reproducible.\n", "for method, model_path in trained_paths.items():\n", " if method == 'tmft_combined':\n", " continue\n", " model_path = Path(model_path)\n", " if (model_path / 'adapter_config.json').exists():\n", " api.upload_folder(\n", " repo_id=HF_REPO_ID,\n", " repo_type='model',\n", " folder_path=str(model_path),\n", " path_in_repo=f'experiment/adapters/{method}',\n", " ignore_patterns=['checkpoint-*', '*.pt', 'optimizer.pt', 'scheduler.pt', 'rng_state.pth'],\n", " )\n", "\n", "print(f'Upload complete: https://huggingface.co/{HF_REPO_ID}')\n", "print('Raw data and PII evaluation records were not uploaded.')" ] }, { "cell_type": "markdown", "id": "994ac451-8446-48ea-9715-ff37047dba7e", "metadata": {}, "source": [ "## 12. Download the Complete Vessel Project" ] }, { "cell_type": "code", "execution_count": null, "id": "a3e0fc62-21d8-4f26-ad82-2ee1612851e6", "metadata": {}, "outputs": [], "source": [ "import shutil\n", "from pathlib import Path\n", "from IPython.display import FileLink, display\n", "\n", "project_root = Path.cwd().resolve()\n", "archive_destination = project_root / f'{project_root.name}_complete.zip'\n", "archive_destination.unlink(missing_ok=True)\n", "\n", "temporary_base = Path('/tmp') / f'{project_root.name}_complete'\n", "temporary_zip = Path(shutil.make_archive(\n", " str(temporary_base),\n", " 'zip',\n", " root_dir=str(project_root.parent),\n", " base_dir=project_root.name,\n", "))\n", "shutil.move(str(temporary_zip), str(archive_destination))\n", "\n", "size_gb = archive_destination.stat().st_size / (1024 ** 3)\n", "print(f'Archive ready: {archive_destination} ({size_gb:.2f} GB)')\n", "print('Keep this local backup private; it may contain PII.')\n", "display(FileLink(archive_destination.name, result_html_prefix='Download complete project: '))" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 5 }