{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "6cc7d98d-23c5-4fb3-86eb-4e0665382bb1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[2mInstalled \u001b[1mPython 3.13.8\u001b[0m \u001b[2min 696ms\u001b[0m\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mcpython-3.13.8-linux-x86_64-gnu\u001b[0m (python3.13)\n", "\u001b[1m\u001b[33mwarning\u001b[39m\u001b[0m\u001b[1m:\u001b[0m \u001b[1m`\u001b[36m/root/.local/bin\u001b[39m` is not on your PATH. To use installed Python executables, run `\u001b[32mexport PATH=\"/root/.local/bin:$PATH\"\u001b[39m` or `\u001b[32muv python update-shell\u001b[39m`.\u001b[0m\n", "Pinned `\u001b[36m.python-version\u001b[39m` to `\u001b[32m3.13\u001b[39m`\n" ] } ], "source": [ "!uv python install 313\n", "!uv python pin 313" ] }, { "cell_type": "code", "execution_count": 2, "id": "2d741065-01e2-4551-8a7e-70fecca6759e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Using CPython \u001b[36m3.13.8\u001b[39m\n", "Creating virtual environment at: \u001b[36m.venv\u001b[39m\n", "Activate with: \u001b[32msource .venv/bin/activate\u001b[39m\n", "\u001b[2K\u001b[2mResolved \u001b[1m30 packages\u001b[0m \u001b[2min 63ms\u001b[0m\u001b[0m \u001b[0m\n", "\u001b[2K\u001b[2mPrepared \u001b[1m30 packages\u001b[0m \u001b[2min 703ms\u001b[0m\u001b[0m \n", "\u001b[2K\u001b[2mInstalled \u001b[1m30 packages\u001b[0m \u001b[2min 110ms\u001b[0m\u001b[0m \u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1masttokens\u001b[0m\u001b[2m==3.0.2\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mcomm\u001b[0m\u001b[2m==0.2.3\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mdebugpy\u001b[0m\u001b[2m==1.8.21\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mdecorator\u001b[0m\u001b[2m==5.3.1\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mexecuting\u001b[0m\u001b[2m==2.2.1\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mipykernel\u001b[0m\u001b[2m==7.3.0\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mipython\u001b[0m\u001b[2m==9.15.0\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mipython-pygments-lexers\u001b[0m\u001b[2m==1.1.1\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mjedi\u001b[0m\u001b[2m==0.20.0\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mjupyter-client\u001b[0m\u001b[2m==8.9.1\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mjupyter-core\u001b[0m\u001b[2m==5.9.1\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mmatplotlib-inline\u001b[0m\u001b[2m==0.2.2\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mnest-asyncio2\u001b[0m\u001b[2m==1.7.2\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mpackaging\u001b[0m\u001b[2m==26.2\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mparso\u001b[0m\u001b[2m==0.8.7\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mpexpect\u001b[0m\u001b[2m==4.9.0\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mplatformdirs\u001b[0m\u001b[2m==4.11.0\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mprompt-toolkit\u001b[0m\u001b[2m==3.0.53\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mpsutil\u001b[0m\u001b[2m==7.2.2\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mptyprocess\u001b[0m\u001b[2m==0.7.0\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mpure-eval\u001b[0m\u001b[2m==0.2.3\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mpygments\u001b[0m\u001b[2m==2.20.0\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mpython-dateutil\u001b[0m\u001b[2m==2.9.0.post0\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mpyzmq\u001b[0m\u001b[2m==27.1.0\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1msix\u001b[0m\u001b[2m==1.17.0\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mstack-data\u001b[0m\u001b[2m==0.6.3\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mtornado\u001b[0m\u001b[2m==6.5.7\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mtraitlets\u001b[0m\u001b[2m==5.15.1\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mtyping-extensions\u001b[0m\u001b[2m==4.16.0\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mwcwidth\u001b[0m\u001b[2m==0.8.2\u001b[0m\n", "Installed kernelspec uv-kernel in /root/.local/share/jupyter/kernels/uv-kernel\n" ] } ], "source": [ "!uv venv --clear\n", "!uv pip install ipykernel\n", "!uv run python -m ipykernel install --user --name=uv-kernel --display-name \"Python (uv venv)\"" ] }, { "cell_type": "code", "execution_count": 4, "id": "ae35bf68-b4b1-435e-8a79-61bd140bc77e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[2mAudited \u001b[1m10 packages\u001b[0m \u001b[2min 10ms\u001b[0m\u001b[0m\n" ] } ], "source": [ "!uv pip install transformers huggingface_hub datasets fastrand torch sentencepiece protobuf evaluate scikit-learn accelerate>1.1.0" ] }, { "cell_type": "code", "execution_count": 1, "id": "345615b8-726e-4c9a-ab56-deb322632c82", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/workspace/.venv/lib/python3.13/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "import numpy as np\n", "import torch\n", "from datasets import load_dataset\n", "from transformers import (\n", " AutoTokenizer, \n", " AutoModelForSequenceClassification, \n", " TrainingArguments, \n", " Trainer,\n", " DataCollatorWithPadding\n", ")\n", "from huggingface_hub import notebook_login\n", "import evaluate" ] }, { "cell_type": "code", "execution_count": 2, "id": "033f1b47-2771-48ff-b89f-fcc3d93ab47d", "metadata": {}, "outputs": [], "source": [ "notebook_login()" ] }, { "cell_type": "code", "execution_count": 3, "id": "f0ff7440-f91c-4705-9d32-d37ac1089239", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[transformers] `torch_dtype` is deprecated! Use `dtype` instead!\n", "Loading weights: 100%|██████████| 172/172 [00:00<00:00, 1366.64it/s]\n", "[transformers] \u001b[1mModernBertForSequenceClassification LOAD REPORT\u001b[0m from: jhu-clsp/ettin-encoder-1b\n", "Key | Status | \n", "------------------+------------+-\n", "decoder.bias | UNEXPECTED | \n", "decoder.weight | UNEXPECTED | \n", "classifier.weight | MISSING | \n", "classifier.bias | MISSING | \n", "\n", "Notes:\n", "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.\n", "- MISSING:\tthose params were newly initialized because missing from the checkpoint. Consider training on your downstream task.\n" ] } ], "source": [ "# 1. Load tokenizer and model for classification\n", "model_name = \"jhu-clsp/ettin-encoder-1b\"\n", "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", "model = AutoModelForSequenceClassification.from_pretrained(\n", " model_name, num_labels=2,\n", " torch_dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16\n", ") # Adjust num_labels" ] }, { "cell_type": "code", "execution_count": 4, "id": "46b5f750-1b36-4437-b7cb-26331d92c595", "metadata": {}, "outputs": [], "source": [ "# 2. Load and tokenize dataset (example using IMDb for binary classification)\n", "dataset = load_dataset(\"hanzceo/JOSS-data-L2\")[\"train\"]" ] }, { "cell_type": "code", "execution_count": 5, "id": "37f80b16-499f-4002-93c5-a534c4f7b289", "metadata": {}, "outputs": [], "source": [ "dataset = dataset.train_test_split(test_size=0.2)" ] }, { "cell_type": "code", "execution_count": 6, "id": "4a8f8b10-e197-4d58-a818-2f5e8ba77d0e", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "DatasetDict({\n", " train: Dataset({\n", " features: ['sentence1', 'score'],\n", " num_rows: 1600\n", " })\n", " test: Dataset({\n", " features: ['sentence1', 'score'],\n", " num_rows: 400\n", " })\n", "})" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dataset" ] }, { "cell_type": "code", "execution_count": 7, "id": "fa7253f7-05a7-40a4-9727-d107adac325e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Map: 100%|██████████| 1600/1600 [00:01<00:00, 1399.51 examples/s]\n", "Map: 100%|██████████| 400/400 [00:00<00:00, 1632.19 examples/s]\n" ] } ], "source": [ "def tokenize_function(examples):\n", " return tokenizer(examples[\"sentence1\"], truncation=True)\n", "tokenized_datasets = dataset.map(tokenize_function, batched=True)\n", "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)" ] }, { "cell_type": "code", "execution_count": 8, "id": "49744185-47ff-460a-9a1b-90f33f2bc991", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "DatasetDict({\n", " train: Dataset({\n", " features: ['sentence1', 'labels', 'input_ids', 'attention_mask'],\n", " num_rows: 1600\n", " })\n", " test: Dataset({\n", " features: ['sentence1', 'labels', 'input_ids', 'attention_mask'],\n", " num_rows: 400\n", " })\n", "})" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenized_datasets = tokenized_datasets.rename_column(\"score\", \"labels\")\n", "\n", "tokenized_datasets" ] }, { "cell_type": "code", "execution_count": 9, "id": "93531513-4e7f-4d16-8292-8cecb44e0aa3", "metadata": {}, "outputs": [], "source": [ "# 3. Setup evaluation metrics\n", "metric = evaluate.load(\"accuracy\")\n", "def compute_metrics(eval_pred):\n", " logits, labels = eval_pred\n", " predictions = np.argmax(logits, axis=-1)\n", " return metric.compute(predictions=predictions, references=labels)" ] }, { "cell_type": "code", "execution_count": 10, "id": "b663bbee-15cb-44a9-9393-a78cf051c0fc", "metadata": {}, "outputs": [], "source": [ "# 4. Define Training Arguments\n", "training_args = TrainingArguments(\n", " output_dir=\"models/ettin-classification\",\n", " eval_strategy=\"epoch\",\n", " learning_rate=2e-5,\n", " per_device_train_batch_size=2,\n", " per_device_eval_batch_size=2,\n", " num_train_epochs=3,\n", " weight_decay=0.01,\n", " bf16=True\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "685b3f75-3a68-4965-bf28-ba3e70b97618", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [2400/2400 17:50, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossAccuracy
10.4234330.2264600.947500
20.2170080.2110060.947500

\n", "

\n", " \n", " \n", " [119/200 00:14 < 00:09, 8.22 it/s]\n", "
\n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 1.30it/s]\n", "Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 1.85it/s]\n", "Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 1.28it/s]\n", "Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 1.39it/s]\n", "Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 1.28it/s]\n" ] } ], "source": [ "# 5. Train\n", "trainer = Trainer(\n", " model=model,\n", " args=training_args,\n", " data_collator=data_collator,\n", " train_dataset=tokenized_datasets[\"train\"].shuffle(seed=42),\n", " eval_dataset=tokenized_datasets[\"test\"].shuffle(seed=42),\n", " compute_metrics=compute_metrics,\n", ")\n", "\n", "trainer.train()" ] }, { "cell_type": "code", "execution_count": null, "id": "250248f5-021e-4247-8b2d-1d1098cf8a31", "metadata": {}, "outputs": [], "source": [ "model" ] }, { "cell_type": "code", "execution_count": null, "id": "91374ca9-04ca-4ab7-a136-a5b329071426", "metadata": {}, "outputs": [], "source": [ "model.push_to_hub(\"hanzceo/JOSS-dev1\")" ] }, { "cell_type": "code", "execution_count": null, "id": "ed10d1ca-ada0-43db-ac78-357a865d2b4a", "metadata": {}, "outputs": [], "source": [ "tokenizer.push_to_hub(\"hanzceo/JOSS-dev1\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python (uv venv)", "language": "python", "name": "uv-kernel" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.8" } }, "nbformat": 4, "nbformat_minor": 5 }