Hugging Face's logo

AiisNothing
/
electra-testing

Model card Files
xet
Community
AiisNothing commited on
Commit
177b25f
·
verified ·
1 Parent(s): 886e18d

training file

Browse files
Files changed (1) hide show
  1. electra_discriminator(1).ipynb +632 -0
electra_discriminator(1).ipynb ADDED
@@ -0,0 +1,632 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 1,
6
+ "metadata": {
7
+ "colab": {
8
+ "base_uri": "https://localhost:8080/"
9
+ },
10
+ "id": "5k3Qn8DImEGv",
11
+ "outputId": "d9946915-5fcd-43b3-edc2-e119b15c77c8"
12
+ },
13
+ "outputs": [
14
+ {
15
+ "output_type": "stream",
16
+ "name": "stdout",
17
+ "text": [
18
+ "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.44.2)\n",
19
+ "Collecting transformers\n",
20
+ " Downloading transformers-4.45.2-py3-none-any.whl.metadata (44 kB)\n",
21
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.4/44.4 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
22
+ "\u001b[?25hCollecting datasets\n",
23
+ " Downloading datasets-3.0.1-py3-none-any.whl.metadata (20 kB)\n",
24
+ "Collecting peft\n",
25
+ " Downloading peft-0.13.2-py3-none-any.whl.metadata (13 kB)\n",
26
+ "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.16.1)\n",
27
+ "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.24.7)\n",
28
+ "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.26.4)\n",
29
+ "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.1)\n",
30
+ "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.2)\n",
31
+ "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.9.11)\n",
32
+ "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.32.3)\n",
33
+ "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.5)\n",
34
+ "Collecting tokenizers<0.21,>=0.20 (from transformers)\n",
35
+ " Downloading tokenizers-0.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n",
36
+ "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.5)\n",
37
+ "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (16.1.0)\n",
38
+ "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
39
+ " Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
40
+ "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (2.2.2)\n",
41
+ "Collecting xxhash (from datasets)\n",
42
+ " Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
43
+ "Collecting multiprocess (from datasets)\n",
44
+ " Downloading multiprocess-0.70.17-py310-none-any.whl.metadata (7.2 kB)\n",
45
+ "Requirement already satisfied: fsspec<=2024.6.1,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.6.1,>=2023.1.0->datasets) (2024.6.1)\n",
46
+ "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.10.10)\n",
47
+ "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from peft) (5.9.5)\n",
48
+ "Requirement already satisfied: torch>=1.13.0 in /usr/local/lib/python3.10/dist-packages (from peft) (2.4.1+cu121)\n",
49
+ "Requirement already satisfied: accelerate>=0.21.0 in /usr/local/lib/python3.10/dist-packages (from peft) (0.34.2)\n",
50
+ "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (2.4.3)\n",
51
+ "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
52
+ "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (24.2.0)\n",
53
+ "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
54
+ "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.1.0)\n",
55
+ "Requirement already satisfied: yarl<2.0,>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.14.0)\n",
56
+ "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
57
+ "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (4.12.2)\n",
58
+ "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4.0)\n",
59
+ "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.10)\n",
60
+ "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.2.3)\n",
61
+ "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.8.30)\n",
62
+ "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft) (1.13.3)\n",
63
+ "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft) (3.4)\n",
64
+ "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft) (3.1.4)\n",
65
+ "INFO: pip is looking at multiple versions of multiprocess to determine which version is compatible with other requirements. This could take a while.\n",
66
+ " Downloading multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n",
67
+ "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
68
+ "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.2)\n",
69
+ "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.2)\n",
70
+ "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
71
+ "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from yarl<2.0,>=1.12.0->aiohttp->datasets) (0.2.0)\n",
72
+ "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.13.0->peft) (3.0.1)\n",
73
+ "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.13.0->peft) (1.3.0)\n",
74
+ "Downloading transformers-4.45.2-py3-none-any.whl (9.9 MB)\n",
75
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.9/9.9 MB\u001b[0m \u001b[31m39.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
76
+ "\u001b[?25hDownloading datasets-3.0.1-py3-none-any.whl (471 kB)\n",
77
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m471.6/471.6 kB\u001b[0m \u001b[31m32.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
78
+ "\u001b[?25hDownloading peft-0.13.2-py3-none-any.whl (320 kB)\n",
79
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.7/320.7 kB\u001b[0m \u001b[31m29.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
80
+ "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
81
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
82
+ "\u001b[?25hDownloading tokenizers-0.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.0 MB)\n",
83
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m18.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
84
+ "\u001b[?25hDownloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
85
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
86
+ "\u001b[?25hDownloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
87
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
88
+ "\u001b[?25hInstalling collected packages: xxhash, dill, multiprocess, tokenizers, transformers, peft, datasets\n",
89
+ " Attempting uninstall: tokenizers\n",
90
+ " Found existing installation: tokenizers 0.19.1\n",
91
+ " Uninstalling tokenizers-0.19.1:\n",
92
+ " Successfully uninstalled tokenizers-0.19.1\n",
93
+ " Attempting uninstall: transformers\n",
94
+ " Found existing installation: transformers 4.44.2\n",
95
+ " Uninstalling transformers-4.44.2:\n",
96
+ " Successfully uninstalled transformers-4.44.2\n",
97
+ "Successfully installed datasets-3.0.1 dill-0.3.8 multiprocess-0.70.16 peft-0.13.2 tokenizers-0.20.1 transformers-4.45.2 xxhash-3.5.0\n"
98
+ ]
99
+ }
100
+ ],
101
+ "source": [
102
+ "!pip install -U transformers datasets peft"
103
+ ]
104
+ },
105
+ {
106
+ "cell_type": "code",
107
+ "execution_count": 2,
108
+ "metadata": {
109
+ "id": "F0rYC0S3lhUJ"
110
+ },
111
+ "outputs": [],
112
+ "source": [
113
+ "import torch\n",
114
+ "from torch.utils.data import DataLoader, Dataset\n",
115
+ "from transformers import AutoModel, AdamW, get_linear_schedule_with_warmup,DebertaV2Tokenizer\n",
116
+ "from sklearn.model_selection import train_test_split\n",
117
+ "from datasets import load_dataset\n",
118
+ "import numpy as np\n",
119
+ "import pandas as pd\n",
120
+ "from sklearn.preprocessing import LabelEncoder\n",
121
+ "from transformers import Trainer, TrainingArguments\n",
122
+ "from datasets import Dataset as HFDataset\n",
123
+ "from peft import PeftConfig, PeftModel,LoraConfig,get_peft_model\n",
124
+ "\n",
125
+ "# Define constants\n",
126
+ "MODEL_NAME = 'google/electra-small-discriminator'\n",
127
+ "BATCH_SIZE = 4\n",
128
+ "EPOCHS = 3\n",
129
+ "LEARNING_RATE = 2e-4\n",
130
+ "MAX_LENGTH = 512"
131
+ ]
132
+ },
133
+ {
134
+ "cell_type": "code",
135
+ "execution_count": 7,
136
+ "metadata": {
137
+ "colab": {
138
+ "base_uri": "https://localhost:8080/"
139
+ },
140
+ "id": "wVfJhfyqnur3",
141
+ "outputId": "c4f695e0-0281-43f5-b508-6c58c3971222"
142
+ },
143
+ "outputs": [
144
+ {
145
+ "output_type": "stream",
146
+ "name": "stdout",
147
+ "text": [
148
+ "Columns in /content/ScamDataNew.csv: ['Scammer', 'Label']\n",
149
+ "Columns in /content/cleaned-data.csv: ['input', 'output']\n",
150
+ " text label\n",
151
+ "0 unknown: Hello this is HUGIE Finance calling. ... 1.0\n",
152
+ "1 unknown: Pepperfry item (Yukashi 3 Door Wardro... 0.0\n",
153
+ "2 unknown: Act now to benefit from our unique of... 1.0\n",
154
+ "3 unknown: It's Shoppers Stop BirthYAY & we love... 0.0\n",
155
+ "4 unknown: Hello I'm calling from MUTHOOT Financ... 1.0\n",
156
+ "... ... ...\n",
157
+ "4433 unknown: did you check the email i sent yester... 0.0\n",
158
+ "4434 unknown: Cant wait to see you this weekend, so... 0.0\n",
159
+ "4435 unknown: I think we should leave earlier, traf... 0.0\n",
160
+ "4436 unknown: forgot to bring the umbrella, it's ra... 0.0\n",
161
+ "4437 unknown: is there anything else you need from the 0.0\n",
162
+ "\n",
163
+ "[4438 rows x 2 columns]\n"
164
+ ]
165
+ }
166
+ ],
167
+ "source": [
168
+ "import pandas as pd\n",
169
+ "\n",
170
+ "# List of file paths to the CSV files\n",
171
+ "csv_files = [\n",
172
+ " '/content/ScamDataNew.csv',\n",
173
+ " '/content/cleaned-data.csv',\n",
174
+ "]\n",
175
+ "\n",
176
+ "# Function to load a CSV file and extract two columns\n",
177
+ "def load_and_select_columns(file_path, text_col, label_col):\n",
178
+ " if (file_path=='/content/Data_including_normal.csv'):\n",
179
+ " df = pd.read_csv(file_path, encoding='ISO-8859-1')\n",
180
+ " else:\n",
181
+ " df = pd.read_csv(file_path)\n",
182
+ " print(f\"Columns in {file_path}: {df.columns.tolist()}\")\n",
183
+ " selected_df = df[[text_col, label_col]].copy() # Select the two columns\n",
184
+ " selected_df.columns = ['text', 'label'] # Standardize column names\n",
185
+ " return selected_df\n",
186
+ "\n",
187
+ "# Load each CSV and extract relevant columns\n",
188
+ "# Update 'text_col' and 'label_col' with actual column names from each CSV\n",
189
+ "df1 = load_and_select_columns(csv_files[0], 'Scammer', 'Label')\n",
190
+ "df1['text']='unknown: '+df1['text']\n",
191
+ "df2 = load_and_select_columns(csv_files[1], 'input', 'output')\n",
192
+ "# df3 = load_and_select_columns(csv_files[2], 'dialogue', 'labels')\n",
193
+ "df4=pd.read_excel(\"/content/Old+Improved data.xlsx\")\n",
194
+ "df4 = df4[['content', 'is scam']].copy() # Select the two columns\n",
195
+ "df4.columns = ['text', 'label']\n",
196
+ "df4['text']='unknown: '+df4['text']\n",
197
+ "# Concatenate the selected columns from all files\n",
198
+ "combined_df = pd.concat([df1, df2,df4], ignore_index=True)\n",
199
+ "\n",
200
+ "# Display the combined DataFrame\n",
201
+ "print(combined_df)"
202
+ ]
203
+ },
204
+ {
205
+ "cell_type": "code",
206
+ "source": [
207
+ "combined_df.dropna(inplace=True)\n",
208
+ "combined_df['label'] = combined_df['label'].astype(int)\n",
209
+ "\n",
210
+ "# # Reset the index of the combined DataFrame\n",
211
+ "# combined_df.reset_index(drop=True, inplace=True)\n",
212
+ "combined_df['text']=combined_df['text'].str.lower()\n",
213
+ "# Display the combined DataFrame\n",
214
+ "print(combined_df)"
215
+ ],
216
+ "metadata": {
217
+ "colab": {
218
+ "base_uri": "https://localhost:8080/"
219
+ },
220
+ "id": "ZXcCFIgM08Bp",
221
+ "outputId": "898a7392-0b33-40f6-b274-d01989facd41"
222
+ },
223
+ "execution_count": 8,
224
+ "outputs": [
225
+ {
226
+ "output_type": "stream",
227
+ "name": "stdout",
228
+ "text": [
229
+ " text label\n",
230
+ "0 unknown: hello this is hugie finance calling. ... 1\n",
231
+ "1 unknown: pepperfry item (yukashi 3 door wardro... 0\n",
232
+ "2 unknown: act now to benefit from our unique of... 1\n",
233
+ "3 unknown: it's shoppers stop birthyay & we love... 0\n",
234
+ "4 unknown: hello i'm calling from muthoot financ... 1\n",
235
+ "... ... ...\n",
236
+ "4433 unknown: did you check the email i sent yester... 0\n",
237
+ "4434 unknown: cant wait to see you this weekend, so... 0\n",
238
+ "4435 unknown: i think we should leave earlier, traf... 0\n",
239
+ "4436 unknown: forgot to bring the umbrella, it's ra... 0\n",
240
+ "4437 unknown: is there anything else you need from the 0\n",
241
+ "\n",
242
+ "[4437 rows x 2 columns]\n"
243
+ ]
244
+ }
245
+ ]
246
+ },
247
+ {
248
+ "cell_type": "code",
249
+ "execution_count": 9,
250
+ "metadata": {
251
+ "id": "0M-Psc9XlwCx"
252
+ },
253
+ "outputs": [],
254
+ "source": [
255
+ "combined_df.to_csv('cleaned-data-version2-with-user-unknown.csv', index=False)"
256
+ ]
257
+ },
258
+ {
259
+ "cell_type": "code",
260
+ "execution_count": 11,
261
+ "metadata": {
262
+ "id": "9HEql8ZemQ8V"
263
+ },
264
+ "outputs": [],
265
+ "source": [
266
+ "def load_data_from_csv():\n",
267
+ " df = combined_df\n",
268
+ " texts = df['text'].tolist() # Replace with your text column name\n",
269
+ " label = df['label'].tolist() # Replace with your label column name\n",
270
+ " le = LabelEncoder()\n",
271
+ " label = le.fit_transform(label)\n",
272
+ " return texts, label\n",
273
+ "\n",
274
+ "import pandas as pd\n",
275
+ "from datasets import Dataset as HFDataset\n",
276
+ "import torch\n",
277
+ "\n",
278
+ "def preprocess_data(texts, label, tokenizer, max_length):\n",
279
+ " # Tokenize the input texts\n",
280
+ " encodings = tokenizer(texts, padding='max_length', truncation=True, max_length=max_length, return_tensors='pt')\n",
281
+ "\n",
282
+ " # Convert PyTorch tensors to lists\n",
283
+ " input_ids = encodings['input_ids'].tolist()\n",
284
+ " attention_mask = encodings['attention_mask'].tolist()\n",
285
+ " token_type_ids = encodings['token_type_ids'].tolist() if 'token_type_ids' in encodings else None\n",
286
+ "\n",
287
+ " # Ensure labels are also in list format\n",
288
+ " if isinstance(label, torch.Tensor):\n",
289
+ " label = label.tolist()\n",
290
+ "\n",
291
+ " # Create a dictionary for the dataset\n",
292
+ " dataset_dict = {\n",
293
+ " 'input_ids': input_ids,\n",
294
+ " 'attention_mask': attention_mask,\n",
295
+ " 'token_type_ids': token_type_ids,\n",
296
+ " 'labels': label\n",
297
+ " }\n",
298
+ "\n",
299
+ " # Convert the dictionary to a Pandas DataFrame\n",
300
+ " df = pd.DataFrame(dataset_dict)\n",
301
+ "\n",
302
+ " # Convert the DataFrame to a Hugging Face Dataset\n",
303
+ " dataset = HFDataset.from_pandas(df)\n",
304
+ "\n",
305
+ " print(dataset)\n",
306
+ " return dataset"
307
+ ]
308
+ },
309
+ {
310
+ "cell_type": "code",
311
+ "execution_count": 12,
312
+ "metadata": {
313
+ "colab": {
314
+ "base_uri": "https://localhost:8080/"
315
+ },
316
+ "id": "6VVDZ_WAo9o5",
317
+ "outputId": "897c1876-8636-4079-98ca-d002eeb997c7"
318
+ },
319
+ "outputs": [
320
+ {
321
+ "output_type": "stream",
322
+ "name": "stderr",
323
+ "text": [
324
+ "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at google/electra-small-discriminator and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
325
+ "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
326
+ ]
327
+ },
328
+ {
329
+ "output_type": "stream",
330
+ "name": "stdout",
331
+ "text": [
332
+ "13549314\n",
333
+ "Dataset({\n",
334
+ " features: ['input_ids', 'attention_mask', 'token_type_ids', 'labels'],\n",
335
+ " num_rows: 3940\n",
336
+ "})\n",
337
+ "Dataset({\n",
338
+ " features: ['input_ids', 'attention_mask', 'token_type_ids', 'labels'],\n",
339
+ " num_rows: 986\n",
340
+ "})\n"
341
+ ]
342
+ }
343
+ ],
344
+ "source": [
345
+ "from transformers import AutoModelForSequenceClassification,AutoTokenizer\n",
346
+ "\n",
347
+ "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n",
348
+ "# Load model directly\n",
349
+ "model = AutoModelForSequenceClassification.from_pretrained(MODEL_NAME)\n",
350
+ "\n",
351
+ "\n",
352
+ "def count_trainable_parameters(model):\n",
353
+ " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
354
+ "print(count_trainable_parameters(model))\n",
355
+ "\n",
356
+ "\n",
357
+ "\n",
358
+ "lora_config = LoraConfig(\n",
359
+ " r=8, # Rank of the low-rank matrices\n",
360
+ " lora_alpha=16, # Alpha for the LoRA scaling\n",
361
+ " lora_dropout=0.1 # Dropout for LoRA layers\n",
362
+ ")\n",
363
+ "# peft_config = PeftConfig(\n",
364
+ "# base_model_name_or_path=MODEL_NAME,\n",
365
+ "# adapter_config=lora_config\n",
366
+ "# )\n",
367
+ "# model = get_peft_model(model, peft_config=lora_config)\n",
368
+ "# Load and preprocess data\n",
369
+ "texts, label = load_data_from_csv() # Replace with your file path\n",
370
+ "train_texts, val_texts, train_label, val_label = train_test_split(texts, label, test_size=0.2, random_state=42)\n",
371
+ "\n",
372
+ "train_dataset = preprocess_data(train_texts, train_label, tokenizer, MAX_LENGTH)\n",
373
+ "val_dataset = preprocess_data(val_texts, val_label, tokenizer, MAX_LENGTH)"
374
+ ]
375
+ },
376
+ {
377
+ "cell_type": "code",
378
+ "execution_count": 7,
379
+ "metadata": {
380
+ "id": "l7FiPtRFr9ma",
381
+ "colab": {
382
+ "base_uri": "https://localhost:8080/"
383
+ },
384
+ "outputId": "de83f7ea-0c56-4695-fe73-5ffffc8ca0cc"
385
+ },
386
+ "outputs": [
387
+ {
388
+ "output_type": "stream",
389
+ "name": "stdout",
390
+ "text": [
391
+ "13549314\n"
392
+ ]
393
+ }
394
+ ],
395
+ "source": [
396
+ "print(count_trainable_parameters(model))"
397
+ ]
398
+ },
399
+ {
400
+ "cell_type": "code",
401
+ "execution_count": 14,
402
+ "metadata": {
403
+ "colab": {
404
+ "base_uri": "https://localhost:8080/",
405
+ "height": 422
406
+ },
407
+ "id": "8qZSOElhsDsG",
408
+ "outputId": "46e7b2c1-6ba8-4fb4-c083-7282feab6194"
409
+ },
410
+ "outputs": [
411
+ {
412
+ "output_type": "error",
413
+ "ename": "RuntimeError",
414
+ "evalue": "CUDA error: device-side assert triggered\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n",
415
+ "traceback": [
416
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
417
+ "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
418
+ "\u001b[0;32m<ipython-input-14-3e927ad3458e>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m training_args = TrainingArguments(\n\u001b[1;32m 3\u001b[0m \u001b[0moutput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'./results'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0meval_strategy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'epoch'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mlearning_rate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mLEARNING_RATE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
419
+ "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/cuda/memory.py\u001b[0m in \u001b[0;36mempty_cache\u001b[0;34m()\u001b[0m\n\u001b[1;32m 168\u001b[0m \"\"\"\n\u001b[1;32m 169\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mis_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 170\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cuda_emptyCache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 171\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 172\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
420
+ "\u001b[0;31mRuntimeError\u001b[0m: CUDA error: device-side assert triggered\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n"
421
+ ]
422
+ }
423
+ ],
424
+ "source": [
425
+ "torch.cuda.empty_cache()\n",
426
+ "training_args = TrainingArguments(\n",
427
+ " output_dir='./results',\n",
428
+ " eval_strategy='epoch',\n",
429
+ " learning_rate=LEARNING_RATE,\n",
430
+ " per_device_train_batch_size=16,\n",
431
+ " per_device_eval_batch_size=16,\n",
432
+ " num_train_epochs=6,\n",
433
+ " weight_decay=0.001,\n",
434
+ " logging_dir='./logs',\n",
435
+ " logging_steps=1,\n",
436
+ " remove_unused_columns=False)\n",
437
+ "\n",
438
+ "trainer = Trainer(\n",
439
+ " model=model,\n",
440
+ " args=training_args,\n",
441
+ " train_dataset=train_dataset,\n",
442
+ " eval_dataset=val_dataset,\n",
443
+ ")\n",
444
+ "\n",
445
+ "trainer.train()\n",
446
+ "trainer.evaluate()\n",
447
+ "#trainer.save_model('./final_model')"
448
+ ]
449
+ },
450
+ {
451
+ "cell_type": "code",
452
+ "execution_count": null,
453
+ "metadata": {
454
+ "id": "SHjymNkXuwEY"
455
+ },
456
+ "outputs": [],
457
+ "source": [
458
+ "# from huggingface_hub import notebook_login\n",
459
+ "# notebook_login()\n",
460
+ "# repo_name = \"AiisNothing/electra-discriminator-trained-merged-dataset-version1\"\n",
461
+ "# model = model.merge_and_unload()\n",
462
+ "\n",
463
+ "# model.push_to_hub(repo_name)\n",
464
+ "# tokenizer.push_to_hub(repo_name)"
465
+ ]
466
+ },
467
+ {
468
+ "cell_type": "code",
469
+ "source": [
470
+ "model.save_pretrained('/content/final_model')\n",
471
+ "tokenizer.save_pretrained('/content/final_model')"
472
+ ],
473
+ "metadata": {
474
+ "id": "RfSas7HWwNfG"
475
+ },
476
+ "execution_count": null,
477
+ "outputs": []
478
+ },
479
+ {
480
+ "cell_type": "code",
481
+ "execution_count": null,
482
+ "metadata": {
483
+ "id": "POq_UyFZw-mS"
484
+ },
485
+ "outputs": [],
486
+ "source": [
487
+ "# After inference\n",
488
+ "del tokenized_inputs, outputs, logits\n",
489
+ "torch.cuda.empty_cache() # Clear unused memory\n"
490
+ ]
491
+ },
492
+ {
493
+ "cell_type": "code",
494
+ "execution_count": null,
495
+ "metadata": {
496
+ "id": "s4p4Lv0Ry_J5"
497
+ },
498
+ "outputs": [],
499
+ "source": [
500
+ "from datasets import load_dataset\n",
501
+ "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
502
+ "import torch\n",
503
+ "from sklearn.metrics import accuracy_score\n",
504
+ "\n",
505
+ "# Load the dataset from Hugging Face Hub (test split)\n",
506
+ "dataset = load_dataset(\"AiisNothing/test_data\", split=\"test\")\n",
507
+ "\n",
508
+ "# Load the tokenizer and the model from your Hugging Face model repository\n",
509
+ "repo_name = \"AiisNothing/electra-discriminator-trained-merged-dataset-version1\" # Replace with your repo name\n",
510
+ "tokenizer = AutoTokenizer.from_pretrained(repo_name)\n",
511
+ "model = AutoModelForSequenceClassification.from_pretrained(repo_name)\n",
512
+ "\n",
513
+ "# Move model to GPU if available and set to eval mode\n",
514
+ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
515
+ "model.to(device)\n",
516
+ "model.eval() # Set the model to evaluation mode\n",
517
+ "\n",
518
+ "# Prepare inputs from the dataset (assuming the 'dialogue' column contains the text and 'label' contains the labels)\n",
519
+ "inputs = dataset['dialogue']\n",
520
+ "true_labels = dataset['label']\n",
521
+ "\n",
522
+ "predicted_labels = []\n",
523
+ "\n",
524
+ "# Process each input one by one\n",
525
+ "for i in range(len(inputs)):\n",
526
+ " # Get the current input\n",
527
+ " current_input = inputs[i]\n",
528
+ "\n",
529
+ " # Tokenize the input\n",
530
+ " tokenized_input = tokenizer(current_input, padding=True, truncation=True, return_tensors=\"pt\", max_length=256)\n",
531
+ "\n",
532
+ " # Move the tokenized input to GPU\n",
533
+ " tokenized_input = {k: v.to(device) for k, v in tokenized_input.items()}\n",
534
+ "\n",
535
+ " # Perform inference (disable gradients for faster evaluation)\n",
536
+ " with torch.no_grad():\n",
537
+ " outputs = model(**tokenized_input)\n",
538
+ "\n",
539
+ " # Get the logits (raw predictions)\n",
540
+ " logits = outputs.logits\n",
541
+ "\n",
542
+ " # Convert logits to predicted class (using argmax)\n",
543
+ " predicted_labels.append(torch.argmax(logits, dim=-1).cpu().item()) # Use .item() to get a Python number\n",
544
+ "\n",
545
+ " # Clear GPU memory\n",
546
+ " del tokenized_input, outputs, logits\n",
547
+ " torch.cuda.empty_cache() # Clear unused memory\n",
548
+ "\n",
549
+ "# Calculate accuracy\n",
550
+ "accuracy = accuracy_score(true_labels, predicted_labels)\n",
551
+ "\n",
552
+ "# Report accuracy\n",
553
+ "print(f\"Model Accuracy on Test Split: {accuracy * 100:.2f}%\")\n"
554
+ ]
555
+ },
556
+ {
557
+ "cell_type": "code",
558
+ "execution_count": null,
559
+ "metadata": {
560
+ "id": "9qSC8A70vGJ8"
561
+ },
562
+ "outputs": [],
563
+ "source": [
564
+ "accuracy"
565
+ ]
566
+ },
567
+ {
568
+ "cell_type": "code",
569
+ "execution_count": null,
570
+ "metadata": {
571
+ "id": "I3XtYBPa0UVE"
572
+ },
573
+ "outputs": [],
574
+ "source": [
575
+ "pip install optimum[exporters]"
576
+ ]
577
+ },
578
+ {
579
+ "cell_type": "code",
580
+ "source": [
581
+ "from optimum.onnxruntime import ORTModelForSequenceClassification\n",
582
+ "from transformers import AutoTokenizer\n",
583
+ "from onnxruntime.quantization import quantize_dynamic, QuantType\n",
584
+ "\n",
585
+ "model_checkpoint = \"\"\n",
586
+ "save_directory = \"\"\n",
587
+ "\n",
588
+ "# Load a model from transformers and export it to ONNX\n",
589
+ "ort_model = ORTModelForSequenceClassification.from_pretrained(model_checkpoint, export=True)\n",
590
+ "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)\n",
591
+ "\n",
592
+ "# Save the ONNX model and tokenizer\n",
593
+ "ort_model.save_pretrained(save_directory)\n",
594
+ "tokenizer.save_pretrained(save_directory)\n",
595
+ "\n",
596
+ "# Quantize the exported ONNX model to 8-bit\n",
597
+ "onnx_model_path = f\"{save_directory}/model.onnx\"\n",
598
+ "quantized_model_path = f\"{save_directory}/model-quantized.onnx\"\n",
599
+ "\n",
600
+ "# Apply dynamic quantization\n",
601
+ "quantize_dynamic(\n",
602
+ " model_input=onnx_model_path,\n",
603
+ " model_output=quantized_model_path,\n",
604
+ " weight_type=QuantType.QUInt8 # Quantize weights to 8-bit\n",
605
+ ")\n",
606
+ "\n",
607
+ "print(f\"Quantized model saved to: {quantized_model_path}\")"
608
+ ],
609
+ "metadata": {
610
+ "id": "PFWPfabCwCZe"
611
+ },
612
+ "execution_count": null,
613
+ "outputs": []
614
+ }
615
+ ],
616
+ "metadata": {
617
+ "accelerator": "GPU",
618
+ "colab": {
619
+ "gpuType": "T4",
620
+ "provenance": []
621
+ },
622
+ "kernelspec": {
623
+ "display_name": "Python 3",
624
+ "name": "python3"
625
+ },
626
+ "language_info": {
627
+ "name": "python"
628
+ }
629
+ },
630
+ "nbformat": 4,
631
+ "nbformat_minor": 0
632
+ }