Upload 9 files

.gitattributes

@@ -0,0 +1 @@
+model.safetensors filter=lfs diff=lfs merge=lfs -text

bert.py

@@ -0,0 +1,37 @@
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import streamlit as st
+import torch
+
+
+MODEL_PATH = "rubert-finetuned"
+model = AutoModelForSequenceClassification.from_pretrained(MODEL_PATH)
+tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
+model.eval()
+
+# === Streamlit UI ===
+st.set_page_config(page_title="Оценка токсичности", layout="centered")
+st.title("💬 Оценка токсичности текста")
+
+text = st.text_area("Введите сообщение", "Ты ужасный человек!")
+submit = st.button("Проверить токсичность")
+
+if submit and text.strip():
+    # Токенизация
+    inputs = tokenizer(text, return_tensors="pt", truncation=True)
+
+    # Предсказание
+    with torch.no_grad():
+        outputs = model(**inputs)
+        logits = outputs.logits
+        score = torch.sigmoid(logits).item()  # степень токсичности
+
+    # Вывод
+    st.subheader("Результат:")
+    st.write(f"**Степень токсичности:** `{score:.3f}`")
+
+    if score > 0.8:
+        st.error("⚠️ Высокая токсичность!")
+    elif score > 0.4:
+        st.warning("⚠️ Средняя токсичность")
+    else:
+        st.success("✅ Низкая токсичность")

config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "cointegrated/rubert-tiny2",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "emb_size": 312,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 312,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 600,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 2048,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 3,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.36.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 83828
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ee8832f961188a19c6bd6e55845433ca1ddd79483ebb461cb69ef64c16fa182
+size 116782884

ru2.ipynb

@@ -0,0 +1,557 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "09d75e68",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "d:\\ds\\ds-phase-2-master\\myenv\\Lib\\site-packages\\huggingface_hub\\file_download.py:896: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
+      "  warnings.warn(\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3a5777df3bb04084bb8717df64fdb6d5",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/11529 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f3a855aaeb844c9ba8ab8d2d141f4ea7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/2883 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at cointegrated/rubert-tiny2 and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "\n",
+    "import pandas as pd\n",
+    "import torch\n",
+    "from datasets import Dataset\n",
+    "from transformers import (\n",
+    "    AutoTokenizer,\n",
+    "    AutoModelForSequenceClassification,\n",
+    "    DataCollatorWithPadding,\n",
+    "    TrainingArguments,\n",
+    "    Trainer\n",
+    ")\n",
+    "\n",
+    "# === 1. Загрузка и преобразование датасета ===\n",
+    "df = pd.read_csv(r\"D:\\ds\\ds-phase-2-master\\Rubert\\labeled(1).csv\").dropna()\n",
+    "df = df[[\"comment\", \"toxic\"]].rename(columns={\"comment\": \"text\", \"toxic\": \"label\"})\n",
+    "df[\"label\"] = df[\"label\"].astype(int)\n",
+    "\n",
+    "ds = Dataset.from_pandas(df)\n",
+    "ds = ds.train_test_split(test_size=0.2, seed=42)\n",
+    "\n",
+    "# === 2. Токенизация ===\n",
+    "model_name = \"cointegrated/rubert-tiny2\"\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+    "\n",
+    "def tokenize(example):\n",
+    "    return tokenizer(example[\"text\"], truncation=True)\n",
+    "\n",
+    "tokenized_ds = ds.map(tokenize, batched=True)\n",
+    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
+    "\n",
+    "# === 3. Загрузка модели ===\n",
+    "model = AutoModelForSequenceClassification.from_pretrained(\n",
+    "    model_name,\n",
+    "    num_labels=1,\n",
+    "    problem_type=\"single_label_classification\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "c6a96f56",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import evaluate\n",
+    "import numpy as np\n",
+    "\n",
+    "accuracy = evaluate.load(\"accuracy\")\n",
+    "\n",
+    "def compute_metrics(eval_pred):\n",
+    "    logits, labels = eval_pred\n",
+    "    preds = (torch.sigmoid(torch.tensor(logits)) > 0.5).int().numpy()  # бинаризация\n",
+    "    return accuracy.compute(predictions=preds, references=labels)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "ed9a53f6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# === 4. Кастомный Trainer с float-метками ===\n",
+    "class CustomTrainer(Trainer):\n",
+    "    def compute_loss(self, model, inputs, return_outputs=False):\n",
+    "        labels = inputs.pop(\"labels\").float().squeeze()\n",
+    "        inputs = {k: v for k, v in inputs.items()}\n",
+    "        outputs = model(**inputs)\n",
+    "        logits = outputs.logits.squeeze()\n",
+    "\n",
+    "        # Проверка размерности\n",
+    "        assert logits.shape == labels.shape, f\"Shape mismatch: logits {logits.shape}, labels {labels.shape}\"\n",
+    "\n",
+    "        loss_fn = torch.nn.BCEWithLogitsLoss()\n",
+    "        loss = loss_fn(logits, labels)\n",
+    "\n",
+    "        return (loss, outputs) if return_outputs else loss\n",
+    "model = model.to(\"cpu\")\n",
+    "\n",
+    "# === 5. Настройки обучения ===\n",
+    "training_args = TrainingArguments(\n",
+    "    output_dir=\"./rubert-finetuned\",\n",
+    "    evaluation_strategy=\"epoch\",\n",
+    "    save_strategy=\"epoch\",\n",
+    "    learning_rate=2e-5,\n",
+    "    per_device_train_batch_size=8,\n",
+    "    per_device_eval_batch_size=8,\n",
+    "    num_train_epochs=10,\n",
+    "    weight_decay=0.01,\n",
+    "    save_total_limit=1,\n",
+    "    load_best_model_at_end=True,\n",
+    "    logging_dir=\"./logs\",)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "fa0b11a9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6394c2f19b0b4730af7de82fdd4b9c8d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/14420 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'loss': 0.41, 'learning_rate': 1.9306518723994454e-05, 'epoch': 0.35}\n",
+      "{'loss': 0.2648, 'learning_rate': 1.8613037447988906e-05, 'epoch': 0.69}\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "feda23f3072d484fa6c0a9bf0a625c7d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/361 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Checkpoint destination directory ./rubert-finetuned\\checkpoint-1442 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'eval_loss': 0.23574143648147583, 'eval_accuracy': 0.9219562955254943, 'eval_runtime': 9.7843, 'eval_samples_per_second': 294.655, 'eval_steps_per_second': 36.896, 'epoch': 1.0}\n",
+      "{'loss': 0.2889, 'learning_rate': 1.791955617198336e-05, 'epoch': 1.04}\n",
+      "{'loss': 0.2153, 'learning_rate': 1.7226074895977807e-05, 'epoch': 1.39}\n",
+      "{'loss': 0.2467, 'learning_rate': 1.6532593619972263e-05, 'epoch': 1.73}\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "475198dc77b04764bc26b06630c99993",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/361 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'eval_loss': 0.2591148912906647, 'eval_accuracy': 0.9223031564342699, 'eval_runtime': 9.2707, 'eval_samples_per_second': 310.978, 'eval_steps_per_second': 38.94, 'epoch': 2.0}\n",
+      "{'loss': 0.2281, 'learning_rate': 1.5839112343966715e-05, 'epoch': 2.08}\n",
+      "{'loss': 0.1865, 'learning_rate': 1.5145631067961166e-05, 'epoch': 2.43}\n",
+      "{'loss': 0.1844, 'learning_rate': 1.4452149791955618e-05, 'epoch': 2.77}\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "06f7aff6306f4c16a4356b590ff94327",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/361 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'eval_loss': 0.26540955901145935, 'eval_accuracy': 0.9288935137010059, 'eval_runtime': 9.2567, 'eval_samples_per_second': 311.45, 'eval_steps_per_second': 38.999, 'epoch': 3.0}\n",
+      "{'loss': 0.1857, 'learning_rate': 1.375866851595007e-05, 'epoch': 3.12}\n",
+      "{'loss': 0.1412, 'learning_rate': 1.3065187239944523e-05, 'epoch': 3.47}\n",
+      "{'loss': 0.1687, 'learning_rate': 1.2371705963938973e-05, 'epoch': 3.81}\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "8cde784edb3848739c66b7262d335a8e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/361 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'eval_loss': 0.3157169818878174, 'eval_accuracy': 0.9219562955254943, 'eval_runtime': 9.2591, 'eval_samples_per_second': 311.368, 'eval_steps_per_second': 38.988, 'epoch': 4.0}\n",
+      "{'loss': 0.1426, 'learning_rate': 1.1678224687933426e-05, 'epoch': 4.16}\n",
+      "{'loss': 0.1228, 'learning_rate': 1.0984743411927878e-05, 'epoch': 4.51}\n",
+      "{'loss': 0.1185, 'learning_rate': 1.029126213592233e-05, 'epoch': 4.85}\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "009d58cef57d4fbeb14ce06b6e2c9c00",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/361 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'eval_loss': 0.31897348165512085, 'eval_accuracy': 0.9257717655220257, 'eval_runtime': 9.2918, 'eval_samples_per_second': 310.274, 'eval_steps_per_second': 38.851, 'epoch': 5.0}\n",
+      "{'loss': 0.1039, 'learning_rate': 9.597780859916783e-06, 'epoch': 5.2}\n",
+      "{'loss': 0.1146, 'learning_rate': 8.904299583911235e-06, 'epoch': 5.55}\n",
+      "{'loss': 0.0903, 'learning_rate': 8.210818307905687e-06, 'epoch': 5.89}\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9e64d3f3da6a4db697a30b93fa80f944",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/361 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'eval_loss': 0.3766789138317108, 'eval_accuracy': 0.9236906000693722, 'eval_runtime': 9.5138, 'eval_samples_per_second': 303.034, 'eval_steps_per_second': 37.945, 'epoch': 6.0}\n",
+      "{'loss': 0.0931, 'learning_rate': 7.5173370319001396e-06, 'epoch': 6.24}\n",
+      "{'loss': 0.0809, 'learning_rate': 6.823855755894592e-06, 'epoch': 6.59}\n",
+      "{'loss': 0.0645, 'learning_rate': 6.130374479889043e-06, 'epoch': 6.93}\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "46b7197e07b642d78e508adf22d06ad4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/361 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'eval_loss': 0.3949281573295593, 'eval_accuracy': 0.9219562955254943, 'eval_runtime': 9.9838, 'eval_samples_per_second': 288.767, 'eval_steps_per_second': 36.158, 'epoch': 7.0}\n",
+      "{'loss': 0.09, 'learning_rate': 5.436893203883496e-06, 'epoch': 7.28}\n",
+      "{'loss': 0.0623, 'learning_rate': 4.743411927877948e-06, 'epoch': 7.63}\n",
+      "{'loss': 0.0651, 'learning_rate': 4.049930651872399e-06, 'epoch': 7.98}\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b36046deee6d49c999568ec571721a86",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/361 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'eval_loss': 0.4220237731933594, 'eval_accuracy': 0.920568851890392, 'eval_runtime': 9.8029, 'eval_samples_per_second': 294.096, 'eval_steps_per_second': 36.826, 'epoch': 8.0}\n",
+      "{'loss': 0.0466, 'learning_rate': 3.356449375866852e-06, 'epoch': 8.32}\n",
+      "{'loss': 0.0724, 'learning_rate': 2.662968099861304e-06, 'epoch': 8.67}\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "38d29c2f867b421e8242b58df331dbe2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/361 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'eval_loss': 0.4382440149784088, 'eval_accuracy': 0.9181408255289629, 'eval_runtime': 9.9388, 'eval_samples_per_second': 290.076, 'eval_steps_per_second': 36.322, 'epoch': 9.0}\n",
+      "{'loss': 0.069, 'learning_rate': 1.969486823855756e-06, 'epoch': 9.02}\n",
+      "{'loss': 0.0614, 'learning_rate': 1.276005547850208e-06, 'epoch': 9.36}\n",
+      "{'loss': 0.0588, 'learning_rate': 5.825242718446603e-07, 'epoch': 9.71}\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6a8d5ddb3c724b7ca59613d8cecf2c09",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/361 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'eval_loss': 0.4383704662322998, 'eval_accuracy': 0.9171002428026361, 'eval_runtime': 9.3759, 'eval_samples_per_second': 307.491, 'eval_steps_per_second': 38.503, 'epoch': 10.0}\n",
+      "{'train_runtime': 422.8343, 'train_samples_per_second': 272.66, 'train_steps_per_second': 34.103, 'train_loss': 0.13939501686995634, 'epoch': 10.0}\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "('rubert-finetuned\\\\tokenizer_config.json',\n",
+       " 'rubert-finetuned\\\\special_tokens_map.json',\n",
+       " 'rubert-finetuned\\\\vocab.txt',\n",
+       " 'rubert-finetuned\\\\added_tokens.json',\n",
+       " 'rubert-finetuned\\\\tokenizer.json')"
+      ]
+     },
+     "execution_count": 30,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# === 6. Trainer ===\n",
+    "trainer = CustomTrainer(\n",
+    "    model=model,\n",
+    "    args=training_args,\n",
+    "    train_dataset=tokenized_ds[\"train\"],\n",
+    "    eval_dataset=tokenized_ds[\"test\"],\n",
+    "    tokenizer=tokenizer,\n",
+    "    data_collator=data_collator,\n",
+    "    compute_metrics=compute_metrics,\n",
+    ")\n",
+    "\n",
+    "# === 7. Обучение ===\n",
+    "trainer.train()\n",
+    "\n",
+    "# === 8. Сохранение ===\n",
+    "model.save_pretrained(\"rubert-finetuned\")\n",
+    "tokenizer.save_pretrained(\"rubert-finetuned\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "e28f42e7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4c469848135a496f9a0d690c367b274f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/361 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy: 0.9220\n"
+     ]
+    }
+   ],
+   "source": [
+    "metrics = trainer.evaluate()\n",
+    "print(f\"Accuracy: {metrics['eval_accuracy']:.4f}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "ccdb2304",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Токсичность: 0.968\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
+    "\n",
+    "\n",
+    "model_path = \"rubert-finetuned\"\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
+    "model = AutoModelForSequenceClassification.from_pretrained(model_path)\n",
+    "model = model.to(\"cuda\")  # или \"cpu\", если хочешь\n",
+    "\n",
+    "def predict_toxicity(text: str) -> float:\n",
+    "    inputs = tokenizer(text, return_tensors=\"pt\", truncation=True, padding=True).to(\"cuda\")\n",
+    "    with torch.no_grad():\n",
+    "        outputs = model(**inputs)\n",
+    "        logits = outputs.logits\n",
+    "        prob = torch.sigmoid(logits).item() \n",
+    "    return prob\n",
+    "\n",
+    "text = \"Ты говно\"\n",
+    "score = predict_toxicity(text)\n",
+    "print(f\"Токсичность: {score:.3f}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "myenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,64 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": false,
+  "mask_token": "[MASK]",
+  "max_length": 512,
+  "model_max_length": 2048,
+  "never_split": null,
+  "pad_to_multiple_of": null,
+  "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "stride": 0,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]"
+}