Varine
/

opus-mt-zh-en-model

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "0b15c2b0-69f1-4201-a937-36bd89b97d3d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Defaulting to user installation because normal site-packages is not writeable\n",
+      "Requirement already satisfied: transformers in /home/ubuntu/.local/lib/python3.10/site-packages (4.40.0)\n",
+      "Requirement already satisfied: datasets in /home/ubuntu/.local/lib/python3.10/site-packages (2.19.0)\n",
+      "Requirement already satisfied: sentencepiece in /home/ubuntu/.local/lib/python3.10/site-packages (0.2.0)\n",
+      "Requirement already satisfied: accelerate in /home/ubuntu/.local/lib/python3.10/site-packages (0.29.3)\n",
+      "Requirement already satisfied: sacrebleu in /home/ubuntu/.local/lib/python3.10/site-packages (2.4.2)\n",
+      "Requirement already satisfied: huggingface_hub in /home/ubuntu/.local/lib/python3.10/site-packages (0.22.2)\n",
+      "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from transformers) (3.6.0)\n",
+      "Requirement already satisfied: numpy>=1.17 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (1.25.2)\n",
+      "Requirement already satisfied: packaging>=20.0 in /usr/lib/python3/dist-packages (from transformers) (21.3)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from transformers) (5.4.1)\n",
+      "Requirement already satisfied: regex!=2019.12.17 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (2024.4.16)\n",
+      "Requirement already satisfied: requests in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (2.31.0)\n",
+      "Requirement already satisfied: tokenizers<0.20,>=0.19 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (0.19.1)\n",
+      "Requirement already satisfied: safetensors>=0.4.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (0.4.3)\n",
+      "Requirement already satisfied: tqdm>=4.27 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (4.66.1)\n",
+      "Requirement already satisfied: pyarrow>=12.0.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (16.0.0)\n",
+      "Requirement already satisfied: pyarrow-hotfix in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.6)\n",
+      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.3.8)\n",
+      "Requirement already satisfied: pandas in /usr/lib/python3/dist-packages (from datasets) (1.3.5)\n",
+      "Requirement already satisfied: xxhash in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (3.4.1)\n",
+      "Requirement already satisfied: multiprocess in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.70.16)\n",
+      "Requirement already satisfied: fsspec<=2024.3.1,>=2023.1.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from fsspec[http]<=2024.3.1,>=2023.1.0->datasets) (2024.3.1)\n",
+      "Requirement already satisfied: aiohttp in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (3.9.5)\n",
+      "Requirement already satisfied: psutil in /usr/lib/python3/dist-packages (from accelerate) (5.9.0)\n",
+      "Requirement already satisfied: torch>=1.10.0 in /usr/lib/python3/dist-packages (from accelerate) (2.0.1)\n",
+      "Requirement already satisfied: portalocker in /home/ubuntu/.local/lib/python3.10/site-packages (from sacrebleu) (2.8.2)\n",
+      "Requirement already satisfied: tabulate>=0.8.9 in /home/ubuntu/.local/lib/python3.10/site-packages (from sacrebleu) (0.9.0)\n",
+      "Requirement already satisfied: colorama in /usr/lib/python3/dist-packages (from sacrebleu) (0.4.4)\n",
+      "Requirement already satisfied: lxml in /usr/lib/python3/dist-packages (from sacrebleu) (4.8.0)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ubuntu/.local/lib/python3.10/site-packages (from huggingface_hub) (4.8.0)\n",
+      "Requirement already satisfied: aiosignal>=1.1.2 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\n",
+      "Requirement already satisfied: attrs>=17.3.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (23.1.0)\n",
+      "Requirement already satisfied: frozenlist>=1.1.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.1)\n",
+      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.5)\n",
+      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.4)\n",
+      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.3)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/.local/lib/python3.10/site-packages (from requests->transformers) (3.3.2)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers) (3.3)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/lib/python3/dist-packages (from requests->transformers) (1.26.5)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->transformers) (2020.6.20)\n",
+      "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
+      "\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "! pip install transformers datasets sentencepiece accelerate sacrebleu huggingface_hub"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "5ce67d29-b717-48a1-9bb1-6436e2a700e2",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024-04-23 05:47:46.579485: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
+      "2024-04-23 05:47:46.810151: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
+      "To enable the following instructions: AVX512F AVX512_VNNI, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
+      "/usr/lib/python3/dist-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.17.3 and <1.25.0 is required for this version of SciPy (detected version 1.25.2\n",
+      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, pipeline\n",
+    "from datasets import load_dataset, load_metric\n",
+    "import numpy as np"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "249efb1d-4723-4544-98c1-4f75dea692a4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/ubuntu/.local/lib/python3.10/site-packages/transformers/models/marian/tokenization_marian.py:175: UserWarning: Recommended: pip install sacremoses.\n",
+      "  warnings.warn(\"Recommended: pip install sacremoses.\")\n"
+     ]
+    }
+   ],
+   "source": [
+    "model_path = 'opus-mt-zh-en-finetuned-zhen-checkpoints'  # 模型保存路径\n",
+    "model = AutoModelForSeq2SeqLM.from_pretrained(model_path)\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "747b0aff-1eb3-42cc-84df-0f6c11cd9fcb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def translate(text, model, tokenizer):\n",
+    "    encoded_text = tokenizer(text, return_tensors=\"pt\", padding=True, truncation=True)\n",
+    "    translated = model.generate(**encoded_text)\n",
+    "    translated_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]\n",
+    "    return translated_text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "ba6141cf-ab92-407c-b7de-2da6f8ad7573",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "input_text = \"Scipy 是 Python 的一个核心科学计算库，它基于 NumPy，并为我们提供了许多用户友好的数学、科学以及工程方面的功能。Scipy 的功能涵盖了多个子模块，如优化、信号处理、图像处理、统计等。\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "748b9ae1-145d-47aa-86c8-7ea924906921",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Translated Text: ['Scippy, a core scientific computing base of Python, is based on NumPy and provides us with many user-friendly mathematics, science, engineering functions. It covers several sub-modules such as optimization, signal processing, image processing, statistics, etc.']\n"
+     ]
+    }
+   ],
+   "source": [
+    "output_text = translate(input_text, model, tokenizer)\n",
+    "print(\"Translated Text:\", output_text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "120a56ac-f6d7-4bb6-893e-bfec6eadd0e8",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}