[base][abaoxomtieu]: upload folder src

app.py

@@ -18,8 +18,6 @@ import asyncio
 import os
 import functools
 import threading
-import sys
-sys.path.append("./src")
 load_dotenv()
 app = FastAPI(docs_url="/")
 app.add_middleware(

src/config/llm.py

@@ -0,0 +1,5 @@
+from langchain_google_genai import GoogleGenerativeAI
+llm = GoogleGenerativeAI(
+    model="gemini-1.5-flash",
+    temperature=0,
+)

src/inference/segment_inference.py

@@ -0,0 +1,23 @@
+import onnxruntime as ort
+from src.utils.utils_segment import preprocess, postprocess
+import numpy as np
+
+
+def inference(image: np.array, model_path, threshold_confidence=0.5, threshold_iou=0.7):
+    model = ort.InferenceSession(
+        model_path,
+    )
+    input = preprocess(image)
+    outputs = postprocess(
+        model.run(None, {"images": input}),
+        threshold_confidence=threshold_confidence,
+        threshold_iou=threshold_iou,
+    )
+
+    return outputs
+
+
+if __name__ == "__main__":
+    model_path = "../model/segment.onnx"
+    image_path = "../../test.jpg"
+    print(inference(image_path, model_path))

src/model/segment.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e38660d6cb501bc21d33249f1e1dffd9038d57a82f27bd089746e1aa8eca53a9
+size 109150130

src/notebook/data.yaml

@@ -0,0 +1,6 @@
+train: /cvparsing-2/train/images
+val: /cvparsing-2/valid/images
+test: /cvparsing-2/test/images
+
+nc: 14
+names: ['Achievement', 'Certifications', 'Community', 'Contact', 'Education', 'Experience', 'Interests', 'Languages', 'Name', 'Profil', 'Projects', 'image', 'resume', 'skills']

src/notebook/notebook.ipynb

@@ -0,0 +1,455 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading Roboflow workspace...\n",
+      "loading Roboflow project...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Downloading Dataset Version Zip in cvparsing-2 to yolov9:: 100%|██████████| 63864/63864 [00:04<00:00, 15236.33it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting Dataset Version Zip to cvparsing-2 in yolov9:: 100%|██████████| 2344/2344 [00:00<00:00, 5118.00it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install roboflow\n",
+    "\n",
+    "from roboflow import Roboflow\n",
+    "rf = Roboflow(api_key=\"ZvM6LUyWI7hiVw6K64bt\")\n",
+    "project = rf.workspace(\"capitaletech-wrnth\").project(\"annotation-moxcs\")\n",
+    "version = project.version(2)\n",
+    "dataset = version.download(\"yolov8\")\n",
+    "                "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: ultralytics in d:\\fu\\dat\\.venv\\lib\\site-packages (8.2.90)\n",
+      "Requirement already satisfied: numpy<2.0.0,>=1.23.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (1.26.4)\n",
+      "Requirement already satisfied: matplotlib>=3.3.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (3.9.2)\n",
+      "Requirement already satisfied: opencv-python>=4.6.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (4.10.0.84)\n",
+      "Requirement already satisfied: pillow>=7.1.2 in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (10.4.0)\n",
+      "Requirement already satisfied: pyyaml>=5.3.1 in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (6.0.2)\n",
+      "Requirement already satisfied: requests>=2.23.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (2.32.3)\n",
+      "Requirement already satisfied: scipy>=1.4.1 in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (1.14.1)\n",
+      "Requirement already satisfied: torch>=1.8.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (2.4.1)\n",
+      "Requirement already satisfied: torchvision>=0.9.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (0.19.1)\n",
+      "Requirement already satisfied: tqdm>=4.64.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (4.66.5)\n",
+      "Requirement already satisfied: psutil in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (6.0.0)\n",
+      "Requirement already satisfied: py-cpuinfo in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (9.0.0)\n",
+      "Requirement already satisfied: pandas>=1.1.4 in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (2.2.2)\n",
+      "Requirement already satisfied: seaborn>=0.11.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (0.13.2)\n",
+      "Requirement already satisfied: ultralytics-thop>=2.0.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from ultralytics) (2.0.6)\n",
+      "Requirement already satisfied: contourpy>=1.0.1 in d:\\fu\\dat\\.venv\\lib\\site-packages (from matplotlib>=3.3.0->ultralytics) (1.3.0)\n",
+      "Requirement already satisfied: cycler>=0.10 in d:\\fu\\dat\\.venv\\lib\\site-packages (from matplotlib>=3.3.0->ultralytics) (0.12.1)\n",
+      "Requirement already satisfied: fonttools>=4.22.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from matplotlib>=3.3.0->ultralytics) (4.53.1)\n",
+      "Requirement already satisfied: kiwisolver>=1.3.1 in d:\\fu\\dat\\.venv\\lib\\site-packages (from matplotlib>=3.3.0->ultralytics) (1.4.7)\n",
+      "Requirement already satisfied: packaging>=20.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from matplotlib>=3.3.0->ultralytics) (24.1)\n",
+      "Requirement already satisfied: pyparsing>=2.3.1 in d:\\fu\\dat\\.venv\\lib\\site-packages (from matplotlib>=3.3.0->ultralytics) (3.1.4)\n",
+      "Requirement already satisfied: python-dateutil>=2.7 in d:\\fu\\dat\\.venv\\lib\\site-packages (from matplotlib>=3.3.0->ultralytics) (2.9.0.post0)\n",
+      "Requirement already satisfied: pytz>=2020.1 in d:\\fu\\dat\\.venv\\lib\\site-packages (from pandas>=1.1.4->ultralytics) (2024.1)\n",
+      "Requirement already satisfied: tzdata>=2022.7 in d:\\fu\\dat\\.venv\\lib\\site-packages (from pandas>=1.1.4->ultralytics) (2024.1)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in d:\\fu\\dat\\.venv\\lib\\site-packages (from requests>=2.23.0->ultralytics) (3.3.2)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in d:\\fu\\dat\\.venv\\lib\\site-packages (from requests>=2.23.0->ultralytics) (3.7)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in d:\\fu\\dat\\.venv\\lib\\site-packages (from requests>=2.23.0->ultralytics) (2.2.2)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in d:\\fu\\dat\\.venv\\lib\\site-packages (from requests>=2.23.0->ultralytics) (2024.8.30)\n",
+      "Requirement already satisfied: filelock in d:\\fu\\dat\\.venv\\lib\\site-packages (from torch>=1.8.0->ultralytics) (3.16.0)\n",
+      "Requirement already satisfied: typing-extensions>=4.8.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from torch>=1.8.0->ultralytics) (4.12.2)\n",
+      "Requirement already satisfied: sympy in d:\\fu\\dat\\.venv\\lib\\site-packages (from torch>=1.8.0->ultralytics) (1.13.2)\n",
+      "Requirement already satisfied: networkx in d:\\fu\\dat\\.venv\\lib\\site-packages (from torch>=1.8.0->ultralytics) (3.3)\n",
+      "Requirement already satisfied: jinja2 in d:\\fu\\dat\\.venv\\lib\\site-packages (from torch>=1.8.0->ultralytics) (3.1.4)\n",
+      "Requirement already satisfied: fsspec in d:\\fu\\dat\\.venv\\lib\\site-packages (from torch>=1.8.0->ultralytics) (2024.9.0)\n",
+      "Requirement already satisfied: colorama in d:\\fu\\dat\\.venv\\lib\\site-packages (from tqdm>=4.64.0->ultralytics) (0.4.6)\n",
+      "Requirement already satisfied: six>=1.5 in d:\\fu\\dat\\.venv\\lib\\site-packages (from python-dateutil>=2.7->matplotlib>=3.3.0->ultralytics) (1.16.0)\n",
+      "Requirement already satisfied: MarkupSafe>=2.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from jinja2->torch>=1.8.0->ultralytics) (2.1.5)\n",
+      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in d:\\fu\\dat\\.venv\\lib\\site-packages (from sympy->torch>=1.8.0->ultralytics) (1.3.0)\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install ultralytics"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "yaml_text = \"\"\"train: /cvparsing-2/train/images\n",
+    "val: /cvparsing-2/valid/images\n",
+    "test: /cvparsing-2/test/images\n",
+    "\n",
+    "nc: 14\n",
+    "names: ['Achievement', 'Certifications', 'Community', 'Contact', 'Education', 'Experience', 'Interests', 'Languages', 'Name', 'Profil', 'Projects', 'image', 'resume', 'skills']\"\"\"\n",
+    "\n",
+    "with open(\"./data.yaml\", 'w') as file:\n",
+    "    file.write(yaml_text),\n",
+    "\n",
+    "# To display the content of the file, you can use the 'cat' command like this:\n",
+    "# %cat /kaggle/working/data.yaml\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!yolo train model=yolov9c.yaml data=D:/FU/DAT/src/notebook/datasets/data.yaml epochs=100 imgsz=640 device=0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Ultralytics YOLOv8.2.90  Python-3.11.9 torch-2.4.1+cu118 CUDA:0 (NVIDIA GeForce RTX 4050 Laptop GPU, 6140MiB)\n",
+      "Setup complete  (20 CPUs, 15.7 GB RAM, 33.9/97.7 GB disk)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# %pip install ultralytics\n",
+    "import ultralytics\n",
+    "ultralytics.checks()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "d:\\FU\\DAT\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n",
+      "d:\\FU\\DAT\\.venv\\Lib\\site-packages\\huggingface_hub\\file_download.py:159: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\htbqn\\.cache\\huggingface\\hub\\models--microsoft--trocr-base-handwritten. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
+      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
+      "  warnings.warn(message)\n",
+      "d:\\FU\\DAT\\.venv\\Lib\\site-packages\\transformers\\tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
+      "  warnings.warn(\n",
+      "Some weights of VisionEncoderDecoderModel were not initialized from the model checkpoint at microsoft/trocr-base-handwritten and are newly initialized: ['encoder.pooler.dense.bias', 'encoder.pooler.dense.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
+      "d:\\FU\\DAT\\.venv\\Lib\\site-packages\\transformers\\generation\\utils.py:1258: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import TrOCRProcessor, VisionEncoderDecoderModel\n",
+    "from PIL import Image\n",
+    "import requests\n",
+    "\n",
+    "# load image from the IAM database\n",
+    "# url = 'https://fki.tic.heia-fr.ch/static/img/a01-122-02-00.jpg'\n",
+    "image = Image.open(r'./images.png').convert(\"RGB\")\n",
+    "\n",
+    "processor = TrOCRProcessor.from_pretrained('microsoft/trocr-base-handwritten')\n",
+    "model = VisionEncoderDecoderModel.from_pretrained('microsoft/trocr-base-handwritten')\n",
+    "pixel_values = processor(images=image, return_tensors=\"pt\").pixel_values\n",
+    "\n",
+    "generated_ids = model.generate(pixel_values)\n",
+    "generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([[  2, 288, 321,   2]])"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "generated_ids"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import onnxruntime as ort\n",
+    "import numpy as np\n",
+    "from PIL import Image\n",
+    "\n",
+    "# Load the ONNX model\n",
+    "model_path = \"../model/section_detection.onnx\"\n",
+    "session = ort.InferenceSession(model_path)\n",
+    "\n",
+    "# Load and preprocess the image\n",
+    "image_path = 'D:/FU/DAT/src/notebook/datasets/train/images/1629756071561_jpg.rf.05f192117b5f0f8125474abdf3392f72.jpg'\n",
+    "image = Image.open(image_path)\n",
+    "image_data = np.array(image).astype('float32').transpose(2, 0, 1)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(1, 3, 640, 640)"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "image_data = np.expand_dims(image_data, axis=0)\n",
+    "image_data.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "input_name = session.get_inputs()[0].name\n",
+    "output_name = session.get_outputs()[0].name"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result = session.run([output_name], {input_name: image_data})[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(18, 8400)"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "result[0].shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "WARNING  Unable to automatically guess model task, assuming 'task=detect'. Explicitly define task for your model, i.e. 'task=detect', 'segment', 'classify','pose' or 'obb'.\n",
+      "Loading ..\\model\\section_detection.onnx for ONNX Runtime inference...\n",
+      "\n",
+      "image 1/1 D:\\FU\\DAT\\src\\notebook\\datasets\\train\\images\\1629756071561_jpg.rf.05f192117b5f0f8125474abdf3392f72.jpg: 640x640 2 Achievements, 147.6ms\n",
+      "Speed: 2.5ms preprocess, 147.6ms inference, 2.5ms postprocess per image at shape (1, 3, 640, 640)\n"
+     ]
+    }
+   ],
+   "source": [
+    "from ultralytics import YOLO\n",
+    "\n",
+    "# Load the YOLOv8 model'\n",
+    "\n",
+    "# Load the exported ONNX model\n",
+    "onnx_model = YOLO(\"../model/section_detection.onnx\")\n",
+    "\n",
+    "# Run inference\n",
+    "results = onnx_model(\"D:/FU/DAT/src/notebook/datasets/train/images/1629756071561_jpg.rf.05f192117b5f0f8125474abdf3392f72.jpg\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for result in results:\n",
+    "    boxes = result.boxes  # Boxes object for bounding box outputs\n",
+    "    masks = result.masks  # Masks object for segmentation masks outputs\n",
+    "    keypoints = result.keypoints  # Keypoints object for pose outputs\n",
+    "    probs = result.probs  # Probs object for classification outputs\n",
+    "    obb = result.obb  # Oriented boxes object for OBB outputs\n",
+    "    result.show()  # display to screen\n",
+    "    result.save(filename=\"result.jpg\")  # save to disk"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "You are already logged into Roboflow. To make a different login,run roboflow.login(force=True).\n"
+     ]
+    }
+   ],
+   "source": [
+    "!roboflow login"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "RoboflowAPINotAuthorizedError",
+     "evalue": "Unauthorized access to roboflow API - check API key. Visit https://docs.roboflow.com/api-reference/authentication#retrieve-an-api-key to learn how to retrieve one.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mHTTPError\u001b[0m                                 Traceback (most recent call last)",
+      "File \u001b[1;32mc:\\Users\\htbqn\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\inference\\core\\roboflow_api.py:80\u001b[0m, in \u001b[0;36mwrap_roboflow_api_errors.<locals>.decorator.<locals>.wrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     79\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m---> 80\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     81\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (requests\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mConnectionError, \u001b[38;5;167;01mConnectionError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m error:\n",
+      "File \u001b[1;32mc:\\Users\\htbqn\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\inference\\core\\roboflow_api.py:227\u001b[0m, in \u001b[0;36mget_roboflow_model_data\u001b[1;34m(api_key, model_id, endpoint_type, device_id)\u001b[0m\n\u001b[0;32m    223\u001b[0m api_url \u001b[38;5;241m=\u001b[39m _add_params_to_url(\n\u001b[0;32m    224\u001b[0m     url\u001b[38;5;241m=\u001b[39m\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mAPI_BASE_URL\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mendpoint_type\u001b[38;5;241m.\u001b[39mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodel_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m    225\u001b[0m     params\u001b[38;5;241m=\u001b[39mparams,\n\u001b[0;32m    226\u001b[0m )\n\u001b[1;32m--> 227\u001b[0m api_data \u001b[38;5;241m=\u001b[39m \u001b[43m_get_from_url\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mapi_url\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    228\u001b[0m cache\u001b[38;5;241m.\u001b[39mset(\n\u001b[0;32m    229\u001b[0m     api_data_cache_key,\n\u001b[0;32m    230\u001b[0m     api_data,\n\u001b[0;32m    231\u001b[0m     expire\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m,\n\u001b[0;32m    232\u001b[0m )\n",
+      "File \u001b[1;32mc:\\Users\\htbqn\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\inference\\core\\roboflow_api.py:473\u001b[0m, in \u001b[0;36m_get_from_url\u001b[1;34m(url, json_response)\u001b[0m\n\u001b[0;32m    472\u001b[0m response \u001b[38;5;241m=\u001b[39m requests\u001b[38;5;241m.\u001b[39mget(wrap_url(url))\n\u001b[1;32m--> 473\u001b[0m \u001b[43mapi_key_safe_raise_for_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresponse\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    474\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m json_response:\n",
+      "File \u001b[1;32mc:\\Users\\htbqn\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\inference\\core\\utils\\requests.py:15\u001b[0m, in \u001b[0;36mapi_key_safe_raise_for_status\u001b[1;34m(response)\u001b[0m\n\u001b[0;32m     14\u001b[0m response\u001b[38;5;241m.\u001b[39murl \u001b[38;5;241m=\u001b[39m API_KEY_PATTERN\u001b[38;5;241m.\u001b[39msub(deduct_api_key, response\u001b[38;5;241m.\u001b[39murl)\n\u001b[1;32m---> 15\u001b[0m \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraise_for_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[1;32mc:\\Users\\htbqn\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\requests\\models.py:1021\u001b[0m, in \u001b[0;36mResponse.raise_for_status\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1020\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m http_error_msg:\n\u001b[1;32m-> 1021\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m HTTPError(http_error_msg, response\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m)\n",
+      "\u001b[1;31mHTTPError\u001b[0m: 401 Client Error: Unauthorized for url: https://api.roboflow.com/ort/annotation-moxcs/2?nocache=true&device=ABAOXOMTIEU&dynamic=true",
+      "\nThe above exception was the direct cause of the following exception:\n",
+      "\u001b[1;31mRoboflowAPINotAuthorizedError\u001b[0m             Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[4], line 10\u001b[0m\n\u001b[0;32m      7\u001b[0m image \u001b[38;5;241m=\u001b[39m cv2\u001b[38;5;241m.\u001b[39mimread(image_file)\n\u001b[0;32m      9\u001b[0m \u001b[38;5;66;03m# load a pre-trained yolov8n model\u001b[39;00m\n\u001b[1;32m---> 10\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[43mget_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mannotation-moxcs/2\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m     12\u001b[0m \u001b[38;5;66;03m# run inference on our chosen image, image can be a url, a numpy array, a PIL image, etc.\u001b[39;00m\n\u001b[0;32m     13\u001b[0m results \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39minfer(image)[\u001b[38;5;241m0\u001b[39m]\n",
+      "File \u001b[1;32mc:\\Users\\htbqn\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\inference\\models\\utils.py:275\u001b[0m, in \u001b[0;36mget_model\u001b[1;34m(model_id, api_key, **kwargs)\u001b[0m\n\u001b[0;32m    274\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_model\u001b[39m(model_id, api_key\u001b[38;5;241m=\u001b[39mAPI_KEY, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Model:\n\u001b[1;32m--> 275\u001b[0m     task, model \u001b[38;5;241m=\u001b[39m \u001b[43mget_model_type\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mapi_key\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mapi_key\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    276\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m ROBOFLOW_MODEL_TYPES[(task, model)](model_id, api_key\u001b[38;5;241m=\u001b[39mapi_key, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+      "File \u001b[1;32mc:\\Users\\htbqn\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\inference\\core\\registries\\roboflow.py:115\u001b[0m, in \u001b[0;36mget_model_type\u001b[1;34m(model_id, api_key)\u001b[0m\n\u001b[0;32m    108\u001b[0m     save_model_metadata_in_cache(\n\u001b[0;32m    109\u001b[0m         dataset_id\u001b[38;5;241m=\u001b[39mdataset_id,\n\u001b[0;32m    110\u001b[0m         version_id\u001b[38;5;241m=\u001b[39mversion_id,\n\u001b[0;32m    111\u001b[0m         project_task_type\u001b[38;5;241m=\u001b[39mproject_task_type,\n\u001b[0;32m    112\u001b[0m         model_type\u001b[38;5;241m=\u001b[39mmodel_type,\n\u001b[0;32m    113\u001b[0m     )\n\u001b[0;32m    114\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m project_task_type, model_type\n\u001b[1;32m--> 115\u001b[0m api_data \u001b[38;5;241m=\u001b[39m \u001b[43mget_roboflow_model_data\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    116\u001b[0m \u001b[43m    \u001b[49m\u001b[43mapi_key\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mapi_key\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    117\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmodel_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    118\u001b[0m \u001b[43m    \u001b[49m\u001b[43mendpoint_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mModelEndpointType\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mORT\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    119\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdevice_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mGLOBAL_DEVICE_ID\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    120\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mort\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    121\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m api_data \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    122\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m ModelArtefactError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mError loading model artifacts from Roboflow API.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "File \u001b[1;32mc:\\Users\\htbqn\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\inference\\core\\roboflow_api.py:93\u001b[0m, in \u001b[0;36mwrap_roboflow_api_errors.<locals>.decorator.<locals>.wrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     91\u001b[0m     error_handler \u001b[38;5;241m=\u001b[39m user_handler_override\u001b[38;5;241m.\u001b[39mget(status_code, default_handler)\n\u001b[0;32m     92\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m error_handler \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m---> 93\u001b[0m         \u001b[43merror_handler\u001b[49m\u001b[43m(\u001b[49m\u001b[43merror\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     94\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m RoboflowAPIUnsuccessfulRequestError(\n\u001b[0;32m     95\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnsuccessful request to Roboflow API with response code: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mstatus_code\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     96\u001b[0m     ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merror\u001b[39;00m\n\u001b[0;32m     97\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mInvalidJSONError \u001b[38;5;28;01mas\u001b[39;00m error:\n",
+      "File \u001b[1;32mc:\\Users\\htbqn\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\inference\\core\\roboflow_api.py:60\u001b[0m, in \u001b[0;36m<lambda>\u001b[1;34m(e)\u001b[0m\n\u001b[0;32m     53\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mraise_from_lambda\u001b[39m(\n\u001b[0;32m     54\u001b[0m     inner_error: \u001b[38;5;167;01mException\u001b[39;00m, exception_type: Type[\u001b[38;5;167;01mException\u001b[39;00m], message: \u001b[38;5;28mstr\u001b[39m\n\u001b[0;32m     55\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m     56\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m exception_type(message) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01minner_error\u001b[39;00m\n\u001b[0;32m     59\u001b[0m DEFAULT_ERROR_HANDLERS \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m---> 60\u001b[0m     \u001b[38;5;241m401\u001b[39m: \u001b[38;5;28;01mlambda\u001b[39;00m e: \u001b[43mraise_from_lambda\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m     61\u001b[0m \u001b[43m        \u001b[49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     62\u001b[0m \u001b[43m        \u001b[49m\u001b[43mRoboflowAPINotAuthorizedError\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     63\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mUnauthorized access to roboflow API - check API key. Visit \u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\n\u001b[0;32m     64\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mhttps://docs.roboflow.com/api-reference/authentication#retrieve-an-api-key to learn how to retrieve one.\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m     65\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m,\n\u001b[0;32m     66\u001b[0m     \u001b[38;5;241m404\u001b[39m: \u001b[38;5;28;01mlambda\u001b[39;00m e: raise_from_lambda(\n\u001b[0;32m     67\u001b[0m         e, RoboflowAPINotNotFoundError, NOT_FOUND_ERROR_MESSAGE\n\u001b[0;32m     68\u001b[0m     ),\n\u001b[0;32m     69\u001b[0m }\n\u001b[0;32m     72\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrap_roboflow_api_errors\u001b[39m(\n\u001b[0;32m     73\u001b[0m     http_errors_handlers: Optional[\n\u001b[0;32m     74\u001b[0m         Dict[\u001b[38;5;28mint\u001b[39m, Callable[[Union[requests\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mHTTPError]], \u001b[38;5;28;01mNone\u001b[39;00m]]\n\u001b[0;32m     75\u001b[0m     ] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m     76\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mcallable\u001b[39m:\n\u001b[0;32m     77\u001b[0m     \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorator\u001b[39m(function: \u001b[38;5;28mcallable\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mcallable\u001b[39m:\n",
+      "File \u001b[1;32mc:\\Users\\htbqn\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\inference\\core\\roboflow_api.py:56\u001b[0m, in \u001b[0;36mraise_from_lambda\u001b[1;34m(inner_error, exception_type, message)\u001b[0m\n\u001b[0;32m     53\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mraise_from_lambda\u001b[39m(\n\u001b[0;32m     54\u001b[0m     inner_error: \u001b[38;5;167;01mException\u001b[39;00m, exception_type: Type[\u001b[38;5;167;01mException\u001b[39;00m], message: \u001b[38;5;28mstr\u001b[39m\n\u001b[0;32m     55\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m---> 56\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m exception_type(message) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01minner_error\u001b[39;00m\n",
+      "\u001b[1;31mRoboflowAPINotAuthorizedError\u001b[0m: Unauthorized access to roboflow API - check API key. Visit https://docs.roboflow.com/api-reference/authentication#retrieve-an-api-key to learn how to retrieve one."
+     ]
+    }
+   ],
+   "source": [
+    "from inference import get_model\n",
+    "import supervision as sv\n",
+    "import cv2\n",
+    "\n",
+    "# define the image url to use for inference\n",
+    "image_file = \"taylor-swift-album-1989.jpeg\"\n",
+    "image = cv2.imread(image_file)\n",
+    "\n",
+    "# load a pre-trained yolov8n model\n",
+    "model = get_model(model_id=\"annotation-moxcs/2\")\n",
+    "\n",
+    "# run inference on our chosen image, image can be a url, a numpy array, a PIL image, etc.\n",
+    "results = model.infer(image)[0]\n",
+    "\n",
+    "# load the results into the supervision Detections api\n",
+    "detections = sv.Detections.from_inference(results)\n",
+    "\n",
+    "# create supervision annotators\n",
+    "bounding_box_annotator = sv.BoundingBoxAnnotator()\n",
+    "label_annotator = sv.LabelAnnotator()\n",
+    "\n",
+    "# annotate the image with our inference results\n",
+    "annotated_image = bounding_box_annotator.annotate(\n",
+    "    scene=image, detections=detections)\n",
+    "annotated_image = label_annotator.annotate(\n",
+    "    scene=annotated_image, detections=detections)\n",
+    "\n",
+    "# display the image\n",
+    "sv.plot_image(annotated_image)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/notebook/seg.ipynb

@@ -0,0 +1,86 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from PIL import Image \n",
+    "from pytesseract import pytesseract "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "It was the best of\n",
+      "times, it was the worst\n",
+      "of times, it was the age\n",
+      "of wisdom, it was the\n",
+      "age of foolishness...\n"
+     ]
+    }
+   ],
+   "source": [
+    "from PIL import Image \n",
+    "from pytesseract import pytesseract \n",
+    " \n",
+    "path_to_tesseract = r\"C:\\Program Files\\Tesseract-OCR\\tesseract.exe\"\n",
+    "image_path = r\"./images.png\"\n",
+    "\n",
+    "img = Image.open(image_path) \n",
+    "\n",
+    "\n",
+    "pytesseract.tesseract_cmd = path_to_tesseract \n",
+    "\n",
+    "\n",
+    "text = pytesseract.image_to_string(img) \n",
+    "\n",
+    "print(text[:-1])\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from "
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/prompt/promt.py

@@ -0,0 +1,13 @@
+format_prompt = """
+#Role: You are an expert at correcting spelling errors from interviewee's resume information.
+#Instruction:
+You are provided with a dictionary containing information from the user's resume by an OCR model. It may have misspellings or wrong entries.
+Please correct the spelling of each field.
+Move the content of the fields to more appropriate fields if necessary.
+You must not fabricate information and create new information.
+
+You must return JSON containing the same format as the original format:
+
+#Input:
+My resume is as follows: {input}
+"""

src/training/segment_training.ipynb.ipynb

@@ -0,0 +1 @@
+{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2024-09-08T06:52:15.629296Z","iopub.status.busy":"2024-09-08T06:52:15.628873Z","iopub.status.idle":"2024-09-08T06:52:35.266264Z","shell.execute_reply":"2024-09-08T06:52:35.265350Z","shell.execute_reply.started":"2024-09-08T06:52:15.629258Z"},"trusted":true},"outputs":[],"source":["!pip install roboflow\n","\n","from roboflow import Roboflow\n","rf = Roboflow(api_key=\"ZvM6LUyWI7hiVw6K64bt\")\n","project = rf.workspace(\"capitaletech-wrnth\").project(\"annotation-moxcs\")\n","version = project.version(2)\n","dataset = version.download(\"yolov8\")\n","                "]},{"cell_type":"code","execution_count":4,"metadata":{"execution":{"iopub.execute_input":"2024-09-08T06:52:38.656706Z","iopub.status.busy":"2024-09-08T06:52:38.655712Z","iopub.status.idle":"2024-09-08T06:52:38.661352Z","shell.execute_reply":"2024-09-08T06:52:38.660330Z","shell.execute_reply.started":"2024-09-08T06:52:38.656651Z"},"trusted":true},"outputs":[],"source":["import os\n","os.environ[\"WANDB_DISABLED\"] = \"true\""]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-09-08T06:52:39.882023Z","iopub.status.busy":"2024-09-08T06:52:39.881309Z","iopub.status.idle":"2024-09-08T06:52:54.091799Z","shell.execute_reply":"2024-09-08T06:52:54.090833Z","shell.execute_reply.started":"2024-09-08T06:52:39.881985Z"},"trusted":true},"outputs":[],"source":["!pip install ultralytics==8.0.28"]},{"cell_type":"code","execution_count":13,"metadata":{"execution":{"iopub.execute_input":"2024-09-08T06:56:25.315009Z","iopub.status.busy":"2024-09-08T06:56:25.314467Z","iopub.status.idle":"2024-09-08T06:56:25.322391Z","shell.execute_reply":"2024-09-08T06:56:25.321282Z","shell.execute_reply.started":"2024-09-08T06:56:25.314961Z"},"trusted":true},"outputs":[],"source":["yaml_text = \"\"\"train: /kaggle/working/annotation-2/train\n","val: /kaggle/working/annotation-2/valid\n","test: /kaggle/working/annotation-2/test\n","\n","names:\n","- Certifications\n","- Community\n","- Contact\n","- Education\n","- Experience\n","- Interests\n","- Languages\n","- Name\n","- Profil\n","- Projects\n","- skills\n","nc: 11\n","roboflow:\n","  license: CC BY 4.0\n","  project: annotation-moxcs\n","  url: https://universe.roboflow.com/capitaletech-wrnth/annotation-moxcs/dataset/2\n","  version: 2\n","  workspace: capitaletech-wrnth\n","\"\"\"\n","with open(\"./data.yaml\", 'w') as file:\n","    file.write(yaml_text),\n"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-09-08T06:56:26.844555Z","iopub.status.busy":"2024-09-08T06:56:26.844168Z","iopub.status.idle":"2024-09-08T09:22:07.888695Z","shell.execute_reply":"2024-09-08T09:22:07.887404Z","shell.execute_reply.started":"2024-09-08T06:56:26.844520Z"},"trusted":true},"outputs":[],"source":["!yolo task=segment mode=train model=yolov8m-seg.pt  data=/kaggle/working/data.yaml epochs=100 imgsz=640\n","# !ls {HOME}/runs/segment/train/\n"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-09-08T09:22:07.892085Z","iopub.status.busy":"2024-09-08T09:22:07.891230Z","iopub.status.idle":"2024-09-08T09:22:25.963401Z","shell.execute_reply":"2024-09-08T09:22:25.962222Z","shell.execute_reply.started":"2024-09-08T09:22:07.892036Z"},"trusted":true},"outputs":[],"source":["!yolo export model=/kaggle/working/runs/segment/train4/weights/best.pt format=onnx "]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-09-08T06:53:10.988929Z","iopub.status.busy":"2024-09-08T06:53:10.988489Z","iopub.status.idle":"2024-09-08T06:53:11.045752Z","shell.execute_reply":"2024-09-08T06:53:11.044376Z","shell.execute_reply.started":"2024-09-08T06:53:10.988883Z"},"trusted":true},"outputs":[],"source":["from PIL import Image\n","Image(filename=f'/kaggle/working/runs/segment/train/train_batch0.jpg', width=600)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[],"dockerImageVersionId":30762,"isGpuEnabled":true,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"}},"nbformat":4,"nbformat_minor":4}

src/utils/utils_segment.py

@@ -0,0 +1,283 @@
+from PIL import Image
+import numpy as np
+import cv2
+from typing import Tuple
+from pytesseract import pytesseract
+
+# path_to_tesseract = r"C:\Program Files\Tesseract-OCR\tesseract.exe"
+# pytesseract.tesseract_cmd = path_to_tesseract
+class_names = [
+    "Certifications",
+    "Community",
+    "Contact",
+    "Education",
+    "Experience",
+    "Interests",
+    "Languages",
+    "Name",
+    "Profile",
+    "Projects",
+    "Skills",
+]
+number_class_custom = int(len(class_names) + 4)
+img_width, img_height = None, None
+left = None
+top = None
+ratio = None
+
+
+def preprocess(img: np.array, shape=(640, 640)) -> np.array:
+    global img_width, img_height, left, top, ratio
+    img, ratio, (left, top) = resize_and_pad(img, new_shape=shape)
+    img_height, img_width, _ = img.shape
+    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+    img = img.transpose(2, 0, 1)
+    img = img.reshape(1, 3, 640, 640).astype("float32")
+    img = img / 255.0
+    return img
+
+
+def extract_box(outputs):
+    output0 = outputs[0]
+    output1 = outputs[1]
+    output0 = output0[0].transpose()
+    output1 = output1[0]
+    boxes = output0[:, 0:number_class_custom]
+    masks = output0[:, number_class_custom:]
+    output1 = output1.reshape(32, 160 * 160)
+    output1 = output1.reshape(32, 160 * 160)
+    masks = masks @ output1
+    boxes = np.hstack([boxes, masks])
+    return boxes
+
+
+def intersection(box1, box2):
+    box1_x1, box1_y1, box1_x2, box1_y2 = box1[:4]
+    box2_x1, box2_y1, box2_x2, box2_y2 = box2[:4]
+    x1 = max(box1_x1, box2_x1)
+    y1 = max(box1_y1, box2_y1)
+    x2 = min(box1_x2, box2_x2)
+    y2 = min(box1_y2, box2_y2)
+    return (x2 - x1) * (y2 - y1)
+
+
+def union(box1, box2):
+    box1_x1, box1_y1, box1_x2, box1_y2 = box1[:4]
+    box2_x1, box2_y1, box2_x2, box2_y2 = box2[:4]
+    box1_area = (box1_x2 - box1_x1) * (box1_y2 - box1_y1)
+    box2_area = (box2_x2 - box2_x1) * (box2_y2 - box2_y1)
+    return box1_area + box2_area - intersection(box1, box2)
+
+
+def iou(box1, box2):
+    return intersection(box1, box2) / union(box1, box2)
+
+
+def sigmoid(z):
+    return 1 / (1 + np.exp(-z))
+
+
+def get_mask(row, box, img_width, img_height, threshold):
+    mask = row.reshape(160, 160)
+    mask = sigmoid(mask)
+    mask = (mask > threshold).astype("uint8") * 255
+    x1, y1, x2, y2 = box
+    mask_x1 = round(x1 / img_width * 160)
+    mask_y1 = round(y1 / img_height * 160)
+    mask_x2 = round(x2 / img_width * 160)
+    mask_y2 = round(y2 / img_height * 160)
+    mask = mask[mask_y1:mask_y2, mask_x1:mask_x2]
+    img_mask = Image.fromarray(mask, "L")
+    img_mask = img_mask.resize((round(x2 - x1), round(y2 - y1)))
+    mask = np.array(img_mask)
+    return mask
+
+
+def get_polygon(mask):
+    contours = cv2.findContours(mask, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
+    polygon = [[contour[0][0], contour[0][1]] for contour in contours[0][0]]
+    return polygon
+
+
+def postprocess(outputs, threshold_confidence, threshold_iou):
+    objects = []
+    for row in extract_box(outputs):
+        xc, yc, w, h = row[:4]
+        x1 = (xc - w / 2) / 640 * img_width
+        y1 = (yc - h / 2) / 640 * img_height
+        x2 = (xc + w / 2) / 640 * img_width
+        y2 = (yc + h / 2) / 640 * img_height
+        prob = row[4:number_class_custom].max()
+        if prob < threshold_confidence:
+            continue
+        class_id = row[4:number_class_custom].argmax()
+        label = class_names[class_id]
+        # mask = get_mask(
+        #     row[number_class_custom:25684],
+        #     (x1, y1, x2, y2),
+        #     img_width,
+        #     img_height,
+        #     threshold=threshold,
+        # )
+        # polygon = get_polygon(mask)
+        # objects.append([x1, y1, x2, y2, label, prob, mask, polygon])
+        objects.append([x1, y1, x2, y2, label, prob])
+
+    # apply non-maximum suppression
+    objects.sort(key=lambda x: x[5], reverse=True)
+    result = []
+    while objects:
+        obj = objects.pop(0)
+        result.append(obj)
+        objects = [
+            other_obj for other_obj in objects if iou(other_obj, obj) < threshold_iou
+        ]
+    del objects
+
+    cropped_images = [
+        {
+            "box": list(map(int, unpad_and_resize_boxes(obj[:4], ratio, left, top))),
+            "label": obj[4],
+            "prob": int(obj[5]),
+        }
+        for obj in result
+    ]
+    return cropped_images
+
+
+def extract_text_dict(outputs):
+    result_dict = {}
+    for output in outputs:
+        label = output.get("label").lower()
+        text = output.get("text")
+        if label in result_dict:
+            result_dict[label] += " " + text
+        else:
+            result_dict[label] = text
+
+    return result_dict
+
+
+def extract_text(outputs, image_origin):
+    for i in range(len(outputs)):
+        image = crop_image(image_origin, outputs[i].get("box"))
+        text = pytesseract.image_to_string(image)
+        outputs[i].update({"text": text})
+        if "text" in outputs[i]:
+            outputs[i]["text"] += text
+        else:
+            outputs[i].update({"text": text})
+    return extract_text_dict(outputs)
+
+
+def crop_image(image, box):
+
+    x1, y1, x2, y2 = map(int, box)
+    cropped_image = image[y1:y2, x1:x2]
+    return cropped_image
+
+
+def resize_and_pad(
+    image: np.array,
+    new_shape: Tuple[int, int],
+    padding_color: Tuple[int] = (144, 144, 144),
+) -> np.array:
+    h_org, w_org = image.shape[:2]
+    w_new, h_new = new_shape
+    padd_left, padd_right, padd_top, padd_bottom = 0, 0, 0, 0
+
+    # Padding left to right
+    if h_org >= w_org:
+        img_resize = cv2.resize(image, (int(w_org * h_new / h_org), h_new))
+        h, w = img_resize.shape[:2]
+        padd_left = (w_new - w) // 2
+        padd_right = w_new - w - padd_left
+        ratio = h_new / h_org
+
+    # Padding top to bottom
+    if h_org < w_org:
+        img_resize = cv2.resize(image, (w_new, int(h_org * w_new / w_org)))
+        h, w = img_resize.shape[:2]
+        padd_top = (h_new - h) // 2
+        padd_bottom = h_new - h - padd_top
+        ratio = w_new / w_org
+
+    image = cv2.copyMakeBorder(
+        img_resize,
+        padd_top,
+        padd_bottom,
+        padd_left,
+        padd_right,
+        cv2.BORDER_CONSTANT,
+        None,
+        value=padding_color,
+    )
+
+    return image, ratio, (padd_left, padd_top)
+
+
+def unpad_and_resize_boxes(boxes, ratio, left, top):
+
+    if len(boxes) == 0:
+        return boxes
+    boxes = np.array(boxes)
+    if boxes.ndim == 1:
+        boxes = boxes.reshape(-1, 4)
+    boxes[:, [0, 2]] -= left
+    boxes[:, [1, 3]] -= top
+    boxes[:, :4] /= ratio
+    if len(boxes) == 1:
+        return boxes.flatten().tolist()
+    else:
+        return boxes.tolist()
+
+
+def draw_bounding_boxes(image, outputs):
+    # Create a copy of the image to draw on
+    image_with_boxes = image.copy()
+
+    # Define a list of colors for the bounding boxes
+    label_colors = {
+        "Certifications": (255, 0, 0),
+        "Community": (0, 255, 0),
+        "Contact": (0, 0, 255),
+        "Education": (255, 128, 0),
+        "Experience": (255, 0, 255),
+        "Interests": (128, 128, 128),
+        "Languages": (128, 0, 0),
+        "Name": (0, 128, 0),
+        "Profile": (0, 0, 128),
+        "Projects": (128, 128, 0),
+        "Skills": (128, 0, 128),
+    }
+
+    # Draw each bounding box and text
+    for output in outputs:
+        box = output["box"]
+        label = output["label"]
+        text = output.get("text", "")
+
+        # Get the color for the label
+        color = label_colors.get(
+            label, (255, 255, 255)
+        )  # Default to white if label not found
+
+        # Draw the bounding box
+        x1, y1, x2, y2 = box
+        cv2.rectangle(image_with_boxes, (x1, y1), (x2, y2), color, 2)
+        # Draw the label and text
+        cv2.putText(
+            image_with_boxes,
+            f"{label}",
+            (x1, y1 - 10),
+            cv2.FONT_ITALIC,
+            2,
+            color,
+            2,
+        )
+    image_with_boxes_rgb = cv2.cvtColor(image_with_boxes, cv2.COLOR_BGR2RGB)
+
+    # Convert the OpenCV image (numpy array) to a PIL image
+    image_pil = Image.fromarray(image_with_boxes_rgb)
+
+    return image_pil