{ "cells": [ { "cell_type": "markdown", "id": "6867cb23", "metadata": {}, "source": [ "# Triplets pre-processing pipeline" ] }, { "cell_type": "markdown", "id": "4274cfc0", "metadata": {}, "source": [ "This notebook is used for normalization on triplets produced by the triplets extraction pipeline." ] }, { "cell_type": "markdown", "id": "8aecc813", "metadata": {}, "source": [ "Pre-processing configuration:" ] }, { "cell_type": "code", "execution_count": 177, "id": "8c06d7cd", "metadata": {}, "outputs": [], "source": [ "filepath = \"./artifacts/raw_triplets/\"\n", "\n", "triplet_files = [# Category_name, file_name in each category\n", " (\"core_clinical\", [\"core_clinical_numsamples2000_start0.csv\"]),\n", " (\"basic_biology\", [\"basic_biology_numsamples2000_start0.csv\"]),\n", " (\"pharmacology\", [\"pharmacology_numsamples2000_start0.csv\"]),\n", " (\"psychiatry\", [\"psychiatry_numsamples2000_start0.csv\"])\n", "]\n", "\n", "output_path = \"./artifacts/graph_triplets\"\n", "\n", "# minimum length threshold for strings to be considered valid\n", "k = 2 \n", "\n", "# Minimum occurence of entities\n", "min_entity_support = 2\n", "\n", "# Minimum occurence of relation types\n", "min_edge_support = 1 # Set to 1 as the number of relation types is small (73, see below)" ] }, { "cell_type": "code", "execution_count": 178, "id": "3a404b60", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import os" ] }, { "cell_type": "markdown", "id": "1071af95", "metadata": {}, "source": [ "Load and perform basic cleaning on data:" ] }, { "cell_type": "code", "execution_count": null, "id": "34e5445b", "metadata": {}, "outputs": [], "source": [ "# Read triplets from files and perform basic normalization\n", "\n", "CLEAN_REGEX = r\"[^a-z0-9\\s\\-\\.\\+]+\" # allow letters, numbers, space, -, ., +\n", "\n", "def normalize_text(col: pd.Series):\n", " return (\n", " col.astype(str)\n", " .str.replace('_', ' ', regex=False)\n", " .str.strip()\n", " .str.lower()\n", " # Remove unwanted chars but KEEP spaces\n", " .str.replace(CLEAN_REGEX, \" \", regex=True)\n", " # Normalize hyphens (keep them, but spaced)\n", " .str.replace(r\"\\s*-\\s*\", \"-\", regex=True)\n", " # Collapse multiple spaces\n", " .str.replace(r\"\\s+\", \" \", regex=True)\n", " # Remove leading/trailing punctuation\n", " .str.strip(\" .-+\")\n", " )\n", "\n", "clean_dfs = dict()\n", "\n", "# Read and process separated triplets file \n", "for ctg, files in triplet_files:\n", " ctg_df = pd.concat([\n", " pd.read_csv(\n", " os.path.join(filepath, filename),\n", " on_bad_lines='skip',\n", " encoding='utf-8',\n", " encoding_errors='ignore',\n", " names=['Subject', 'Predicate', 'Object']\n", " )\n", " for filename in files\n", " ]) \n", "\n", " # Normalize texts\n", " ctg_df = ctg_df.assign(\n", " Subject=lambda df: normalize_text(df['Subject']),\n", " Predicate=lambda df: normalize_text(df['Predicate']),\n", " Object=lambda df: normalize_text(df['Object'])\n", " ).replace(\"\", pd.NA)\\\n", " .dropna(subset=['Subject', 'Predicate', 'Object']) # Fill empty fields with NA and remove malformed rows\n", " \n", " # Remove rows whose fields do not satisfy minimum length threshold\n", " ctg_df = ctg_df.loc[lambda df: (\n", " df['Subject'].str.len().ge(k) &\n", " df['Predicate'].str.len().ge(k) &\n", " df['Object'].str.len().ge(k)\n", " )]\n", " \n", " # Drop obvious garbage rows and remove duplications\n", " ctg_df = ctg_df.loc[lambda df: (\n", " ~df['Subject'].str.match(r\"^(and|or|of|in|on|at)$\") &\n", " ~df['Object'].str.match(r\"^(and|or|of|in|on|at)$\")\n", " )]\\\n", " .drop_duplicates(subset=['Subject', 'Predicate', 'Object'])\\\n", " .reset_index(drop=True)\n", "\n", " # Remove self-loops (Subject == Object)\n", " ctg_df = ctg_df[ctg_df['Subject'] != ctg_df['Object']]\n", " \n", " # Remove rows containing \"figure\" in any column\n", " ctg_df = ctg_df[\n", " ~ctg_df['Subject'].str.contains('figure', case=False, na=False) &\n", " ~ctg_df['Predicate'].str.contains('figure', case=False, na=False) &\n", " ~ctg_df['Object'].str.contains('figure', case=False, na=False)\n", " ]\n", "\n", " clean_dfs[ctg] = ctg_df" ] }, { "cell_type": "code", "execution_count": null, "id": "4cc784d3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Category: core_clinical Number of triplets: 16563 Number of entites: 7875\n", "Category: basic_biology Number of triplets: 8668 Number of entites: 5805\n", "Category: pharmacology Number of triplets: 9555 Number of entites: 7412\n", "Category: psychiatry Number of triplets: 8762 Number of entites: 6548\n" ] } ], "source": [ "for ctg, df in clean_dfs.items():\n", " entity_count = pd.concat([df['Subject'], df['Object']]).unique().shape[0]\n", " print(f\"Category: {ctg} Number of triplets: {df.shape[0]} Number of entites: {entity_count}\")" ] }, { "cell_type": "markdown", "id": "5cdbef41", "metadata": {}, "source": [ "Concatenate DataFrames into a single unit and perform global de-duplication:" ] }, { "cell_type": "code", "execution_count": 181, "id": "1363290e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Entity count: (24647,)\n", "Relation count: (73,)\n" ] }, { "data": { "text/html": [ "
| \n", " | Subject | \n", "Predicate | \n", "Object | \n", "
|---|---|---|---|
| 0 | \n", "anatomy | \n", "part of | \n", "medicine | \n", "
| 1 | \n", "clinical sign | \n", "facet of | \n", "disease | \n", "
| 2 | \n", "anatomy | \n", "practice by | \n", "physician | \n", "
| 3 | \n", "disease | \n", "study by | \n", "anatomy | \n", "
| 4 | \n", "structure | \n", "study by | \n", "macroscopic anatomy | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "
| 43084 | \n", "sexual sadism disorder | \n", "instance of | \n", "comorbiditie | \n", "
| 43085 | \n", "sub stance use disorder | \n", "instance of | \n", "comorbid | \n", "
| 43086 | \n", "sexual interest in child | \n", "instance of | \n", "paraphilia | \n", "
| 43087 | \n", "pedophilic sexual orientation | \n", "different from | \n", "pedophilic disorder | \n", "
| 43088 | \n", "incest | \n", "subclass of | \n", "pedophilic disorder | \n", "
43089 rows × 3 columns
\n", "| \n", " | Subject | \n", "Predicate | \n", "Object | \n", "
|---|---|---|---|
| 0 | \n", "anatomy | \n", "part of | \n", "medicine | \n", "
| 1 | \n", "clinical sign | \n", "facet of | \n", "disease | \n", "
| 2 | \n", "anatomy | \n", "practice by | \n", "physician | \n", "
| 3 | \n", "disease | \n", "study by | \n", "anatomy | \n", "
| 4 | \n", "structure | \n", "study by | \n", "macroscopic anatomy | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "
| 43081 | \n", "f65.4 | \n", "have | \n", "diagnostic criterion 302.2 | \n", "
| 43082 | \n", "comorbid | \n", "subclass of | \n", "disorder | \n", "
| 43083 | \n", "diagnostic criterion 302.2 | \n", "part of | \n", "f65.4 | \n", "
| 43084 | \n", "sexual sadism disorder | \n", "instance of | \n", "comorbiditie | \n", "
| 43085 | \n", "sub stance use disorder | \n", "instance of | \n", "comorbid | \n", "
31033 rows × 3 columns
\n", "