Upload 2 files

environment.yml

@@ -0,0 +1,180 @@
+name: graph_rag
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - ca-certificates=2024.3.11=h06a4308_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - ncurses=6.4=h6a678d5_0
+  - openssl=3.0.13=h7f8727e_0
+  - pip=23.3.1=py39h06a4308_0
+  - python=3.9.19=h955ad1f_0
+  - readline=8.2=h5eee18b_0
+  - setuptools=68.2.2=py39h06a4308_0
+  - sqlite=3.41.2=h5eee18b_0
+  - tk=8.6.12=h1ccaba5_0
+  - wheel=0.41.2=py39h06a4308_0
+  - xz=5.4.6=h5eee18b_0
+  - zlib=1.2.13=h5eee18b_0
+  - pip:
+      - aiohttp==3.9.5
+      - aiosignal==1.3.1
+      - annotated-types==0.6.0
+      - anyio==4.3.0
+      - argon2-cffi==23.1.0
+      - argon2-cffi-bindings==21.2.0
+      - arrow==1.3.0
+      - asttokens==2.4.1
+      - async-lru==2.0.4
+      - async-timeout==4.0.3
+      - attrs==23.2.0
+      - babel==2.14.0
+      - beautifulsoup4==4.12.3
+      - bleach==6.1.0
+      - certifi==2024.2.2
+      - cffi==1.16.0
+      - charset-normalizer==3.3.2
+      - click==8.1.7
+      - comm==0.2.2
+      - dataclasses-json==0.6.4
+      - debugpy==1.8.1
+      - decorator==5.1.1
+      - defusedxml==0.7.1
+      - deprecated==1.2.14
+      - dirtyjson==1.0.8
+      - distro==1.9.0
+      - exceptiongroup==1.2.1
+      - executing==2.0.1
+      - fastjsonschema==2.19.1
+      - fqdn==1.5.1
+      - frozenlist==1.4.1
+      - fsspec==2024.3.1
+      - greenlet==3.0.3
+      - h11==0.14.0
+      - httpcore==1.0.5
+      - httpx==0.27.0
+      - idna==3.7
+      - importlib-metadata==7.1.0
+      - ipykernel==6.29.4
+      - ipython==8.18.1
+      - isoduration==20.11.0
+      - jedi==0.19.1
+      - jinja2==3.1.3
+      - joblib==1.4.0
+      - json5==0.9.25
+      - jsonpatch==1.33
+      - jsonpointer==2.4
+      - jsonschema==4.21.1
+      - jsonschema-specifications==2023.12.1
+      - jupyter-client==8.6.1
+      - jupyter-core==5.7.2
+      - jupyter-events==0.10.0
+      - jupyter-lsp==2.2.5
+      - jupyter-server==2.14.0
+      - jupyter-server-terminals==0.5.3
+      - jupyterlab==4.1.6
+      - jupyterlab-pygments==0.3.0
+      - jupyterlab-server==2.26.0
+      - langchain==0.1.16
+      - langchain-community==0.0.34
+      - langchain-core==0.1.45
+      - langchain-openai==0.1.3
+      - langchain-text-splitters==0.0.1
+      - langsmith==0.1.49
+      - llama-index==0.10.30
+      - llama-index-agent-openai==0.2.2
+      - llama-index-cli==0.1.12
+      - llama-index-core==0.10.30
+      - llama-index-embeddings-openai==0.1.8
+      - llama-index-indices-managed-llama-cloud==0.1.5
+      - llama-index-legacy==0.9.48
+      - llama-index-llms-openai==0.1.16
+      - llama-index-multi-modal-llms-openai==0.1.5
+      - llama-index-program-openai==0.1.5
+      - llama-index-question-gen-openai==0.1.3
+      - llama-index-readers-file==0.1.19
+      - llama-index-readers-llama-parse==0.1.4
+      - llama-parse==0.4.1
+      - llamaindex-py-client==0.1.18
+      - markupsafe==2.1.5
+      - marshmallow==3.21.1
+      - matplotlib-inline==0.1.7
+      - mistune==3.0.2
+      - multidict==6.0.5
+      - mypy-extensions==1.0.0
+      - nbclient==0.10.0
+      - nbconvert==7.16.3
+      - nbformat==5.10.4
+      - neo4j==5.19.0
+      - nest-asyncio==1.6.0
+      - networkx==3.2.1
+      - nltk==3.8.1
+      - notebook==7.1.3
+      - notebook-shim==0.2.4
+      - numpy==1.26.4
+      - openai==1.23.2
+      - orjson==3.10.1
+      - overrides==7.7.0
+      - packaging==23.2
+      - pandas==2.2.2
+      - pandocfilters==1.5.1
+      - parso==0.8.4
+      - pexpect==4.9.0
+      - pillow==10.3.0
+      - platformdirs==4.2.0
+      - prometheus-client==0.20.0
+      - prompt-toolkit==3.0.43
+      - psutil==5.9.8
+      - ptyprocess==0.7.0
+      - pure-eval==0.2.2
+      - pycparser==2.22
+      - pydantic==2.7.0
+      - pydantic-core==2.18.1
+      - pygments==2.17.2
+      - pypdf==4.2.0
+      - python-dateutil==2.9.0.post0
+      - python-json-logger==2.0.7
+      - pytz==2024.1
+      - pyyaml==6.0.1
+      - pyzmq==26.0.2
+      - referencing==0.34.0
+      - regex==2024.4.16
+      - requests==2.31.0
+      - rfc3339-validator==0.1.4
+      - rfc3986-validator==0.1.1
+      - rpds-py==0.18.0
+      - send2trash==1.8.3
+      - six==1.16.0
+      - sniffio==1.3.1
+      - soupsieve==2.5
+      - sqlalchemy==2.0.29
+      - stack-data==0.6.3
+      - striprtf==0.0.26
+      - tenacity==8.2.3
+      - terminado==0.18.1
+      - tiktoken==0.6.0
+      - tinycss2==1.2.1
+      - tomli==2.0.1
+      - tornado==6.4
+      - tqdm==4.66.2
+      - traitlets==5.14.3
+      - types-python-dateutil==2.9.0.20240316
+      - typing-extensions==4.11.0
+      - typing-inspect==0.9.0
+      - tzdata==2024.1
+      - uri-template==1.3.0
+      - urllib3==2.2.1
+      - wcwidth==0.2.13
+      - webcolors==1.13
+      - webencodings==0.5.1
+      - websocket-client==1.7.0
+      - wikipedia==1.4.0
+      - wrapt==1.16.0
+      - yarl==1.9.4
+      - zipp==3.18.1
+prefix: /local/home/pbhandari/miniconda3/envs/graph_rag

kg_creation.ipynb

@@ -0,0 +1,441 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "os.environ['OPENAI_API_KEY'] = \"sk-proj-k8uMlsAJbdAuSWWnvaHyT3BlbkFJyQB8yMQavFuQDVmc4sNs\"\n",
+    "\n",
+    "import logging\n",
+    "import sys\n",
+    "\n",
+    "logging.basicConfig(\n",
+    "    stream=sys.stdout, level=logging.INFO\n",
+    ")  # logging.DEBUG for more verbose output\n",
+    "\n",
+    "\n",
+    "# define LLM\n",
+    "from llama_index.llms.openai import OpenAI\n",
+    "from llama_index.core import Settings\n",
+    "\n",
+    "Settings.llm = OpenAI(temperature=0, model=\"gpt-3.5-turbo-0125\")\n",
+    "Settings.chunk_size = 512"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: langchain in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (0.1.16)\n",
+      "Requirement already satisfied: neo4j in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (5.19.0)\n",
+      "Requirement already satisfied: openai in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (1.23.2)\n",
+      "Requirement already satisfied: wikipedia in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (1.4.0)\n",
+      "Requirement already satisfied: tiktoken in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (0.6.0)\n",
+      "Requirement already satisfied: langchain_openai in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (0.1.3)\n",
+      "Requirement already satisfied: PyYAML>=5.3 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (6.0.1)\n",
+      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (2.0.29)\n",
+      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (3.9.5)\n",
+      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (4.0.3)\n",
+      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (0.6.4)\n",
+      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (1.33)\n",
+      "Requirement already satisfied: langchain-community<0.1,>=0.0.32 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (0.0.34)\n",
+      "Requirement already satisfied: langchain-core<0.2.0,>=0.1.42 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (0.1.45)\n",
+      "Requirement already satisfied: langchain-text-splitters<0.1,>=0.0.1 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (0.0.1)\n",
+      "Requirement already satisfied: langsmith<0.2.0,>=0.1.17 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (0.1.49)\n",
+      "Requirement already satisfied: numpy<2,>=1 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (1.26.4)\n",
+      "Requirement already satisfied: pydantic<3,>=1 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (2.7.0)\n",
+      "Requirement already satisfied: requests<3,>=2 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (2.31.0)\n",
+      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain) (8.2.3)\n",
+      "Requirement already satisfied: pytz in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from neo4j) (2024.1)\n",
+      "Requirement already satisfied: anyio<5,>=3.5.0 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from openai) (4.3.0)\n",
+      "Requirement already satisfied: distro<2,>=1.7.0 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from openai) (1.9.0)\n",
+      "Requirement already satisfied: httpx<1,>=0.23.0 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from openai) (0.27.0)\n",
+      "Requirement already satisfied: sniffio in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from openai) (1.3.1)\n",
+      "Requirement already satisfied: tqdm>4 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from openai) (4.66.2)\n",
+      "Requirement already satisfied: typing-extensions<5,>=4.7 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from openai) (4.11.0)\n",
+      "Requirement already satisfied: beautifulsoup4 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from wikipedia) (4.12.3)\n",
+      "Requirement already satisfied: regex>=2022.1.18 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from tiktoken) (2024.4.16)\n",
+      "Requirement already satisfied: aiosignal>=1.1.2 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
+      "Requirement already satisfied: attrs>=17.3.0 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.2.0)\n",
+      "Requirement already satisfied: frozenlist>=1.1.1 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.1)\n",
+      "Requirement already satisfied: multidict<7.0,>=4.5 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.5)\n",
+      "Requirement already satisfied: yarl<2.0,>=1.0 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.4)\n",
+      "Requirement already satisfied: idna>=2.8 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from anyio<5,>=3.5.0->openai) (3.7)\n",
+      "Requirement already satisfied: exceptiongroup>=1.0.2 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from anyio<5,>=3.5.0->openai) (1.2.1)\n",
+      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (3.21.1)\n",
+      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (0.9.0)\n",
+      "Requirement already satisfied: certifi in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from httpx<1,>=0.23.0->openai) (2024.2.2)\n",
+      "Requirement already satisfied: httpcore==1.* in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from httpx<1,>=0.23.0->openai) (1.0.5)\n",
+      "Requirement already satisfied: h11<0.15,>=0.13 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\n",
+      "Requirement already satisfied: jsonpointer>=1.9 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from jsonpatch<2.0,>=1.33->langchain) (2.4)\n",
+      "Requirement already satisfied: packaging<24.0,>=23.2 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langchain-core<0.2.0,>=0.1.42->langchain) (23.2)\n",
+      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from langsmith<0.2.0,>=0.1.17->langchain) (3.10.1)\n",
+      "Requirement already satisfied: annotated-types>=0.4.0 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from pydantic<3,>=1->langchain) (0.6.0)\n",
+      "Requirement already satisfied: pydantic-core==2.18.1 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from pydantic<3,>=1->langchain) (2.18.1)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from requests<3,>=2->langchain) (3.3.2)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from requests<3,>=2->langchain) (2.2.1)\n",
+      "Requirement already satisfied: greenlet!=0.4.17 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from SQLAlchemy<3,>=1.4->langchain) (3.0.3)\n",
+      "Requirement already satisfied: soupsieve>1.2 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from beautifulsoup4->wikipedia) (2.5)\n",
+      "Requirement already satisfied: mypy-extensions>=0.3.0 in /local/home/pbhandari/miniconda3/envs/graph_rag/lib/python3.9/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain) (1.0.0)\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install langchain neo4j openai wikipedia tiktoken langchain_openai"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.graphs import Neo4jGraph\n",
+    "\n",
+    "url = \"neo4j+s://2f409740.databases.neo4j.io\"\n",
+    "username =\"neo4j\"\n",
+    "password = \"oe7A9ugxhxcuEtwci8khPIt2TTdz_am9AYDx1r9e9Tw\"\n",
+    "graph = Neo4jGraph(\n",
+    "    url=url,\n",
+    "    username=username,\n",
+    "    password=password\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.graphs.graph_document import (\n",
+    "    Node as BaseNode,\n",
+    "    Relationship as BaseRelationship,\n",
+    "    GraphDocument,\n",
+    ")\n",
+    "from langchain.schema import Document\n",
+    "from typing import List, Dict, Any, Optional\n",
+    "from langchain.pydantic_v1 import Field, BaseModel\n",
+    "\n",
+    "class Property(BaseModel):\n",
+    "  \"\"\"A single property consisting of key and value\"\"\"\n",
+    "  key: str = Field(..., description=\"key\")\n",
+    "  value: str = Field(..., description=\"value\")\n",
+    "\n",
+    "class Node(BaseNode):\n",
+    "    properties: Optional[List[Property]] = Field(\n",
+    "        None, description=\"List of node properties\")\n",
+    "\n",
+    "class Relationship(BaseRelationship):\n",
+    "    properties: Optional[List[Property]] = Field(\n",
+    "        None, description=\"List of relationship properties\"\n",
+    "    )\n",
+    "\n",
+    "class KnowledgeGraph(BaseModel):\n",
+    "    \"\"\"Generate a knowledge graph with entities and relationships.\"\"\"\n",
+    "    nodes: List[Node] = Field(\n",
+    "        ..., description=\"List of nodes in the knowledge graph\")\n",
+    "    rels: List[Relationship] = Field(\n",
+    "        ..., description=\"List of relationships in the knowledge graph\"\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def format_property_key(s: str) -> str:\n",
+    "    words = s.split()\n",
+    "    if not words:\n",
+    "        return s\n",
+    "    first_word = words[0].lower()\n",
+    "    capitalized_words = [word.capitalize() for word in words[1:]]\n",
+    "    return \"\".join([first_word] + capitalized_words)\n",
+    "\n",
+    "def props_to_dict(props) -> dict:\n",
+    "    \"\"\"Convert properties to a dictionary.\"\"\"\n",
+    "    properties = {}\n",
+    "    if not props:\n",
+    "      return properties\n",
+    "    for p in props:\n",
+    "        properties[format_property_key(p.key)] = p.value\n",
+    "    return properties\n",
+    "\n",
+    "def map_to_base_node(node: Node) -> BaseNode:\n",
+    "    \"\"\"Map the KnowledgeGraph Node to the base Node.\"\"\"\n",
+    "    properties = props_to_dict(node.properties) if node.properties else {}\n",
+    "    # Add name property for better Cypher statement generation\n",
+    "    properties[\"name\"] = node.id.title()\n",
+    "    return BaseNode(\n",
+    "        id=node.id.title(), type=node.type.capitalize(), properties=properties\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "def map_to_base_relationship(rel: Relationship) -> BaseRelationship:\n",
+    "    \"\"\"Map the KnowledgeGraph Relationship to the base Relationship.\"\"\"\n",
+    "    source = map_to_base_node(rel.source)\n",
+    "    target = map_to_base_node(rel.target)\n",
+    "    properties = props_to_dict(rel.properties) if rel.properties else {}\n",
+    "    return BaseRelationship(\n",
+    "        source=source, target=target, type=rel.type, properties=properties\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from langchain.chains.openai_functions import (\n",
+    "    create_openai_fn_chain,\n",
+    "    create_structured_output_chain,\n",
+    ")\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "from langchain.prompts import ChatPromptTemplate\n",
+    "\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"sk-proj-k8uMlsAJbdAuSWWnvaHyT3BlbkFJyQB8yMQavFuQDVmc4sNs\"\n",
+    "llm = ChatOpenAI(model=\"gpt-3.5-turbo-16k\", temperature=0)\n",
+    "\n",
+    "def get_extraction_chain(\n",
+    "    allowed_nodes: Optional[List[str]] = None,\n",
+    "    allowed_rels: Optional[List[str]] = None\n",
+    "    ):\n",
+    "    prompt = ChatPromptTemplate.from_messages(\n",
+    "        [(\n",
+    "          \"system\",\n",
+    "          f\"\"\"# Knowledge Graph Instructions for GPT-4\n",
+    "## 1. Overview\n",
+    "You are a top-tier algorithm designed for extracting information in structured formats to build a knowledge graph.\n",
+    "- **Nodes** represent entities and concepts. They're akin to Wikipedia nodes.\n",
+    "- The aim is to achieve simplicity and clarity in the knowledge graph, making it accessible for a vast audience.\n",
+    "## 2. Labeling Nodes\n",
+    "- **Consistency**: Ensure you use basic or elementary types for node labels.\n",
+    "  - For example, when you identify an entity representing a person, always label it as **\"person\"**. Avoid using more specific terms like \"mathematician\" or \"scientist\".\n",
+    "- **Node IDs**: Never utilize integers as node IDs. Node IDs should be names or human-readable identifiers found in the text.\n",
+    "{'- **Allowed Node Labels:**' + \", \".join(allowed_nodes) if allowed_nodes else \"\"}\n",
+    "{'- **Allowed Relationship Types**:' + \", \".join(allowed_rels) if allowed_rels else \"\"}\n",
+    "## 3. Handling Numerical Data and Dates\n",
+    "- Numerical data, like age or other related information, should be incorporated as attributes or properties of the respective nodes.\n",
+    "- **No Separate Nodes for Dates/Numbers**: Do not create separate nodes for dates or numerical values. Always attach them as attributes or properties of nodes.\n",
+    "- **Property Format**: Properties must be in a key-value format.\n",
+    "- **Quotation Marks**: Never use escaped single or double quotes within property values.\n",
+    "- **Naming Convention**: Use camelCase for property keys, e.g., `birthDate`.\n",
+    "## 4. Coreference Resolution\n",
+    "- **Maintain Entity Consistency**: When extracting entities, it's vital to ensure consistency.\n",
+    "If an entity, such as \"John Doe\", is mentioned multiple times in the text but is referred to by different names or pronouns (e.g., \"Joe\", \"he\"),\n",
+    "always use the most complete identifier for that entity throughout the knowledge graph. In this example, use \"John Doe\" as the entity ID.\n",
+    "Remember, the knowledge graph should be coherent and easily understandable, so maintaining consistency in entity references is crucial.\n",
+    "## 5. Strict Compliance\n",
+    "Adhere to the rules strictly. Non-compliance will result in termination.\n",
+    "          \"\"\"),\n",
+    "            (\"human\", \"Use the given format to extract information from the following input: {input}\"),\n",
+    "            (\"human\", \"Tip: Make sure to answer in the correct format\"),\n",
+    "        ])\n",
+    "    return create_structured_output_chain(KnowledgeGraph, llm, prompt, verbose=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def extract_and_store_graph(\n",
+    "    document: Document,\n",
+    "    nodes:Optional[List[str]] = None,\n",
+    "    rels:Optional[List[str]]=None) -> None:\n",
+    "    # Extract graph data using OpenAI functions\n",
+    "    extract_chain = get_extraction_chain(nodes, rels)\n",
+    "    data = extract_chain.invoke(document.page_content)['function']\n",
+    "    # Construct a graph document\n",
+    "    graph_document = GraphDocument(\n",
+    "      nodes = [map_to_base_node(node) for node in data.nodes],\n",
+    "      relationships = [map_to_base_relationship(rel) for rel in data.rels],\n",
+    "      source = document\n",
+    "    )\n",
+    "    # Store information into a graph\n",
+    "    graph.add_graph_documents([graph_document])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import WikipediaLoader\n",
+    "from langchain.text_splitter import TokenTextSplitter\n",
+    "\n",
+    "# Read the wikipedia article\n",
+    "raw_documents = WikipediaLoader(query=\"Chemotherapy\").load()\n",
+    "# Define chunking strategy\n",
+    "text_splitter = TokenTextSplitter(chunk_size=2048, chunk_overlap=24)\n",
+    "\n",
+    "# Only take the first the raw_documents\n",
+    "documents = text_splitter.split_documents(raw_documents[:3])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  0%|          | 0/3 [00:00<?, ?it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 429 Too Many Requests\"\n",
+      "INFO:openai._base_client:Retrying request to /chat/completions in 0.903231 seconds\n",
+      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 33%|███▎      | 1/3 [00:44<01:29, 44.80s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 67%|██████▋   | 2/3 [00:59<00:27, 27.11s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 3/3 [01:07<00:00, 22.36s/it]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from tqdm import tqdm\n",
+    "\n",
+    "for i, d in tqdm(enumerate(documents), total=len(documents)):\n",
+    "    extract_and_store_graph(d)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Query the knowledge graph in a RAG application\n",
+    "from langchain.chains import GraphCypherQAChain\n",
+    "\n",
+    "graph.refresh_schema()\n",
+    "\n",
+    "cypher_chain = GraphCypherQAChain.from_llm(\n",
+    "    graph=graph,\n",
+    "    cypher_llm=ChatOpenAI(temperature=0, model=\"gpt-4\"),\n",
+    "    qa_llm=ChatOpenAI(temperature=0, model=\"gpt-3.5-turbo-16k\"),\n",
+    "    validate_cypher=True, # Validate relationship directions\n",
+    "    verbose=True\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new GraphCypherQAChain chain...\u001b[0m\n",
+      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "Generated Cypher:\n",
+      "\u001b[32;1m\u001b[1;3mMATCH (c:Concept {name: \"hormonal therapies\"})-[:ARE_NOW_CALLED]->(newName:Concept) RETURN newName.name\u001b[0m\n",
+      "Full Context:\n",
+      "\u001b[32;1m\u001b[1;3m[]\u001b[0m\n",
+      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "\n",
+      "\u001b[1m> Finished chain.\u001b[0m\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'query': 'What are hormonal therapies now called?',\n",
+       " 'result': \"I'm sorry, but I don't have the information to answer your question.\"}"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "cypher_chain.invoke({\"query\": \"What are hormonal therapies now called?\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "my_project_env",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}