{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "IJo4BWsb1Nq2"
      },
      "outputs": [],
      "source": [
        "from google.colab import files\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "uploaded = files.upload()\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "id": "I7QzXzU4167E",
        "outputId": "eaa7a228-040a-4ca3-91de-c26a38d1f758"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-bbe130df-ab5a-434d-b245-6694cab53bb1\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-bbe130df-ab5a-434d-b245-6694cab53bb1\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving volve_well_daily.csv to volve_well_daily.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "uploaded = files.upload()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "id": "V1KOZdBT8wO8",
        "outputId": "9c69ea6b-dd0b-40db-ad33-207f6baf893a"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-5abb33f7-0fa4-489f-b550-1571ab444f83\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-5abb33f7-0fa4-489f-b550-1571ab444f83\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving volve_grounded_events.csv to volve_grounded_events.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "metadata": {
        "id": "OjEVqvlM2wqu"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.preprocessing import StandardScaler"
      ],
      "metadata": {
        "id": "akJ-BhXw5XjV"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# All telemetry comes from ONE source: the \"Daily Production Data\" sheet of the Equinor Volve production export. Provenance stated in paper.\n",
        "baseline_data = pd.read_csv('volve_well_daily.csv', parse_dates=['date'])\n",
        "\n",
        "print(\"Loaded:\", baseline_data.shape)\n",
        "baseline_data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 417
        },
        "id": "Im2YvfD420S6",
        "outputId": "db8a49e6-5cb4-4f59-f95f-cdd31a53fed6"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loaded: (15634, 17)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "        date        well well_type   flow_kind  on_stream_hrs  dh_pressure  \\\n",
              "0 2014-04-07  15/9-F-1 C        WI  production            0.0      0.00000   \n",
              "1 2014-04-08  15/9-F-1 C        OP  production            0.0          NaN   \n",
              "2 2014-04-09  15/9-F-1 C        OP  production            0.0          NaN   \n",
              "3 2014-04-10  15/9-F-1 C        OP  production            0.0          NaN   \n",
              "4 2014-04-11  15/9-F-1 C        OP  production            0.0    310.37614   \n",
              "\n",
              "    dh_temp  dp_tubing  annulus_press       whp       wht  choke_pct  oil_vol  \\\n",
              "0   0.00000    0.00000            0.0   0.00000   0.00000   0.000000      0.0   \n",
              "1       NaN        NaN            0.0   0.00000   0.00000   1.003059      0.0   \n",
              "2       NaN        NaN            0.0   0.00000   0.00000   0.979008      0.0   \n",
              "3       NaN        NaN            0.0   0.00000   0.00000   0.545759      0.0   \n",
              "4  96.87589  277.27826            0.0  33.09788  10.47992   1.215987      0.0   \n",
              "\n",
              "   gas_vol  water_vol  wi_vol  water_cut  \n",
              "0      0.0        0.0     NaN        NaN  \n",
              "1      0.0        0.0     NaN        NaN  \n",
              "2      0.0        0.0     NaN        NaN  \n",
              "3      0.0        0.0     NaN        NaN  \n",
              "4      0.0        0.0     NaN        NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6bfa3e14-99c0-403d-96d4-70df53370633\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>well</th>\n",
              "      <th>well_type</th>\n",
              "      <th>flow_kind</th>\n",
              "      <th>on_stream_hrs</th>\n",
              "      <th>dh_pressure</th>\n",
              "      <th>dh_temp</th>\n",
              "      <th>dp_tubing</th>\n",
              "      <th>annulus_press</th>\n",
              "      <th>whp</th>\n",
              "      <th>wht</th>\n",
              "      <th>choke_pct</th>\n",
              "      <th>oil_vol</th>\n",
              "      <th>gas_vol</th>\n",
              "      <th>water_vol</th>\n",
              "      <th>wi_vol</th>\n",
              "      <th>water_cut</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2014-04-07</td>\n",
              "      <td>15/9-F-1 C</td>\n",
              "      <td>WI</td>\n",
              "      <td>production</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2014-04-08</td>\n",
              "      <td>15/9-F-1 C</td>\n",
              "      <td>OP</td>\n",
              "      <td>production</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>1.003059</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2014-04-09</td>\n",
              "      <td>15/9-F-1 C</td>\n",
              "      <td>OP</td>\n",
              "      <td>production</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.979008</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2014-04-10</td>\n",
              "      <td>15/9-F-1 C</td>\n",
              "      <td>OP</td>\n",
              "      <td>production</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.545759</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2014-04-11</td>\n",
              "      <td>15/9-F-1 C</td>\n",
              "      <td>OP</td>\n",
              "      <td>production</td>\n",
              "      <td>0.0</td>\n",
              "      <td>310.37614</td>\n",
              "      <td>96.87589</td>\n",
              "      <td>277.27826</td>\n",
              "      <td>0.0</td>\n",
              "      <td>33.09788</td>\n",
              "      <td>10.47992</td>\n",
              "      <td>1.215987</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6bfa3e14-99c0-403d-96d4-70df53370633')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-6bfa3e14-99c0-403d-96d4-70df53370633 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-6bfa3e14-99c0-403d-96d4-70df53370633');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "baseline_data",
              "summary": "{\n  \"name\": \"baseline_data\",\n  \"rows\": 15634,\n  \"fields\": [\n    {\n      \"column\": \"date\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": \"2007-09-01 00:00:00\",\n        \"max\": \"2016-12-01 00:00:00\",\n        \"num_unique_values\": 3327,\n        \"samples\": [\n          \"2015-06-19 00:00:00\",\n          \"2013-01-10 00:00:00\",\n          \"2011-06-27 00:00:00\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"well\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 7,\n        \"samples\": [\n          \"15/9-F-1 C\",\n          \"15/9-F-11\",\n          \"15/9-F-4\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"well_type\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"OP\",\n          \"WI\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"flow_kind\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"injection\",\n          \"production\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"on_stream_hrs\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 8.369978226621157,\n        \"min\": 0.0,\n        \"max\": 25.0,\n        \"num_unique_values\": 925,\n        \"samples\": [\n          15.97685,\n          7.50834\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dh_pressure\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 109.712362966277,\n        \"min\": 0.0,\n        \"max\": 397.58855,\n        \"num_unique_values\": 6567,\n        \"samples\": [\n          257.39503729166665,\n          266.45499916666665\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dh_temp\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 45.65794750924677,\n        \"min\": 0.0,\n        \"max\": 108.50217791666668,\n        \"num_unique_values\": 6461,\n        \"samples\": [\n          107.34037770833334,\n          99.86840166666668\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dp_tubing\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 76.7523727642968,\n        \"min\": 0.0,\n        \"max\": 345.90677,\n        \"num_unique_values\": 8684,\n        \"samples\": [\n          192.84992518641124,\n          183.48821666666663\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"annulus_press\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 8.406822005218148,\n        \"min\": 0.0,\n        \"max\": 30.019828125,\n        \"num_unique_values\": 6644,\n        \"samples\": [\n          23.920048541666667,\n          22.007243541666668\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"whp\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 24.75263111913396,\n        \"min\": 0.0,\n        \"max\": 137.31103,\n        \"num_unique_values\": 8829,\n        \"samples\": [\n          65.39948625,\n          49.58651276595744\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"wht\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 27.719027718942336,\n        \"min\": 0.0,\n        \"max\": 93.50958395833334,\n        \"num_unique_values\": 8793,\n        \"samples\": [\n          70.84700666666667,\n          73.61219854166667\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"choke_pct\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 36.692924152913925,\n        \"min\": 0.0,\n        \"max\": 100.0,\n        \"num_unique_values\": 6419,\n        \"samples\": [\n          0.2627485416666666,\n          26.97214645833333\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"oil_vol\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1323.538150992862,\n        \"min\": 0.0,\n        \"max\": 5901.84,\n        \"num_unique_values\": 7818,\n        \"samples\": [\n          1299.1,\n          183.08\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"gas_vol\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 188136.4104343701,\n        \"min\": 0.0,\n        \"max\": 851131.52,\n        \"num_unique_values\": 8005,\n        \"samples\": [\n          16789.85,\n          85785.41\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"water_vol\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1706.9828532909603,\n        \"min\": -457.84,\n        \"max\": 8019.74,\n        \"num_unique_values\": 7361,\n        \"samples\": [\n          4893.56,\n          192.39\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"wi_vol\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2181.486695244425,\n        \"min\": 0.0,\n        \"max\": 10013.6,\n        \"num_unique_values\": 5258,\n        \"samples\": [\n          8748.599670208334,\n          4186.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"water_cut\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.34524541295613415,\n        \"min\": -2.61443581544084,\n        \"max\": 1.0,\n        \"num_unique_values\": 7635,\n        \"samples\": [\n          0.1482174225345101,\n          0.9127332456978396\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Condition-monitoring target = oil producers (gas-lift wells).\n",
        "# Injectors behave differently and are excluded (choice stated in paper).\n",
        "op = baseline_data[baseline_data.well_type == 'OP'].copy()\n",
        "\n",
        "features = ['dh_pressure','dh_temp','dp_tubing','annulus_press',\n",
        "            'whp','wht','choke_pct','oil_vol','gas_vol','water_vol']\n",
        "\n",
        "# In this export a 0 in a pressure channel means \"no reading\", not a true zero.\n",
        "# We treat zeros as missing before training. (Stated in paper.)\n",
        "op[features] = op[features].replace(0, np.nan)\n",
        "\n",
        "print(\"Oil-producer rows:\", len(op))\n",
        "print(\"Wells:\", sorted(op.well.unique()))\n",
        "op[features].isna().mean().round(2)   # shows missing fraction per channel"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 432
        },
        "id": "ceUOirXr3ZSR",
        "outputId": "b014d91b-86ac-44c5-a02a-89267e049117"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Oil-producer rows: 9143\n",
            "Wells: ['15/9-F-1 C', '15/9-F-11', '15/9-F-12', '15/9-F-14', '15/9-F-15 D', '15/9-F-5']\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dh_pressure      0.27\n",
              "dh_temp          0.27\n",
              "dp_tubing        0.04\n",
              "annulus_press    0.27\n",
              "whp              0.03\n",
              "wht              0.03\n",
              "choke_pct        0.05\n",
              "oil_vol          0.13\n",
              "gas_vol          0.12\n",
              "water_vol        0.17\n",
              "dtype: float64"
            ],
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>dh_pressure</th>\n",
              "      <td>0.27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>dh_temp</th>\n",
              "      <td>0.27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>dp_tubing</th>\n",
              "      <td>0.04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>annulus_press</th>\n",
              "      <td>0.27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>whp</th>\n",
              "      <td>0.03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>wht</th>\n",
              "      <td>0.03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>choke_pct</th>\n",
              "      <td>0.05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>oil_vol</th>\n",
              "      <td>0.13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gas_vol</th>\n",
              "      <td>0.12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>water_vol</th>\n",
              "      <td>0.17</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div><br><label><b>dtype:</b> float64</label>"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Sort chronologically within each well so time-based operations are valid.\n",
        "op = op.sort_values(['well','date']).reset_index(drop=True)\n",
        "\n",
        "'''\n",
        "Fill gaps of at most 3 days, per well, carrying the last reading forward. Reason: a 1-2 day gap is likely a brief telemetry dropout where the well's true state barely changed. Longer gaps mean we genuinely don't know, so we\n",
        "leave them empty rather than invent values. (Assumption stated in paper.)\n",
        "'''\n",
        "op[features] = op.groupby('well')[features].ffill(limit=3)\n",
        "\n",
        "print(\"After 3-day fill, missing fraction per channel:\")\n",
        "op[features].isna().mean().round(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 415
        },
        "id": "5j-K-eA_3x2k",
        "outputId": "174641f1-cb1c-4424-86b8-72082957a6a1"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "After 3-day fill, missing fraction per channel:\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dh_pressure      0.27\n",
              "dh_temp          0.27\n",
              "dp_tubing        0.04\n",
              "annulus_press    0.26\n",
              "whp              0.03\n",
              "wht              0.03\n",
              "choke_pct        0.04\n",
              "oil_vol          0.10\n",
              "gas_vol          0.10\n",
              "water_vol        0.14\n",
              "dtype: float64"
            ],
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>dh_pressure</th>\n",
              "      <td>0.27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>dh_temp</th>\n",
              "      <td>0.27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>dp_tubing</th>\n",
              "      <td>0.04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>annulus_press</th>\n",
              "      <td>0.26</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>whp</th>\n",
              "      <td>0.03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>wht</th>\n",
              "      <td>0.03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>choke_pct</th>\n",
              "      <td>0.04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>oil_vol</th>\n",
              "      <td>0.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gas_vol</th>\n",
              "      <td>0.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>water_vol</th>\n",
              "      <td>0.14</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div><br><label><b>dtype:</b> float64</label>"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "For each sensor, add its day-over-day change. Reason: anomalies often show as sudden movement (a jump) more than as an unusual absolute level.\n",
        "(Assumption stated in paper.)\n",
        "'''\n",
        "for c in features:\n",
        "    op[c + '_delta'] = op.groupby('well')[c].diff()\n",
        "\n",
        "delta_features = [c + '_delta' for c in features]\n",
        "all_features = features + delta_features\n",
        "\n",
        "print(\"Total features:\", len(all_features), \"(10 sensor values + 10 daily changes)\")\n",
        "op[all_features].head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 243
        },
        "id": "Jtx0O43v38ii",
        "outputId": "b9dc1d3a-27e8-4bd9-9b16-28843426f27c"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total features: 20 (10 sensor values + 10 daily changes)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   dh_pressure   dh_temp  dp_tubing  annulus_press       whp       wht  \\\n",
              "0          NaN       NaN        NaN            NaN       NaN       NaN   \n",
              "1          NaN       NaN        NaN            NaN       NaN       NaN   \n",
              "2          NaN       NaN        NaN            NaN       NaN       NaN   \n",
              "3    310.37614  96.87589  277.27826            NaN  33.09788  10.47992   \n",
              "4    303.50078  96.92339  281.44744            NaN  22.05334   8.70429   \n",
              "\n",
              "   choke_pct  oil_vol  gas_vol  water_vol  dh_pressure_delta  dh_temp_delta  \\\n",
              "0   1.003059      NaN      NaN        NaN                NaN            NaN   \n",
              "1   0.979008      NaN      NaN        NaN                NaN            NaN   \n",
              "2   0.545759      NaN      NaN        NaN                NaN            NaN   \n",
              "3   1.215987      NaN      NaN        NaN                NaN            NaN   \n",
              "4   3.087015      NaN      NaN        NaN           -6.87536         0.0475   \n",
              "\n",
              "   dp_tubing_delta  annulus_press_delta  whp_delta  wht_delta  \\\n",
              "0              NaN                  NaN        NaN        NaN   \n",
              "1              NaN                  NaN        NaN        NaN   \n",
              "2              NaN                  NaN        NaN        NaN   \n",
              "3              NaN                  NaN        NaN        NaN   \n",
              "4          4.16918                  NaN  -11.04454   -1.77563   \n",
              "\n",
              "   choke_pct_delta  oil_vol_delta  gas_vol_delta  water_vol_delta  \n",
              "0              NaN            NaN            NaN              NaN  \n",
              "1        -0.024051            NaN            NaN              NaN  \n",
              "2        -0.433249            NaN            NaN              NaN  \n",
              "3         0.670228            NaN            NaN              NaN  \n",
              "4         1.871029            NaN            NaN              NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-9a043075-80c2-4bf6-afde-47a1a9763521\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>dh_pressure</th>\n",
              "      <th>dh_temp</th>\n",
              "      <th>dp_tubing</th>\n",
              "      <th>annulus_press</th>\n",
              "      <th>whp</th>\n",
              "      <th>wht</th>\n",
              "      <th>choke_pct</th>\n",
              "      <th>oil_vol</th>\n",
              "      <th>gas_vol</th>\n",
              "      <th>water_vol</th>\n",
              "      <th>dh_pressure_delta</th>\n",
              "      <th>dh_temp_delta</th>\n",
              "      <th>dp_tubing_delta</th>\n",
              "      <th>annulus_press_delta</th>\n",
              "      <th>whp_delta</th>\n",
              "      <th>wht_delta</th>\n",
              "      <th>choke_pct_delta</th>\n",
              "      <th>oil_vol_delta</th>\n",
              "      <th>gas_vol_delta</th>\n",
              "      <th>water_vol_delta</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.003059</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.979008</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>-0.024051</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.545759</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>-0.433249</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>310.37614</td>\n",
              "      <td>96.87589</td>\n",
              "      <td>277.27826</td>\n",
              "      <td>NaN</td>\n",
              "      <td>33.09788</td>\n",
              "      <td>10.47992</td>\n",
              "      <td>1.215987</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.670228</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>303.50078</td>\n",
              "      <td>96.92339</td>\n",
              "      <td>281.44744</td>\n",
              "      <td>NaN</td>\n",
              "      <td>22.05334</td>\n",
              "      <td>8.70429</td>\n",
              "      <td>3.087015</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>-6.87536</td>\n",
              "      <td>0.0475</td>\n",
              "      <td>4.16918</td>\n",
              "      <td>NaN</td>\n",
              "      <td>-11.04454</td>\n",
              "      <td>-1.77563</td>\n",
              "      <td>1.871029</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9a043075-80c2-4bf6-afde-47a1a9763521')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-9a043075-80c2-4bf6-afde-47a1a9763521 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-9a043075-80c2-4bf6-afde-47a1a9763521');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"op[all_features]\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"dh_pressure\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 4.861613679098742,\n        \"min\": 303.50078,\n        \"max\": 310.37614,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          303.50078,\n          310.37614\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dh_temp\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.033587572106360604,\n        \"min\": 96.87589,\n        \"max\": 96.92339,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          96.92339,\n          96.87589\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dp_tubing\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2.948055449987318,\n        \"min\": 277.27826,\n        \"max\": 281.44744,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          281.44744,\n          277.27826\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"annulus_press\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": null,\n        \"max\": null,\n        \"num_unique_values\": 0,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"whp\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 7.809669129086075,\n        \"min\": 22.05334,\n        \"max\": 33.09788,\n        \"num_unique_values\": 2,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"wht\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.255560013878269,\n        \"min\": 8.70429,\n        \"max\": 10.47992,\n        \"num_unique_values\": 2,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"choke_pct\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.992305644928373,\n        \"min\": 0.5457591666666667,\n        \"max\": 3.0870154166666666,\n        \"num_unique_values\": 5,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"oil_vol\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": null,\n        \"max\": null,\n        \"num_unique_values\": 0,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"gas_vol\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": null,\n        \"max\": null,\n        \"num_unique_values\": 0,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"water_vol\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": null,\n        \"max\": null,\n        \"num_unique_values\": 0,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dh_pressure_delta\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": -6.875360000000001,\n        \"max\": -6.875360000000001,\n        \"num_unique_values\": 1,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dh_temp_delta\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": 0.04749999999999943,\n        \"max\": 0.04749999999999943,\n        \"num_unique_values\": 1,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dp_tubing_delta\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": 4.169179999999983,\n        \"max\": 4.169179999999983,\n        \"num_unique_values\": 1,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"annulus_press_delta\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": null,\n        \"max\": null,\n        \"num_unique_values\": 0,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"whp_delta\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": -11.044540000000005,\n        \"max\": -11.044540000000005,\n        \"num_unique_values\": 1,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"wht_delta\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": -1.7756299999999996,\n        \"max\": -1.7756299999999996,\n        \"num_unique_values\": 1,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"choke_pct_delta\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.0087149156682447,\n        \"min\": -0.4332485416666665,\n        \"max\": 1.8710285416666665,\n        \"num_unique_values\": 4,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"oil_vol_delta\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": null,\n        \"max\": null,\n        \"num_unique_values\": 0,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"gas_vol_delta\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": null,\n        \"max\": null,\n        \"num_unique_values\": 0,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"water_vol_delta\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": null,\n        \"max\": null,\n        \"num_unique_values\": 0,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "We train one  model per well (each well has its own normal behaviour, baselines, and pressure levels — pooling them would blur real differences).\n",
        "For each well we keep only rows where all 20 features are present, because the model cannot score a row with missing inputs.\n",
        "Rows dropped here are the ones with unfilled gaps (Reason: see zero=missing + 3-day-fill choices).\n",
        "We standardize (mean 0, variance 1) so no single channel dominates by scale.\n",
        "'''\n",
        "def prepare_well(g):\n",
        "    g = g.sort_values('date').copy()\n",
        "    X = g[all_features]\n",
        "    complete = X.notna().all(axis=1)         # keep only fully-observed days\n",
        "    return g[complete], X[complete]\n",
        "\n",
        "for well, g in op.groupby('well'):\n",
        "    gw, Xw = prepare_well(g)\n",
        "    print(f\"{well:14s}: {len(gw):4d} usable days out of {len(g)}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C5XhCh-C4gqI",
        "outputId": "2be8ee9c-ccfb-48a8-fedd-20ea4df1ffb7"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "15/9-F-1 C    :    0 usable days out of 744\n",
            "15/9-F-11     : 1113 usable days out of 1165\n",
            "15/9-F-12     :  859 usable days out of 3056\n",
            "15/9-F-14     : 1390 usable days out of 3056\n",
            "15/9-F-15 D   :  546 usable days out of 978\n",
            "15/9-F-5      :    0 usable days out of 144\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "At this point, feature engineering would help much. But since we're gearing towards unsupervised learning, there is no baseline to measure the usability of features against. But, we can ask the questions of \"Which features are actually available?\", \"Which features carry information vs. which are redundant?\", and \"Which features does each event type actually depend on?\""
      ],
      "metadata": {
        "id": "WQWCB-Dz6ZYG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Question: which channels are available, and which go missing TOGETHER?\n",
        "# Co-missingness matters: if the downhole channels are always missing on the same days, requiring all of them is what destroys usable days counts.\n",
        "\n",
        "# per-channel coverage on oil producers\n",
        "coverage = op[features].notna().mean().sort_values()\n",
        "print(\"Coverage per channel (fraction of days present):\")\n",
        "print(coverage.round(2).to_string())\n",
        "\n",
        "# co-missingness: correlation of \"is-missing\" flags between channels\n",
        "missing = op[features].isna().astype(int)\n",
        "print(\"\\nCo-missingness (correlation of missing-flags; 1.0 = always missing together):\")\n",
        "print(missing.corr().round(2).to_string())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SMSL_RD460Wv",
        "outputId": "d4bfe875-f1f8-4809-a1c3-d230adc94632"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Coverage per channel (fraction of days present):\n",
            "dh_pressure      0.73\n",
            "dh_temp          0.73\n",
            "annulus_press    0.74\n",
            "water_vol        0.86\n",
            "oil_vol          0.90\n",
            "gas_vol          0.90\n",
            "dp_tubing        0.96\n",
            "choke_pct        0.96\n",
            "wht              0.97\n",
            "whp              0.97\n",
            "\n",
            "Co-missingness (correlation of missing-flags; 1.0 = always missing together):\n",
            "               dh_pressure  dh_temp  dp_tubing  annulus_press   whp   wht  choke_pct  oil_vol  gas_vol  water_vol\n",
            "dh_pressure           1.00     1.00       0.32          -0.29  0.17  0.18       0.08     0.08     0.08      -0.00\n",
            "dh_temp               1.00     1.00       0.32          -0.29  0.17  0.18       0.08     0.08     0.08      -0.00\n",
            "dp_tubing             0.32     0.32       1.00           0.03  0.62  0.50       0.42     0.33     0.33       0.27\n",
            "annulus_press        -0.29    -0.29       0.03           1.00  0.17  0.15       0.21     0.23     0.23       0.15\n",
            "whp                   0.17     0.17       0.62           0.17  1.00  0.83       0.51     0.52     0.52       0.43\n",
            "wht                   0.18     0.18       0.50           0.15  0.83  1.00       0.38     0.55     0.55       0.45\n",
            "choke_pct             0.08     0.08       0.42           0.21  0.51  0.38       1.00     0.59     0.59       0.48\n",
            "oil_vol               0.08     0.08       0.33           0.23  0.52  0.55       0.59     1.00     1.00       0.82\n",
            "gas_vol               0.08     0.08       0.33           0.23  0.52  0.55       0.59     1.00     1.00       0.82\n",
            "water_vol            -0.00    -0.00       0.27           0.15  0.43  0.45       0.48     0.82     0.82       1.00\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The next pertinent questions is: which channels move during each event type?"
      ],
      "metadata": {
        "id": "p-9DQtwU781b"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's load the grounded events and mark each producer-day with its event type (if any).\n",
        "ev = pd.read_csv('volve_grounded_events.csv', parse_dates=['onset','offset'])\n",
        "\n",
        "op['event_type'] = 'none'\n",
        "for _, e in ev.iterrows():\n",
        "    m = (op.well == e.well) & (op.date >= e.onset) & (op.date <= e.offset)\n",
        "    op.loc[m, 'event_type'] = e.event_type\n",
        "\n",
        "'''\n",
        "For each event type, we measure how far each channel's DELTA deviates from its normal-day behaviour.\n",
        "We use the delta (day-over-day change) because events show up as movement. Metric: standardized mean absolute delta during the event type,\n",
        "relative to normal days. Higher = that channel \"reacts\" to that event.\n",
        "'''\n",
        "normal = op[op.event_type == 'none']\n",
        "norm_scale = op[delta_features].abs().mean()   # scale reference\n",
        "\n",
        "print(\"How strongly each channel reacts to each event type\")\n",
        "print(\"(values = mean |delta| during event / mean |delta| overall; >1 means reacts)\\n\")\n",
        "\n",
        "rows = []\n",
        "for et in ['shut_in','restart_transient','water_breakthrough',\n",
        "           'productivity_loss','gaslift_instability']:\n",
        "    sub = op[op.event_type == et]\n",
        "    if len(sub) == 0:\n",
        "        continue\n",
        "    ratio = (sub[delta_features].abs().mean() / norm_scale).round(2)\n",
        "    ratio.index = [c.replace('_delta','') for c in ratio.index]\n",
        "    rows.append(ratio.rename(et))\n",
        "\n",
        "react = pd.DataFrame(rows)\n",
        "react"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 258
        },
        "id": "7LdtVlvh78KO",
        "outputId": "00a9cb43-a227-49dc-f876-96ee17618a88"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "How strongly each channel reacts to each event type\n",
            "(values = mean |delta| during event / mean |delta| overall; >1 means reacts)\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                     dh_pressure  dh_temp  dp_tubing  annulus_press    whp  \\\n",
              "shut_in                     2.37     2.90       3.58           1.48   3.70   \n",
              "restart_transient           7.02     7.85       8.22           7.87  10.38   \n",
              "water_breakthrough          0.77     0.27       0.69           1.47   0.70   \n",
              "productivity_loss           0.61     0.50       0.83           2.09   0.86   \n",
              "gaslift_instability         3.25     4.27       4.31           2.35   3.48   \n",
              "\n",
              "                      wht  choke_pct  oil_vol  gas_vol  water_vol  \n",
              "shut_in              2.18       0.80     0.23     0.26       0.25  \n",
              "restart_transient    9.55       5.81     5.32     5.37       5.38  \n",
              "water_breakthrough   1.01       0.79     1.00     0.98       1.02  \n",
              "productivity_loss    1.27       0.33     1.62     1.71       0.07  \n",
              "gaslift_instability  3.30       1.58     1.28     1.25       1.38  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0f32de93-0ab2-4c53-a508-51653398ed89\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>dh_pressure</th>\n",
              "      <th>dh_temp</th>\n",
              "      <th>dp_tubing</th>\n",
              "      <th>annulus_press</th>\n",
              "      <th>whp</th>\n",
              "      <th>wht</th>\n",
              "      <th>choke_pct</th>\n",
              "      <th>oil_vol</th>\n",
              "      <th>gas_vol</th>\n",
              "      <th>water_vol</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>shut_in</th>\n",
              "      <td>2.37</td>\n",
              "      <td>2.90</td>\n",
              "      <td>3.58</td>\n",
              "      <td>1.48</td>\n",
              "      <td>3.70</td>\n",
              "      <td>2.18</td>\n",
              "      <td>0.80</td>\n",
              "      <td>0.23</td>\n",
              "      <td>0.26</td>\n",
              "      <td>0.25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>restart_transient</th>\n",
              "      <td>7.02</td>\n",
              "      <td>7.85</td>\n",
              "      <td>8.22</td>\n",
              "      <td>7.87</td>\n",
              "      <td>10.38</td>\n",
              "      <td>9.55</td>\n",
              "      <td>5.81</td>\n",
              "      <td>5.32</td>\n",
              "      <td>5.37</td>\n",
              "      <td>5.38</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>water_breakthrough</th>\n",
              "      <td>0.77</td>\n",
              "      <td>0.27</td>\n",
              "      <td>0.69</td>\n",
              "      <td>1.47</td>\n",
              "      <td>0.70</td>\n",
              "      <td>1.01</td>\n",
              "      <td>0.79</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.98</td>\n",
              "      <td>1.02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>productivity_loss</th>\n",
              "      <td>0.61</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.83</td>\n",
              "      <td>2.09</td>\n",
              "      <td>0.86</td>\n",
              "      <td>1.27</td>\n",
              "      <td>0.33</td>\n",
              "      <td>1.62</td>\n",
              "      <td>1.71</td>\n",
              "      <td>0.07</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gaslift_instability</th>\n",
              "      <td>3.25</td>\n",
              "      <td>4.27</td>\n",
              "      <td>4.31</td>\n",
              "      <td>2.35</td>\n",
              "      <td>3.48</td>\n",
              "      <td>3.30</td>\n",
              "      <td>1.58</td>\n",
              "      <td>1.28</td>\n",
              "      <td>1.25</td>\n",
              "      <td>1.38</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0f32de93-0ab2-4c53-a508-51653398ed89')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-0f32de93-0ab2-4c53-a508-51653398ed89 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-0f32de93-0ab2-4c53-a508-51653398ed89');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "  <div id=\"id_f82a9d86-3203-4910-94e9-a6060db1fd62\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('react')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_f82a9d86-3203-4910-94e9-a6060db1fd62 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('react');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "react",
              "summary": "{\n  \"name\": \"react\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"dh_pressure\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2.6034938064070743,\n        \"min\": 0.61,\n        \"max\": 7.02,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          7.02,\n          3.25,\n          0.77\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dh_temp\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3.1114096483748326,\n        \"min\": 0.27,\n        \"max\": 7.85,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          7.85,\n          4.27,\n          0.27\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dp_tubing\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3.080686611779913,\n        \"min\": 0.69,\n        \"max\": 8.22,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          8.22,\n          4.31,\n          0.69\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"annulus_press\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2.720536711753767,\n        \"min\": 1.47,\n        \"max\": 7.87,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          7.87,\n          2.35,\n          1.47\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"whp\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3.926178803875341,\n        \"min\": 0.7,\n        \"max\": 10.38,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          10.38,\n          3.48,\n          0.7\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"wht\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3.5196121945464394,\n        \"min\": 1.01,\n        \"max\": 9.55,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          9.55,\n          3.3,\n          1.01\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"choke_pct\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2.2522366660721955,\n        \"min\": 0.33,\n        \"max\": 5.81,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          5.81,\n          1.58,\n          0.79\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"oil_vol\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.984792180556947,\n        \"min\": 0.23,\n        \"max\": 5.32,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          5.32,\n          1.28,\n          1.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"gas_vol\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2.00215633755209,\n        \"min\": 0.26,\n        \"max\": 5.37,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          5.37,\n          1.25,\n          0.98\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"water_vol\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2.169942395548785,\n        \"min\": 0.07,\n        \"max\": 5.38,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          5.38,\n          1.38,\n          1.02\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's reduce the feature set (drop low-coverage channels), recount usable days"
      ],
      "metadata": {
        "id": "i7K_AIDX9mBW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "Feature analysis showed every event type is detectable through the reliable surface channels. dh_pressure/dh_temp (one gauge, 73% coverage) and\n",
        "annulus_press (74%) are dropped: no event type depends on them, and requiring them was destroying usable days. (Decision stated in paper.)\n",
        "'''\n",
        "core_features = ['dp_tubing','whp','wht','choke_pct','oil_vol','gas_vol','water_vol']\n",
        "core_delta    = [c + '_delta' for c in core_features]\n",
        "model_features = core_features + core_delta\n",
        "\n",
        "def prepare_well(g):\n",
        "    g = g.sort_values('date').copy()\n",
        "    X = g[model_features]\n",
        "    complete = X.notna().all(axis=1)\n",
        "    return g[complete], X[complete]\n",
        "\n",
        "print(\"Usable days per well with reduced feature set:\")\n",
        "for well, g in op.groupby('well'):\n",
        "    gw, _ = prepare_well(g)\n",
        "    print(f\"  {well:14s}: {len(gw):4d} / {len(g)}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-o8nO1bL9lUU",
        "outputId": "d7002448-43a4-45b3-f49b-7da39f13a9a3"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Usable days per well with reduced feature set:\n",
            "  15/9-F-1 C    :  433 / 744\n",
            "  15/9-F-11     : 1114 / 1165\n",
            "  15/9-F-12     : 2830 / 3056\n",
            "  15/9-F-14     : 2769 / 3056\n",
            "  15/9-F-15 D   :  546 / 978\n",
            "  15/9-F-5      :    0 / 144\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's normalize the features"
      ],
      "metadata": {
        "id": "uZbF986S-Kzx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "'''\n",
        "Normalize WITHIN each well: every channel -> mean 0, std 1, using only that well's own days.\n",
        "Reason: (1) channels have very different units (bar vs Sm3 vs %),\n",
        "so without scaling the large-magnitude channels dominate the anomaly score;\n",
        "(2) per-well scaling judges each well against ITS OWN normal, not a pooled baseline that would flag a well as odd just for running at different levels.\n",
        "(Stated in paper.)\n",
        "'''\n",
        "trainable_wells = ['15/9-F-1 C','15/9-F-11','15/9-F-12','15/9-F-14','15/9-F-15 D']\n",
        "\n",
        "well_data = {}   # well -> (rows_df, scaled_feature_matrix)\n",
        "for well in trainable_wells:\n",
        "    g = op[op.well == well]\n",
        "    gw, Xw = prepare_well(g)                 # complete rows only\n",
        "    Xs = StandardScaler().fit_transform(Xw)  # fit on THIS well only\n",
        "    well_data[well] = (gw, Xs)\n",
        "    print(f\"{well:14s}: {Xs.shape[0]} days x {Xs.shape[1]} features, \"\n",
        "          f\"mean~{Xs.mean():.2f}, std~{Xs.std():.2f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VPZ4Vxhr-PQ3",
        "outputId": "3612d0e7-6c06-4177-933d-74a88a22463a"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "15/9-F-1 C    : 433 days x 14 features, mean~-0.00, std~1.00\n",
            "15/9-F-11     : 1114 days x 14 features, mean~0.00, std~1.00\n",
            "15/9-F-12     : 2830 days x 14 features, mean~-0.00, std~1.00\n",
            "15/9-F-14     : 2769 days x 14 features, mean~-0.00, std~1.00\n",
            "15/9-F-15 D   : 546 days x 14 features, mean~0.00, std~1.00\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Contamination parameter\n",
        "\n",
        "We'll use the Isolation Forest algorithm and it needs a prior estimate of the fraction of anomalous days. We set **contamination = 0.08**, matching the ~6–8% of producer well-days that our grounded events cover. Tnis is not an arbitrary guess.\n",
        "\n",
        "*Caveat:* this ties the baseline's alert budget to our constructed labels' prevalence. A defensible choice, not ground truth; a sweep (0.05 / 0.08 / 0.10) can show it isn't an artifact."
      ],
      "metadata": {
        "id": "4NTkdXF8-0uE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import IsolationForest\n",
        "\n",
        "'''\n",
        "Train one unsupervised Isolation Forest per well on that well's normalized features. Per-well because each well has its own normal behaviour.\n",
        "The model sees NO labels, then it learns \"normal\" from the telemetry alone, then scores each day by how anomalous it is.\n",
        "Labels are used only later, to evaluate. contamination = 0.08 (see note above): matches grounded-event prevalence.\n",
        "'''\n",
        "for well in trainable_wells:\n",
        "    gw, Xs = well_data[well]                       # rows + normalized features\n",
        "    iso = IsolationForest(\n",
        "        n_estimators=200,        # 200 random trees — enough to stabilize scores\n",
        "        contamination=0.08,      # stated assumption: ~event prevalence\n",
        "        random_state=0           # fixed seed = reproducible (same result every run)\n",
        "    )\n",
        "    iso.fit(Xs)                                    # train on this well only\n",
        "    scores = -iso.score_samples(Xs)                # higher = more anomalous\n",
        "    well_data[well] = (gw, Xs, scores)             # store scores alongside\n",
        "\n",
        "    print(f\"{well:14s}: trained on {len(gw)} days, \"\n",
        "          f\"score range [{scores.min():.2f}, {scores.max():.2f}]\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FXDHpMWM-2U2",
        "outputId": "d0203120-6c30-4a68-9c6d-11c3b4a299ba"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "15/9-F-1 C    : trained on 433 days, score range [0.33, 0.66]\n",
            "15/9-F-11     : trained on 1114 days, score range [0.34, 0.72]\n",
            "15/9-F-12     : trained on 2830 days, score range [0.34, 0.70]\n",
            "15/9-F-14     : trained on 2769 days, score range [0.34, 0.72]\n",
            "15/9-F-15 D   : trained on 546 days, score range [0.33, 0.72]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Evaluating the baseline against the grounded events\n",
        "\n",
        "The models above were trained **with  no labels**. They never saw the  grounded events, only the raw telemetry. We now ask the key question:\n",
        "\n",
        "**Do the days this blind model flagged as anomalous line up with the grounded events our rules identified?**\n",
        "\n",
        "If they do, that is independent evidence our grounded labels are not arbitrary: a method that knew nothing about our rules still found the same regions unusual. This is a check on the *labels*, not just the model, and it will be one of the paper's central points.\n",
        "\n",
        "*Note:* the grounded events are used here **only for evaluation**, never for training. Training stays unsupervised throughout."
      ],
      "metadata": {
        "id": "dyf6BqcOAsCF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import roc_auc_score, average_precision_score\n",
        "\n",
        "'''\n",
        "We assemble all scored days into one table, tagging each with whether it falls inside a grounded event.\n",
        "(Events used for EVALUATION ONLY)\n",
        "'''\n",
        "records = []\n",
        "for well in trainable_wells:\n",
        "    gw, Xs, scores = well_data[well]\n",
        "    tmp = gw[['date','well']].copy()\n",
        "    tmp['anom_score'] = scores\n",
        "    records.append(tmp)\n",
        "scored = pd.concat(records, ignore_index=True)\n",
        "\n",
        "# Mark each scored day as event / non-event using the grounded events.\n",
        "scored['is_event'] = False\n",
        "for _, e in ev.iterrows():\n",
        "    m = (scored.well == e.well) & (scored.date >= e.onset) & (scored.date <= e.offset)\n",
        "    scored.loc[m, 'is_event'] = True\n",
        "\n",
        "# Two standard measures of \"do high scores land on event days?\":\n",
        "y = scored['is_event'].astype(int).values\n",
        "s = scored['anom_score'].values\n",
        "base_rate = y.mean()\n",
        "\n",
        "auc = roc_auc_score(y, s)          # 0.5 = chance, 1.0 = perfect ranking\n",
        "ap  = average_precision_score(y, s)  # precision-recall area; compare to base_rate\n",
        "\n",
        "print(f\"Scored days: {len(scored)}   event-day base rate: {base_rate:.3f}\")\n",
        "print(f\"ROC-AUC : {auc:.3f}   (0.5 = chance)\")\n",
        "print(f\"PR-AUC  : {ap:.3f}   (lift over base = {ap/base_rate:.2f}x)\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xa3TJSXk-8dM",
        "outputId": "2056fae2-cef0-44b4-d4ed-d738a50a5b33"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Scored days: 7692   event-day base rate: 0.066\n",
            "ROC-AUC : 0.825   (0.5 = chance)\n",
            "PR-AUC  : 0.379   (lift over base = 5.75x)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Per-event-type breakdown\n",
        "\n",
        "The aggregate ROC-AUC of 0.825 can hide variation across event types. We showed earlier that gradual eventsexpress weakly in our delta features, so we expect the model to catch **abrupt** events better than **gradual** ones. We check that honestly here.\n",
        "\n",
        "**Method:** at a fixed alert budget (flag the top 8% most anomalous days per well, matching the contamination rate), what fraction of each event type gets flagged at all?"
      ],
      "metadata": {
        "id": "t3e7cfAgCQC4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "scored['alert'] = False\n",
        "for well in trainable_wells:\n",
        "    m = scored.well == well\n",
        "    thr = scored.loc[m, 'anom_score'].quantile(0.92)\n",
        "    scored.loc[m, 'alert'] = scored.loc[m, 'anom_score'] >= thr\n",
        "\n",
        "# for each event type: how many of its events overlap at least one alerted day?\n",
        "print(\"Per-event-type detection (event flagged if any of its days is alerted):\\n\")\n",
        "print(f\"{'event_type':22s} {'detected':>10s} {'total':>7s} {'rate':>7s}\")\n",
        "for et in ['shut_in','restart_transient','water_breakthrough',\n",
        "           'productivity_loss','gaslift_instability']:\n",
        "    evt = ev[ev.event_type == et]\n",
        "    if len(evt) == 0:\n",
        "        continue\n",
        "    hits = 0\n",
        "    for _, e in evt.iterrows():\n",
        "        if e.well not in trainable_wells:\n",
        "            continue\n",
        "        days = scored[(scored.well == e.well) &\n",
        "                      (scored.date >= e.onset) & (scored.date <= e.offset)]\n",
        "        if len(days) and days['alert'].any():\n",
        "            hits += 1\n",
        "    n = (evt.well.isin(trainable_wells)).sum()\n",
        "    rate = hits / n if n else float('nan')\n",
        "    print(f\"{et:22s} {hits:>10d} {n:>7d} {rate:>7.2f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RAASp90CCJ7B",
        "outputId": "9583d58a-9182-445e-9767-0e9667c87ef5"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Per-event-type detection (event flagged if any of its days is alerted):\n",
            "\n",
            "event_type               detected   total    rate\n",
            "shut_in                        70      86    0.81\n",
            "restart_transient              59      78    0.76\n",
            "water_breakthrough              3       5    0.60\n",
            "productivity_loss               3       5    0.60\n",
            "gaslift_instability            23      55    0.42\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "RRO7nVio770Y"
      }
    }
  ]
}