{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "print(\"RUNNING CLEAN pythonanalysis.ipynb\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SvvHVGkaLNXt",
        "outputId": "164fc247-99d2-463d-a994-b22481a1547f"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "RUNNING CLEAN pythonanalysis.ipynb\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "from pathlib import Path\n",
        "print(list(Path(\"data\").glob(\"*\")))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OMloI8HHdXF-",
        "outputId": "2ac478b1-8c54-4e7d-8b4f-5c23bdf72c11"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"RUNNING CLEAN pythonanalysis.ipynb\")\n",
        "\n",
        "import pandas as pd\n",
        "from pathlib import Path\n",
        "\n",
        "DATA_DIR = Path(\"data\")\n",
        "print(\"Files in data folder:\", list(DATA_DIR.glob(\"*\")))\n",
        "\n",
        "books_df = pd.read_csv(DATA_DIR / \"books_real_world.csv\")\n",
        "sales_df = pd.read_csv(DATA_DIR / \"books_sales.csv\")\n",
        "reviews_df = pd.read_csv(DATA_DIR / \"books_reviews.csv\")\n",
        "features_df = pd.read_csv(DATA_DIR / \"books_features.csv\")\n",
        "master_df = pd.read_csv(DATA_DIR / \"books_master.csv\")\n",
        "\n",
        "sales_df[\"date\"] = pd.to_datetime(sales_df[\"date\"], errors=\"coerce\")\n",
        "reviews_df[\"review_date\"] = pd.to_datetime(reviews_df[\"review_date\"], errors=\"coerce\")\n",
        "\n",
        "print(\"Files loaded successfully.\")\n",
        "master_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 418
        },
        "id": "U-ihZ7BphbXd",
        "outputId": "6fe7c7d6-1b8a-4360-ec53-bf54f34384f3"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "RUNNING CLEAN pythonanalysis.ipynb\n",
            "Files in data folder: []\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "[Errno 2] No such file or directory: 'data/books_real_world.csv'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m/tmp/ipykernel_12271/775790723.py\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Files in data folder:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDATA_DIR\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglob\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"*\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mbooks_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDATA_DIR\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m\"books_real_world.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0msales_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDATA_DIR\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m\"books_sales.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mreviews_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDATA_DIR\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m\"books_reviews.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[0m\n\u001b[1;32m   1024\u001b[0m     \u001b[0mkwds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1026\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1027\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1028\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    618\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    619\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 620\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    621\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    622\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m   1618\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1619\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandles\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIOHandles\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1620\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1621\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1622\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, f, engine)\u001b[0m\n\u001b[1;32m   1878\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1879\u001b[0m                     \u001b[0mmode\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m\"b\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1880\u001b[0;31m             self.handles = get_handle(\n\u001b[0m\u001b[1;32m   1881\u001b[0m                 \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1882\u001b[0m                 \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/common.py\u001b[0m in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m    871\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    872\u001b[0m             \u001b[0;31m# Encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 873\u001b[0;31m             handle = open(\n\u001b[0m\u001b[1;32m    874\u001b[0m                 \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    875\u001b[0m                 \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data/books_real_world.csv'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from pathlib import Path\n",
        "\n",
        "DATA_DIR = Path(\"data\")\n",
        "\n",
        "books_df = pd.read_csv(DATA_DIR / \"books_real_world.csv\")\n",
        "sales_df = pd.read_csv(DATA_DIR / \"books_sales.csv\")\n",
        "reviews_df = pd.read_csv(DATA_DIR / \"books_reviews.csv\")\n",
        "features_df = pd.read_csv(DATA_DIR / \"books_features.csv\")\n",
        "master_df = pd.read_csv(DATA_DIR / \"books_master.csv\")\n",
        "\n",
        "sales_df[\"date\"] = pd.to_datetime(sales_df[\"date\"], errors=\"coerce\")\n",
        "reviews_df[\"review_date\"] = pd.to_datetime(reviews_df[\"review_date\"], errors=\"coerce\")\n",
        "\n",
        "print(\"Files loaded successfully.\")\n",
        "master_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 382
        },
        "id": "k2WwKFrVdYW-",
        "outputId": "6c4f7de3-5ec6-4115-beab-1e131e642a7c"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "[Errno 2] No such file or directory: 'data/books_real_world.csv'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m/tmp/ipykernel_12271/3373472929.py\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mDATA_DIR\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"data\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mbooks_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDATA_DIR\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m\"books_real_world.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0msales_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDATA_DIR\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m\"books_sales.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mreviews_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDATA_DIR\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m\"books_reviews.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[0m\n\u001b[1;32m   1024\u001b[0m     \u001b[0mkwds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1026\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1027\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1028\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    618\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    619\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 620\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    621\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    622\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m   1618\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1619\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandles\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIOHandles\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1620\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1621\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1622\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, f, engine)\u001b[0m\n\u001b[1;32m   1878\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1879\u001b[0m                     \u001b[0mmode\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m\"b\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1880\u001b[0;31m             self.handles = get_handle(\n\u001b[0m\u001b[1;32m   1881\u001b[0m                 \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1882\u001b[0m                 \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/common.py\u001b[0m in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m    871\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    872\u001b[0m             \u001b[0;31m# Encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 873\u001b[0;31m             handle = open(\n\u001b[0m\u001b[1;32m    874\u001b[0m                 \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    875\u001b[0m                 \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data/books_real_world.csv'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pathlib import Path\n",
        "\n",
        "BASE_DIR = Path(\".\")\n",
        "ART_DIR = BASE_DIR / \"artifacts\"\n",
        "PY_FIG_DIR = ART_DIR / \"py\" / \"figures\"\n",
        "PY_TAB_DIR = ART_DIR / \"py\" / \"tables\"\n",
        "\n",
        "PY_FIG_DIR.mkdir(parents=True, exist_ok=True)\n",
        "PY_TAB_DIR.mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "print(\"Saving figures to:\", PY_FIG_DIR.resolve())\n",
        "print(\"Saving tables to:\", PY_TAB_DIR.resolve())"
      ],
      "metadata": {
        "id": "KUzI3Lv9NU1R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"=== MASTER DATASET INFO ===\")\n",
        "print(master_df.info())\n",
        "\n",
        "print(\"\\n=== MISSING VALUES ===\")\n",
        "print(master_df.isnull().sum())\n",
        "\n",
        "print(\"\\n=== SAMPLE ===\")\n",
        "master_df.head()"
      ],
      "metadata": {
        "id": "JMPXM3xadZq-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install vaderSentiment"
      ],
      "metadata": {
        "id": "n1NpS9gBLz5l"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
        "\n",
        "analyzer = SentimentIntensityAnalyzer()\n",
        "\n",
        "reviews_df[\"sentiment_score\"] = reviews_df[\"review_text\"].apply(\n",
        "    lambda x: analyzer.polarity_scores(str(x))[\"compound\"]\n",
        ")\n",
        "\n",
        "def label_sentiment(score):\n",
        "    if score > 0.2:\n",
        "        return \"positive\"\n",
        "    elif score < -0.2:\n",
        "        return \"negative\"\n",
        "    else:\n",
        "        return \"neutral\"\n",
        "\n",
        "reviews_df[\"sentiment_label\"] = reviews_df[\"sentiment_score\"].apply(label_sentiment)\n",
        "\n",
        "reviews_df.head()"
      ],
      "metadata": {
        "id": "-cpX61cddqVd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentiment_summary = reviews_df.groupby(\"book_id\", as_index=False).agg(\n",
        "    avg_sentiment_score=(\"sentiment_score\", \"mean\"),\n",
        "    positive_review_share=(\"sentiment_label\", lambda x: (x == \"positive\").mean()),\n",
        "    negative_review_share=(\"sentiment_label\", lambda x: (x == \"negative\").mean()),\n",
        "    neutral_review_share=(\"sentiment_label\", lambda x: (x == \"neutral\").mean())\n",
        ")\n",
        "\n",
        "sentiment_summary.head()"
      ],
      "metadata": {
        "id": "rKoOhBmGdsmF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "analysis_df = master_df.merge(sentiment_summary, on=\"book_id\", how=\"left\")\n",
        "\n",
        "analysis_df.head()"
      ],
      "metadata": {
        "id": "AG-xSVCDduEl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "analysis_df.describe()"
      ],
      "metadata": {
        "id": "oSwkiEr_dw5l"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "top_books = analysis_df.sort_values(\"total_revenue\", ascending=False)[\n",
        "    [\"title\", \"category\", \"price_gbp\", \"total_units_sold\", \"total_revenue\", \"pricing_action\"]\n",
        "].head(10)\n",
        "\n",
        "top_books"
      ],
      "metadata": {
        "id": "FI5b26FMdzY9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "top_10 = analysis_df.sort_values(\"total_revenue\", ascending=False).head(10)\n",
        "\n",
        "plt.figure(figsize=(10, 6))\n",
        "plt.barh(top_10[\"title\"], top_10[\"total_revenue\"])\n",
        "plt.gca().invert_yaxis()\n",
        "plt.title(\"Top 10 Books by Revenue\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "o2bjBD2Ad18t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(10, 6))\n",
        "plt.barh(top_10[\"title\"], top_10[\"total_revenue\"])\n",
        "plt.gca().invert_yaxis()\n",
        "plt.title(\"Top 10 Books by Revenue\")\n",
        "plt.tight_layout()\n",
        "plt.savefig(PY_FIG_DIR / \"top_10_books_by_revenue.png\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "9gJaV-L8Nnyy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(8, 5))\n",
        "plt.scatter(analysis_df[\"avg_sentiment_score\"], analysis_df[\"total_revenue\"])\n",
        "plt.title(\"Sentiment vs Revenue\")\n",
        "plt.xlabel(\"Average Sentiment Score\")\n",
        "plt.ylabel(\"Total Revenue\")\n",
        "plt.tight_layout()\n",
        "plt.savefig(PY_FIG_DIR / \"sentiment_vs_revenue.png\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "VTERtp3eNxEB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(8, 5))\n",
        "plt.scatter(analysis_df[\"price_gbp\"], analysis_df[\"total_units_sold\"])\n",
        "plt.title(\"Price vs Units Sold\")\n",
        "plt.xlabel(\"Price\")\n",
        "plt.ylabel(\"Units Sold\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "-1Mr_Ez0d6m9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(8, 5))\n",
        "plt.scatter(analysis_df[\"avg_sentiment_score\"], analysis_df[\"total_revenue\"])\n",
        "plt.title(\"Sentiment vs Revenue\")\n",
        "plt.xlabel(\"Sentiment Score\")\n",
        "plt.ylabel(\"Revenue\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "1ar--sRUd9Jj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "category_summary = analysis_df.groupby(\"category\", as_index=False).agg(\n",
        "    total_revenue=(\"total_revenue\", \"sum\"),\n",
        "    avg_review_score=(\"avg_review_score\", \"mean\"),\n",
        "    avg_sentiment_score=(\"avg_sentiment_score\", \"mean\")\n",
        ")\n",
        "\n",
        "category_summary"
      ],
      "metadata": {
        "id": "VoggtyqeeAAp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "category_summary = category_summary.sort_values(\"total_revenue\")\n",
        "\n",
        "plt.figure(figsize=(8, 5))\n",
        "plt.barh(category_summary[\"category\"], category_summary[\"total_revenue\"])\n",
        "plt.title(\"Revenue by Category\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "R7wik3_4eCLh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "model_df = analysis_df.copy()\n",
        "\n",
        "# Fill missing numeric values\n",
        "for col in model_df.select_dtypes(include=\"number\").columns:\n",
        "    model_df[col] = model_df[col].fillna(model_df[col].median())\n",
        "\n",
        "# Encode category\n",
        "encoder = LabelEncoder()\n",
        "model_df[\"category_encoded\"] = encoder.fit_transform(model_df[\"category\"].astype(str))\n",
        "\n",
        "X = model_df[\n",
        "    [\n",
        "        \"price_gbp\",\n",
        "        \"rating\",\n",
        "        \"in_stock\",\n",
        "        \"supplier_cost\",\n",
        "        \"marketing_score\",\n",
        "        \"total_units_sold\",\n",
        "        \"avg_monthly_units\",\n",
        "        \"avg_review_score\",\n",
        "        \"review_count\",\n",
        "        \"avg_sentiment_score\",\n",
        "        \"positive_review_share\",\n",
        "        \"negative_review_share\",\n",
        "        \"neutral_review_share\",\n",
        "        \"category_encoded\"\n",
        "    ]\n",
        "]\n",
        "\n",
        "y = model_df[\"pricing_action\"]\n",
        "\n",
        "X.head()"
      ],
      "metadata": {
        "id": "yf3sS8CJeHeq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    X, y, test_size=0.2, random_state=42\n",
        ")"
      ],
      "metadata": {
        "id": "tiYbWpOceKJJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.metrics import classification_report, accuracy_score\n",
        "\n",
        "clf = RandomForestClassifier(n_estimators=200, random_state=42)\n",
        "clf.fit(X_train, y_train)\n",
        "\n",
        "y_pred = clf.predict(X_test)\n",
        "\n",
        "print(\"Accuracy:\", accuracy_score(y_test, y_pred))\n",
        "print(classification_report(y_test, y_pred))"
      ],
      "metadata": {
        "id": "kxBnZ4IkeNI2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "importance_df = pd.DataFrame({\n",
        "    \"feature\": X.columns,\n",
        "    \"importance\": clf.feature_importances_\n",
        "}).sort_values(\"importance\", ascending=False)\n",
        "\n",
        "importance_df"
      ],
      "metadata": {
        "id": "TLeDMMP2eP_F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(10, 6))\n",
        "plt.barh(importance_df[\"feature\"], importance_df[\"importance\"])\n",
        "plt.gca().invert_yaxis()\n",
        "plt.title(\"Feature Importance\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "c9HRi0q4eSjO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "analysis_df[\"predicted_pricing_action\"] = clf.predict(X)\n",
        "\n",
        "analysis_df[[\"title\", \"pricing_action\", \"predicted_pricing_action\"]].head()"
      ],
      "metadata": {
        "id": "DnKFclJ8eVKF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "best_book = analysis_df.sort_values(\"total_revenue\", ascending=False).iloc[0][\"book_id\"]\n",
        "\n",
        "ts = (\n",
        "    sales_df[sales_df[\"book_id\"] == best_book]\n",
        "    .groupby(\"date\")[\"revenue\"]\n",
        "    .sum()\n",
        ")\n",
        "\n",
        "ts.index = pd.to_datetime(ts.index)\n",
        "ts = ts.asfreq(\"MS\")\n",
        "\n",
        "ts"
      ],
      "metadata": {
        "id": "sFtWYML8eXfd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from statsmodels.tsa.arima.model import ARIMA\n",
        "\n",
        "model = ARIMA(ts, order=(1,1,1))\n",
        "model_fit = model.fit()\n",
        "\n",
        "forecast = model_fit.forecast(steps=3)\n",
        "\n",
        "forecast"
      ],
      "metadata": {
        "id": "gwp9i27meafy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(10, 5))\n",
        "plt.plot(ts, label=\"History\")\n",
        "plt.plot(forecast, label=\"Forecast\")\n",
        "plt.legend()\n",
        "plt.title(\"Revenue Forecast\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "WhwoskjwedVS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def recommend(row):\n",
        "    if row[\"avg_sentiment_score\"] > 0.4 and row[\"avg_monthly_units\"] > 60:\n",
        "        return \"Increase price\"\n",
        "    elif row[\"avg_sentiment_score\"] < 0:\n",
        "        return \"Discount\"\n",
        "    else:\n",
        "        return \"Keep price\"\n",
        "\n",
        "analysis_df[\"recommendation\"] = analysis_df.apply(recommend, axis=1)\n",
        "\n",
        "analysis_df[[\"title\", \"recommendation\"]].head()"
      ],
      "metadata": {
        "id": "W5CQz9uUefy6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "final_output = analysis_df[\n",
        "    [\"book_id\", \"title\", \"category\", \"price_gbp\", \"total_revenue\", \"recommendation\"]\n",
        "]\n",
        "\n",
        "final_output.head()"
      ],
      "metadata": {
        "id": "Wi86eAS7ejBz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "analysis_df.to_csv(\"analysis_results.csv\", index=False)\n",
        "final_output.to_csv(\"final_recommendations.csv\", index=False)\n",
        "\n",
        "print(\"Saved successfully\")"
      ],
      "metadata": {
        "id": "OwQQ-Pm6emoj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "final_output.to_csv(\"final_recommendations.csv\", index=False)"
      ],
      "metadata": {
        "id": "B1tXy1n_igCZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "analysis_df.to_csv(PY_TAB_DIR / \"book_analysis_full.csv\", index=False)\n",
        "final_output.to_csv(PY_TAB_DIR / \"final_recommendations.csv\", index=False)\n",
        "category_summary.to_csv(PY_TAB_DIR / \"category_summary.csv\", index=False)\n",
        "importance_df.to_csv(PY_TAB_DIR / \"feature_importance.csv\", index=False)\n",
        "\n",
        "print(\"Dashboard tables saved.\")"
      ],
      "metadata": {
        "id": "FwP4V2bENfAv"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}