{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "kz8lLSv6mVQo" }, "source": [ "# **🤖 Data Analysis & Visualization**" ] }, { "cell_type": "markdown", "metadata": { "id": "jpASMyIQMaAq" }, "source": [ "## **1.** 📦 Install required packages" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "f48c8f8c", "outputId": "31fb8283-7f35-4fb0-f270-7c040da95ed4", "collapsed": true }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: pandas in /usr/local/lib/python3.12/dist-packages (2.2.2)\n", "Requirement already satisfied: matplotlib in /usr/local/lib/python3.12/dist-packages (3.10.0)\n", "Requirement already satisfied: seaborn in /usr/local/lib/python3.12/dist-packages (0.13.2)\n", "Requirement already satisfied: numpy in /usr/local/lib/python3.12/dist-packages (2.0.2)\n", "Requirement already satisfied: textblob in /usr/local/lib/python3.12/dist-packages (0.19.0)\n", "Collecting faker\n", " Downloading faker-40.8.0-py3-none-any.whl.metadata (16 kB)\n", "Requirement already satisfied: transformers in /usr/local/lib/python3.12/dist-packages (5.0.0)\n", "Collecting vaderSentiment\n", " Downloading vaderSentiment-3.3.2-py2.py3-none-any.whl.metadata (572 bytes)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas) (2.9.0.post0)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas) (2025.2)\n", "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas) (2025.3)\n", "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (1.3.3)\n", "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (0.12.1)\n", "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (4.61.1)\n", "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (1.4.9)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (26.0)\n", "Requirement already satisfied: pillow>=8 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (11.3.0)\n", "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (3.3.2)\n", "Requirement already satisfied: nltk>=3.9 in /usr/local/lib/python3.12/dist-packages (from textblob) (3.9.1)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from transformers) (3.25.0)\n", "Requirement already satisfied: huggingface-hub<2.0,>=1.3.0 in /usr/local/lib/python3.12/dist-packages (from transformers) (1.5.0)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.12/dist-packages (from transformers) (6.0.3)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.12/dist-packages (from transformers) (2025.11.3)\n", "Requirement already satisfied: tokenizers<=0.23.0,>=0.22.0 in /usr/local/lib/python3.12/dist-packages (from transformers) (0.22.2)\n", "Requirement already satisfied: typer-slim in /usr/local/lib/python3.12/dist-packages (from transformers) (0.24.0)\n", "Requirement already satisfied: safetensors>=0.4.3 in /usr/local/lib/python3.12/dist-packages (from transformers) (0.7.0)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.12/dist-packages (from transformers) (4.67.3)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (from vaderSentiment) (2.32.4)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub<2.0,>=1.3.0->transformers) (2025.3.0)\n", "Requirement already satisfied: hf-xet<2.0.0,>=1.2.0 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub<2.0,>=1.3.0->transformers) (1.3.2)\n", "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub<2.0,>=1.3.0->transformers) (0.28.1)\n", "Requirement already satisfied: typer in /usr/local/lib/python3.12/dist-packages (from huggingface-hub<2.0,>=1.3.0->transformers) (0.24.1)\n", "Requirement already satisfied: typing-extensions>=4.1.0 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub<2.0,>=1.3.0->transformers) (4.15.0)\n", "Requirement already satisfied: click in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (8.3.1)\n", "Requirement already satisfied: joblib in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (1.5.3)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n", "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests->vaderSentiment) (3.4.4)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.12/dist-packages (from requests->vaderSentiment) (3.11)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests->vaderSentiment) (2.5.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.12/dist-packages (from requests->vaderSentiment) (2026.2.25)\n", "Requirement already satisfied: anyio in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.3.0->transformers) (4.12.1)\n", "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.3.0->transformers) (1.0.9)\n", "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.3.0->transformers) (0.16.0)\n", "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.12/dist-packages (from typer->huggingface-hub<2.0,>=1.3.0->transformers) (1.5.4)\n", "Requirement already satisfied: rich>=12.3.0 in /usr/local/lib/python3.12/dist-packages (from typer->huggingface-hub<2.0,>=1.3.0->transformers) (13.9.4)\n", "Requirement already satisfied: annotated-doc>=0.0.2 in /usr/local/lib/python3.12/dist-packages (from typer->huggingface-hub<2.0,>=1.3.0->transformers) (0.0.4)\n", "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich>=12.3.0->typer->huggingface-hub<2.0,>=1.3.0->transformers) (4.0.0)\n", "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich>=12.3.0->typer->huggingface-hub<2.0,>=1.3.0->transformers) (2.19.2)\n", "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich>=12.3.0->typer->huggingface-hub<2.0,>=1.3.0->transformers) (0.1.2)\n", "Downloading faker-40.8.0-py3-none-any.whl (2.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m20.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading vaderSentiment-3.3.2-py2.py3-none-any.whl (125 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.0/126.0 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hInstalling collected packages: faker, vaderSentiment\n", "Successfully installed faker-40.8.0 vaderSentiment-3.3.2\n" ] } ], "source": [ "!pip install pandas matplotlib seaborn numpy textblob faker transformers vaderSentiment\n" ] }, { "cell_type": "markdown", "metadata": { "id": "NZd99NpKkKyp" }, "source": [ "## **2.** ✅️ Load & inspect input datasets" ] }, { "cell_type": "markdown", "metadata": { "id": "_JBLmm508Uq2" }, "source": [ "### *a. Initial setup*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "eBDXPQz18Xrs" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import random" ] }, { "cell_type": "markdown", "metadata": { "id": "IL8lZbMm8m3k" }, "source": [ "### *b. ✋🏻🛑⛔️ Create the df_reviews dataframe from the synthetic_book_reviews.csv file*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fdgjghfO8uuq", "colab": { "base_uri": "https://localhost:8080/", "height": 311 }, "outputId": "e14f27ca-3651-4e5f-e8d6-fcb1446e9ec9" }, "outputs": [ { "output_type": "error", "ename": "FileNotFoundError", "evalue": "[Errno 2] No such file or directory: 'synthetic_book_reviews.csv'", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m/tmp/ipykernel_347/448808692.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf_reviews\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"synthetic_book_reviews.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[0m\n\u001b[1;32m 1024\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1026\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1027\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1028\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 618\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 619\u001b[0m \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 620\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 621\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 622\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 1618\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1619\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandles\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIOHandles\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1620\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1621\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1622\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, f, engine)\u001b[0m\n\u001b[1;32m 1878\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1879\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m\"b\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1880\u001b[0;31m self.handles = get_handle(\n\u001b[0m\u001b[1;32m 1881\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1882\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/common.py\u001b[0m in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m 871\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 872\u001b[0m \u001b[0;31m# Encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 873\u001b[0;31m handle = open(\n\u001b[0m\u001b[1;32m 874\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 875\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'synthetic_book_reviews.csv'" ] } ], "source": [ "df_reviews = pd.read_csv(\"synthetic_book_reviews.csv\")" ] }, { "cell_type": "markdown", "metadata": { "id": "N-Dl37J0HLhU" }, "source": [ "### *c. ✋🏻🛑⛔️ Create the df_sales dataframe from the synthetic_sales_data.csv file*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "6XZs3P7fHgQe" }, "outputs": [], "source": [ "df_sales = pd.read_csv(\"synthetic_sales_data.csv\")" ] }, { "cell_type": "markdown", "metadata": { "id": "MUI3SkmyrGQo" }, "source": [ "### *d. ✋🏻🛑⛔️ Visualize the first few lines of the two final datasets: df_reviews and df_sales*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "p8FdQFXErOqE", "collapsed": true }, "outputs": [], "source": [ "df_reviews.head()" ] }, { "cell_type": "code", "source": [ "df_sales.head()" ], "metadata": { "collapsed": true, "id": "EDdSx5KMNjiB" }, "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "Y3oqGHsmrQzx" }, "source": [ "### *d. Run a quality check on the datasets*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "VArQGPoKrfLm", "collapsed": true }, "outputs": [], "source": [ "def quality_check(df, name=\"DataFrame\"):\n", " print(f\"\\n🔍 Quality Check Report for: {name}\")\n", " print(\"=\" * (25 + len(name)))\n", "\n", " # Basic info\n", " print(f\"\\n📏 Shape: {df.shape}\")\n", " print(\"\\n🔠 Column Types:\")\n", " print(df.dtypes)\n", "\n", " # Missing values\n", " print(\"\\n❓ Missing Values:\")\n", " print(df.isnull().sum())\n", "\n", " # Duplicates\n", " duplicate_count = df.duplicated().sum()\n", " print(f\"\\n📋 Duplicate Rows: {duplicate_count}\")\n", "\n", " # Summary stats\n", " print(\"\\n📊 Summary Statistics:\")\n", " display(df.describe(include='all').transpose())\n", "\n", " # Sample rows\n", " print(\"\\n👀 Sample Rows:\")\n", " display(df.sample(5))\n", "\n", "# Run checks\n", "quality_check(df_reviews, \"df_reviews\")\n", "quality_check(df_sales, \"df_sales\")\n" ] }, { "cell_type": "markdown", "metadata": { "id": "TTxUKDYINPxV" }, "source": [ "## **3.** 🎭 Perform sentiment analysis using VADER" ] }, { "cell_type": "markdown", "metadata": { "id": "OqhYU8rDxQRT" }, "source": [ "### *a. Initial setup*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "DNk5w8mNxSZ6" }, "outputs": [], "source": [ "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n", "\n", "# 🤖 Initialize VADER analyzer\n", "analyzer = SentimentIntensityAnalyzer()" ] }, { "cell_type": "markdown", "metadata": { "id": "P123TwSWxVAr" }, "source": [ "### *b. Create a function get_sentiment_label that will return the label negative, neutral, or positive based on the VADER analyzer's scoring of the text*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "89809e6f" }, "outputs": [], "source": [ "def get_sentiment_label(text):\n", " score = analyzer.polarity_scores(text)[\"compound\"]\n", " if score >= 0.05:\n", " return \"positive\"\n", " elif score <= -0.05:\n", " return \"negative\"\n", " else:\n", " return \"neutral\"" ] }, { "cell_type": "markdown", "metadata": { "id": "DS9eCZ95yQn3" }, "source": [ "### *c. ✋🏻🛑⛔️ Apply get_sentiment_label to df_reviews column named review_text to get sentiment_label column*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "SpXzFaDfyM7I" }, "outputs": [], "source": [ "# Create sentiment_label column using VADER sentiment analysis\n", "df_reviews[\"sentiment_label\"] = df_reviews[\"review_text\"].apply(get_sentiment_label)\n", "\n", "# Preview results\n", "df_reviews.head()" ] }, { "cell_type": "markdown", "metadata": { "id": "5cnPCFFnyXN6" }, "source": [ "### *d. ✋🏻🛑⛔️ View the first few lines of the resulting table df_reviews*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ODGyfjBSyZEO" }, "outputs": [], "source": [ "df_reviews.head()" ] }, { "cell_type": "markdown", "metadata": { "id": "Qy3Hqm-FojvT" }, "source": [ "## **4.** 📊 Use the following data visualization code snippets" ] }, { "cell_type": "markdown", "metadata": { "id": "lcjGSw2bzqtZ" }, "source": [ "### *a. Initial setup*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "p5LV2o1rzsiC" }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "import matplotlib.dates as mdates" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tvaBtswpGS__" }, "outputs": [], "source": [ "# ----------------------------\n", "# Outputs (for Hugging Face app)\n", "# ----------------------------\n", "# In the notebook: you still SEE interactive tables/plots inline.\n", "# For the Space dashboard: we also SAVE the same outputs as files.\n", "\n", "from pathlib import Path\n", "\n", "ART_DIR = Path(\"artifacts\")\n", "PY_FIG = ART_DIR / \"py\" / \"figures\"\n", "PY_TAB = ART_DIR / \"py\" / \"tables\"\n", "\n", "for p in [PY_FIG, PY_TAB]:\n", " p.mkdir(parents=True, exist_ok=True)\n", "\n", "print(\"✅ Output folders:\")\n", "print(\" -\", PY_FIG.resolve())\n", "print(\" -\", PY_TAB.resolve())\n" ] }, { "cell_type": "markdown", "metadata": { "id": "b9T1rkBe0AJU" }, "source": [ "### *b. Sample of 5 books for each popularity level for visualizations*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "sLdFmGqXqo_t" }, "outputs": [], "source": [ "sampled_titles = []\n", "for pop_score in sorted(df_reviews[\"popularity_score\"].dropna().unique()):\n", " all_titles = df_reviews[df_reviews[\"popularity_score\"] == pop_score][\"title\"].unique()\n", " sampled = random.sample(list(all_titles), min(5, len(all_titles)))\n", " sampled_titles.extend(sampled)" ] }, { "cell_type": "markdown", "metadata": { "id": "xq7-C8m70mMH" }, "source": [ "### *c. Copy relevant sales, reviews, and book names*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "laDdMece0qrq" }, "outputs": [], "source": [ "sampled_sales = df_sales[df_sales[\"title\"].isin(sampled_titles)].copy()\n", "sampled_reviews = df_reviews[df_reviews[\"title\"].isin(sampled_titles)].copy()\n", "sampled_books = df_reviews[df_reviews[\"title\"].isin(sampled_titles)].copy()" ] }, { "cell_type": "markdown", "metadata": { "id": "8YtfkG_A0wTy" }, "source": [ "### *d. Plot sales trends over time for the sampled books*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1iTVzflW0Rkw" }, "outputs": [], "source": [ "# 🕒 Ensure datetime format\n", "df_sales[\"month\"] = pd.to_datetime(df_sales[\"month\"])\n", "# 🎨 Color mapping\n", "popularity_colors = {\n", " 1: \"darkred\", 2: \"orangered\", 3: \"gold\", 4: \"mediumseagreen\", 5: \"royalblue\"\n", "}\n", "\n", "# 📈 Plot 1: Sales trends\n", "plt.figure(figsize=(20, 8))\n", "for title in sampled_titles:\n", " row = sampled_books[sampled_books[\"title\"] == title].iloc[0]\n", " color = popularity_colors.get(row[\"popularity_score\"], \"gray\")\n", " subset = sampled_sales[sampled_sales[\"title\"] == title]\n", " plt.plot(subset[\"month\"], subset[\"units_sold\"], label=f\"{title} (Pop. {row['popularity_score']})\", color=color)\n", "\n", "plt.title(\"📈 Sales Trends Over Time (5 per Popularity Level)\")\n", "plt.xlabel(\"Month\")\n", "plt.ylabel(\"Units Sold\")\n", "plt.xticks(rotation=45)\n", "plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), fontsize='small')\n", "plt.grid(True)\n", "plt.tight_layout()\n", "plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b %Y'))\n", "plt.savefig(PY_FIG / 'sales_trends_sampled_titles.png', dpi=150)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "lDpMkjDP1K6j" }, "source": [ "### *e. Plot sentiment_label distribution per book*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dn1Jgd5R1KLu" }, "outputs": [], "source": [ "# 🎨 Give a new name to each book that includes the rating together with the title\n", "sampled_reviews[\"grouped_title\"] = sampled_reviews[\"rating\"].astype(str) + \"★ | \" + sampled_reviews[\"title\"]\n", "\n", "# 📊 Aggregate sentiment counts\n", "sentiment_counts = (\n", " sampled_reviews.groupby([\"grouped_title\", \"sentiment_label\"])\n", " .size()\n", " .unstack(fill_value=0)[[\"negative\", \"neutral\", \"positive\"]] # consistent order\n", ")\n", "\n", "# 💾 Save table for HF dashboard\n", "sentiment_counts.reset_index().to_csv(PY_TAB / 'sentiment_counts_sampled.csv', index=False)\n", "\n", "\n", "# ✅ Plot stacked horizontal bars\n", "fig, ax = plt.subplots(figsize=(12, 14))\n", "sentiment_counts.plot.barh(\n", " stacked=True,\n", " ax=ax,\n", " color={\"negative\": \"royalblue\", \"neutral\": \"lightgray\", \"positive\": \"crimson\"}\n", ")\n", "\n", "plt.title(\"💬 Sentiment Distribution in Reviews (5 Books per Popularity Level)\", fontsize=14)\n", "plt.xlabel(\"Number of Reviews\")\n", "plt.ylabel(\"Book Title (Grouped by Popularity Score)\")\n", "plt.legend(title=\"Sentiment\", loc=\"lower right\")\n", "plt.grid(axis=\"x\", linestyle=\"--\", alpha=0.6)\n", "plt.tight_layout()\n", "plt.savefig(PY_FIG / 'sentiment_distribution_sampled_titles.png', dpi=150)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "rmgylC1ENCHy" }, "source": [ "## **5.** 🔮 Forecast book sales with the following ARIMA code" ] }, { "cell_type": "markdown", "metadata": { "id": "jFV4JE1R3FKH" }, "source": [ "### *a. Initial setup*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Mh8Alha03H22" }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "import matplotlib.dates as mdates\n", "import statsmodels.api as sm\n", "from itertools import product\n", "import matplotlib.cm as cm\n", "import warnings" ] }, { "cell_type": "markdown", "metadata": { "id": "gHucD8OW3U0w" }, "source": [ "### *b. Define function find_best_arima to try different ARIMA parameter values and return the best combination for each book's price forecast*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "477fa43f" }, "outputs": [], "source": [ "def find_best_arima(series, p_range=(0, 5), d_range=(0, 2), q_range=(0, 1)):\n", " best_aic = float(\"inf\")\n", " best_order = None\n", " best_model = None\n", "\n", " for p, d, q in product(range(p_range[0], p_range[1] + 1),\n", " range(d_range[0], d_range[1] + 1),\n", " range(q_range[0], q_range[1] + 1)):\n", " try:\n", " model = sm.tsa.ARIMA(series, order=(p, d, q))\n", " results = model.fit()\n", " if results.aic < best_aic:\n", " best_aic = results.aic\n", " best_order = (p, d, q)\n", " best_model = results\n", " except:\n", " continue\n", "\n", " return best_order, best_model" ] }, { "cell_type": "markdown", "metadata": { "id": "Rq5t1Hey3jkD" }, "source": [ "### *c. Plot the figure*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "DmxGdvLE3dHQ" }, "outputs": [], "source": [ "# 🎨 Generate 25 highly distinct colors using HUSL (hue-saturation-lightness)\n", "colors = sns.color_palette(\"tab10\", len(sampled_titles))\n", "\n", "plt.figure(figsize=(16, 10))\n", "\n", "for i, title in enumerate(sampled_titles):\n", " book_sales = sampled_sales[sampled_sales[\"title\"] == title].copy()\n", " book_sales[\"month\"] = pd.to_datetime(book_sales[\"month\"])\n", " book_sales = book_sales.sort_values(\"month\").set_index(\"month\")\n", "\n", " with warnings.catch_warnings():\n", " warnings.simplefilter(\"ignore\")\n", " best_order, best_model = find_best_arima(book_sales[\"units_sold\"])\n", " if best_model is not None:\n", " forecast = best_model.get_forecast(steps=6)\n", " forecast_index = pd.date_range(start=book_sales.index[-1] + pd.DateOffset(months=1), periods=6, freq='MS')\n", "\n", " # 🟦 Plot observed sales (solid line)\n", " plt.plot(book_sales.index, book_sales[\"units_sold\"], color=colors[i], label=title, linewidth=2)\n", "\n", " # 🟠 Plot forecast (dotted line, same color)\n", " plt.plot(forecast_index, forecast.predicted_mean, linestyle=\"--\", color=colors[i], linewidth=2)\n", "\n", "# 📈 Final formatting\n", "plt.title(\"📈 ARIMA Forecasts for Sampled Books (1 per Popularity Level)\", fontsize=14)\n", "plt.xlabel(\"Month\")\n", "plt.ylabel(\"Units Sold\")\n", "plt.xticks(rotation=45)\n", "plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b %Y'))\n", "plt.grid(True)\n", "plt.legend(loc=\"center left\", bbox_to_anchor=(1, 0.5), fontsize=\"small\")\n", "plt.tight_layout()\n", "plt.savefig(PY_FIG / 'arima_forecasts_sampled_titles.png', dpi=150)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "SKBcx3fyCFly" }, "source": [ "## **6.** 🏷️ Decide on price changes with a rule-based approach based on sentiment and future revenue" ] }, { "cell_type": "markdown", "metadata": { "id": "nY-vV2JJDZqu" }, "source": [ "### *a. Calculate average sales per book*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nbDT_RHaDD2R" }, "outputs": [], "source": [ "avg_sales = df_sales.groupby(\"title\")[\"units_sold\"].mean().reset_index()\n", "avg_sales.columns = [\"title\", \"avg_units_sold\"]" ] }, { "cell_type": "markdown", "metadata": { "id": "94wi-RvkDf2z" }, "source": [ "### *b. Calculate sentiment distribution per book*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fWjQ9IOXDk-M" }, "outputs": [], "source": [ "sentiment_counts = df_reviews.groupby([\"title\", \"sentiment_label\"]).size().unstack(fill_value=0)\n", "sentiment_counts[\"total\"] = sentiment_counts.sum(axis=1)\n", "sentiment_counts[\"positive_ratio\"] = sentiment_counts.get(\"positive\") / sentiment_counts[\"total\"]\n", "sentiment_counts[\"negative_ratio\"] = sentiment_counts.get(\"negative\") / sentiment_counts[\"total\"]" ] }, { "cell_type": "markdown", "metadata": { "id": "Vm10ym_iDtEW" }, "source": [ "### *c. Merge the calculated sales and sentiment characteristics*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "T-zlh6rBDpxg" }, "outputs": [], "source": [ "df_decision = avg_sales.merge(sentiment_counts, on=\"title\", how=\"left\")" ] }, { "cell_type": "markdown", "metadata": { "id": "1WIWDojyD7fK" }, "source": [ "### *d. ✋🏻🛑⛔️ Create the pricing_decision function as a basic rule-based pricing decider based on sentiment and revenue*\n", "\n", "\n", "\n", "\n" ] }, { "cell_type": "markdown", "metadata": { "id": "b5qJCb46Dxfb" }, "source": [ "* If there are 120 or more average units sold and 0.6 or higher positive ratio, the decision should be to increase price.\n", "* If there are 60 or less average units sold and 0.4 or higher negative ratio, the decision should be to decrease price.\n", "* Otherwise, the price should be kept the same." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XBzozedwD6yx" }, "outputs": [], "source": [ "def pricing_decision(row):\n", " avg_units = row[\"avg_units_sold\"]\n", " positive_ratio = row.get(\"positive_ratio\", 0)\n", " negative_ratio = row.get(\"negative_ratio\", 0)\n", "\n", " if avg_units >= 120 and positive_ratio >= 0.6:\n", " return \"increase price\"\n", " elif avg_units <= 60 and negative_ratio >= 0.4:\n", " return \"decrease price\"\n", " else:\n", " return \"keep price\"\n", "\n", "df_decision[\"pricing_action\"] = df_decision.apply(pricing_decision, axis=1)\n", "\n", "# Preview results\n", "print(df_decision.head())\n", "\n", "# Optional: distribution of decisions\n", "print(df_decision[\"pricing_action\"].value_counts())" ] }, { "cell_type": "markdown", "metadata": { "id": "xmLEdF14EPAA" }, "source": [ "### *e. ✋🏻🛑⛔️ Run the pricing_decision function and check out the first few decisions*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "TZ0ZhgHrEQJB" }, "outputs": [], "source": [ "# Apply the pricing decision rule to each row\n", "df_decision[\"pricing_action\"] = df_decision.apply(pricing_decision, axis=1)\n", "\n", "# Display the first few pricing decisions\n", "df_decision[[\"title\", \"avg_units_sold\", \"positive_ratio\", \"negative_ratio\", \"pricing_action\"]].head()" ] }, { "cell_type": "markdown", "metadata": { "id": "WTkP2_-EApev" }, "source": [ "\n", "## **7.** 💾 Save Python outputs for the Hugging Face dashboard" ] }, { "cell_type": "markdown", "metadata": { "id": "3EIjfnokGpJv" }, "source": [ "\n", "This section exports **HF-ready artifacts** into a consistent folder structure:\n", "\n", "- `(root folder)py/figures/` (Python-generated visuals)\n", "- `(root folder)py/tables/` (tables/metrics)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ZJJ4PMgIApev" }, "outputs": [], "source": [ "\n", "import json\n", "\n", "# -------------------------\n", "# 1) Dashboard table (monthly) — reuse if already built\n", "# -------------------------\n", "if \"df_monthly\" in globals() and df_monthly is not None:\n", " df_dashboard = df_monthly.copy()\n", "else:\n", " # fallback: monthly units sold only\n", " df_dashboard = (\n", " df_sales.groupby(\"month\", as_index=False)\n", " .agg(total_units_sold=(\"units_sold\", \"sum\"))\n", " .sort_values(\"month\")\n", " )\n", "\n", "# Save the single overview dashboard table\n", "df_dashboard.to_csv(PY_TAB / \"df_dashboard.csv\", index=False)\n", "\n", "# -------------------------\n", "# 2) KPI summary (small json) — computed from raw df_sales + df_dashboard\n", "# -------------------------\n", "kpis = {\n", " \"n_titles\": int(df_sales[\"title\"].nunique()),\n", " \"n_months\": int(df_dashboard[\"month\"].nunique()),\n", " \"total_units_sold\": float(df_sales[\"units_sold\"].sum()),\n", "}\n", "\n", "# Only include revenue KPIs if df_dashboard contains it (since you said monthly revenue already exists)\n", "if \"total_revenue\" in df_dashboard.columns and df_dashboard[\"total_revenue\"].notna().any():\n", " kpis[\"total_revenue\"] = float(df_dashboard[\"total_revenue\"].sum())\n", "\n", "with open(PY_FIG / \"kpis.json\", \"w\", encoding=\"utf-8\") as f:\n", " json.dump(kpis, f, indent=2)\n", "\n", "# -------------------------\n", "# 3) Python tables (title-level quick inspection)\n", "# -------------------------\n", "df_by_title_units = (\n", " df_sales.groupby(\"title\", as_index=False)\n", " .agg(total_units_sold=(\"units_sold\", \"sum\"))\n", " .sort_values(\"total_units_sold\", ascending=False)\n", ")\n", "df_by_title_units.head(10).to_csv(PY_TAB / \"top_titles_by_units_sold.csv\", index=False)\n", "\n", "# Optional: title-level revenue table ONLY if df_sales already has per-row revenue\n", "if \"revenue\" in df_sales.columns and df_sales[\"revenue\"].notna().any():\n", " df_by_title_rev = (\n", " df_sales.groupby(\"title\", as_index=False)\n", " .agg(total_revenue=(\"revenue\", \"sum\"))\n", " .sort_values(\"total_revenue\", ascending=False)\n", " )\n", " df_by_title_rev.head(10).to_csv(PY_TAB / \"top_titles_by_revenue.csv\", index=False)\n", "\n", "print(\"✅ Exports written to artifacts/:\")\n", "print(\" - common/: df_dashboard.csv, kpis.json\")\n", "print(\" - py/tables/: top_titles_by_units_sold.csv (+ optional top_titles_by_revenue.csv)\")\n" ] }, { "cell_type": "markdown", "metadata": { "id": "0b4e76d3" }, "source": [ "✅ **Extra outputs for the R notebook**: `(root folder)common/r_input_title_level.csv` and `(root folder)common/r_input_monthly_revenue.csv` (these are the only two files the R portion needs)." ] } ], "metadata": { "colab": { "collapsed_sections": [ "jpASMyIQMaAq", "NZd99NpKkKyp", "TTxUKDYINPxV", "Qy3Hqm-FojvT", "rmgylC1ENCHy", "SKBcx3fyCFly", "WTkP2_-EApev" ], "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 0 }