{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "4ba6aba8" }, "source": [ "# 🤖 **Data Collection, Creation, Storage, and Processing**\n" ] }, { "cell_type": "markdown", "metadata": { "id": "jpASMyIQMaAq" }, "source": [ "## **1.** 📦 Install required packages" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "f48c8f8c", "outputId": "a3bac449-590f-44e4-fa36-a6b14ff249c4" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.12/dist-packages (4.13.5)\n", "Requirement already satisfied: pandas in /usr/local/lib/python3.12/dist-packages (2.2.2)\n", "Requirement already satisfied: matplotlib in /usr/local/lib/python3.12/dist-packages (3.10.0)\n", "Requirement already satisfied: seaborn in /usr/local/lib/python3.12/dist-packages (0.13.2)\n", "Requirement already satisfied: numpy in /usr/local/lib/python3.12/dist-packages (2.0.2)\n", "Requirement already satisfied: textblob in /usr/local/lib/python3.12/dist-packages (0.19.0)\n", "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.12/dist-packages (from beautifulsoup4) (2.8.3)\n", "Requirement already satisfied: typing-extensions>=4.0.0 in /usr/local/lib/python3.12/dist-packages (from beautifulsoup4) (4.15.0)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas) (2.9.0.post0)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas) (2025.2)\n", "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas) (2025.3)\n", "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (1.3.3)\n", "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (0.12.1)\n", "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (4.61.1)\n", "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (1.4.9)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (26.0)\n", "Requirement already satisfied: pillow>=8 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (11.3.0)\n", "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (3.3.2)\n", "Requirement already satisfied: nltk>=3.9 in /usr/local/lib/python3.12/dist-packages (from textblob) (3.9.1)\n", "Requirement already satisfied: click in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (8.3.1)\n", "Requirement already satisfied: joblib in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (1.5.3)\n", "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (2025.11.3)\n", "Requirement already satisfied: tqdm in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (4.67.3)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n" ] } ], "source": [ "!pip install beautifulsoup4 pandas matplotlib seaborn numpy textblob" ] }, { "cell_type": "markdown", "metadata": { "id": "lquNYCbfL9IM" }, "source": [ "## **2.** ⛏ Web-scrape all book titles, prices, and ratings from books.toscrape.com" ] }, { "cell_type": "markdown", "metadata": { "id": "0IWuNpxxYDJF" }, "source": [ "### *a. Initial setup*\n", "Define the base url of the website you will scrape as well as how and what you will scrape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "91d52125" }, "outputs": [], "source": [ "import requests\n", "from bs4 import BeautifulSoup\n", "import pandas as pd\n", "import time\n", "\n", "base_url = \"https://books.toscrape.com/catalogue/page-{}.html\"\n", "headers = {\"User-Agent\": \"Mozilla/5.0\"}\n", "\n", "titles, prices, ratings = [], [], []" ] }, { "cell_type": "markdown", "metadata": { "id": "oCdTsin2Yfp3" }, "source": [ "### *b. Fill titles, prices, and ratings from the web pages*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "xqO5Y3dnYhxt", "colab": { "base_uri": "https://localhost:8080/", "height": 176 }, "outputId": "e919f895-36dc-4a58-9e42-1beeda845598" }, "outputs": [ { "output_type": "error", "ename": "KeyboardInterrupt", "evalue": "", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[0;32m/tmp/ipykernel_576/2455262737.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0mratings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"class\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.5\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# polite scraping delay\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "# Loop through all 50 pages\n", "for page in range(1, 51):\n", " url = base_url.format(page)\n", " response = requests.get(url, headers=headers)\n", " soup = BeautifulSoup(response.content, \"html.parser\")\n", " books = soup.find_all(\"article\", class_=\"product_pod\")\n", "\n", " for book in books:\n", " titles.append(book.h3.a[\"title\"])\n", " prices.append(float(book.find(\"p\", class_=\"price_color\").text[1:]))\n", " ratings.append(book.p.get(\"class\")[1])\n", "\n", " time.sleep(0.5) # polite scraping delay" ] }, { "cell_type": "markdown", "metadata": { "id": "T0TOeRC4Yrnn" }, "source": [ "### *c. ✋🏻🛑⛔️ Create a dataframe df_books that contains the now complete \"title\", \"price\", and \"rating\" objects*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "l5FkkNhUYTHh", "colab": { "base_uri": "https://localhost:8080/", "height": 473 }, "outputId": "6e91e4cd-0fd4-47b7-f30f-27764a00c162" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", "RangeIndex: 940 entries, 0 to 939\n", "Data columns (total 3 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 title 940 non-null object \n", " 1 price 940 non-null float64\n", " 2 rating 940 non-null int64 \n", "dtypes: float64(1), int64(1), object(1)\n", "memory usage: 22.2+ KB\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ " price rating\n", "count 940.000000 940.000000\n", "mean 34.898543 2.917021\n", "std 14.571643 1.440894\n", "min 10.000000 1.000000\n", "25% 21.857500 2.000000\n", "50% 35.835000 3.000000\n", "75% 47.682500 4.000000\n", "max 59.990000 5.000000" ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
pricerating
count940.000000940.000000
mean34.8985432.917021
std14.5716431.440894
min10.0000001.000000
25%21.8575002.000000
50%35.8350003.000000
75%47.6825004.000000
max59.9900005.000000
\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "summary": "{\n \"name\": \"df_books\",\n \"rows\": 8,\n \"fields\": [\n {\n \"column\": \"price\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 321.4163817567838,\n \"min\": 10.0,\n \"max\": 940.0,\n \"num_unique_values\": 8,\n \"samples\": [\n 34.8985425531915,\n 35.835,\n 940.0\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"rating\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 331.36506283836815,\n \"min\": 1.0,\n \"max\": 940.0,\n \"num_unique_values\": 8,\n \"samples\": [\n 2.9170212765957446,\n 3.0,\n 940.0\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 13 } ], "source": [ "df_books = pd.DataFrame({\n", " \"title\": titles,\n", " \"price\": prices,\n", " \"rating\": ratings\n", "})\n", "\n", "df_books.head()\n", "\n", "rating_map = {\n", " \"One\": 1,\n", " \"Two\": 2,\n", " \"Three\": 3,\n", " \"Four\": 4,\n", " \"Five\": 5\n", "}\n", "\n", "df_books[\"rating\"] = df_books[\"rating\"].map(rating_map)\n", "\n", "df_books.info()\n", "df_books.describe()" ] }, { "cell_type": "markdown", "metadata": { "id": "duI5dv3CZYvF" }, "source": [ "### *d. Save web-scraped dataframe either as a CSV or Excel file*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "lC1U_YHtZifh" }, "outputs": [], "source": [ "# 💾 Save to CSV\n", "df_books.to_csv(\"books_data.csv\", index=False)\n", "\n", "# 💾 Or save to Excel\n", "# df_books.to_excel(\"books_data.xlsx\", index=False)" ] }, { "cell_type": "markdown", "metadata": { "id": "qMjRKMBQZlJi" }, "source": [ "### *e. ✋🏻🛑⛔️ View first fiew lines*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "O_wIvTxYZqCK", "outputId": "51a8f74c-243f-46a0-856c-f7f4d4e9cb06" }, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ " title price rating\n", "0 A Light in the Attic 51.77 3\n", "1 Tipping the Velvet 53.74 1\n", "2 Soumission 50.10 1\n", "3 Sharp Objects 47.82 4\n", "4 Sapiens: A Brief History of Humankind 54.23 5" ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
titlepricerating
0A Light in the Attic51.773
1Tipping the Velvet53.741
2Soumission50.101
3Sharp Objects47.824
4Sapiens: A Brief History of Humankind54.235
\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "variable_name": "df_books", "summary": "{\n \"name\": \"df_books\",\n \"rows\": 940,\n \"fields\": [\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 939,\n \"samples\": [\n \"Hush, Hush (Hush, Hush #1)\",\n \"The Mindfulness and Acceptance Workbook for Anxiety: A Guide to Breaking Free from Anxiety, Phobias, and Worry Using Acceptance and Commitment Therapy\",\n \"The Wedding Dress\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"price\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 14.571643268466614,\n \"min\": 10.0,\n \"max\": 59.99,\n \"num_unique_values\": 854,\n \"samples\": [\n 56.06,\n 36.91,\n 37.8\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"rating\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 1,\n \"min\": 1,\n \"max\": 5,\n \"num_unique_values\": 5,\n \"samples\": [\n 1,\n 2,\n 4\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 15 } ], "source": [ "df_books.head()" ] }, { "cell_type": "markdown", "metadata": { "id": "p-1Pr2szaqLk" }, "source": [ "## **3.** 🧩 Create a meaningful connection between real & synthetic datasets" ] }, { "cell_type": "markdown", "metadata": { "id": "SIaJUGIpaH4V" }, "source": [ "### *a. Initial setup*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "-gPXGcRPuV_9" }, "outputs": [], "source": [ "import numpy as np\n", "import random\n", "from datetime import datetime\n", "import warnings\n", "\n", "warnings.filterwarnings(\"ignore\")\n", "random.seed(2025)\n", "np.random.seed(2025)" ] }, { "cell_type": "markdown", "metadata": { "id": "pY4yCoIuaQqp" }, "source": [ "### *b. Generate popularity scores based on rating (with some randomness) with a generate_popularity_score function*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "mnd5hdAbaNjz" }, "outputs": [], "source": [ "def generate_popularity_score(rating):\n", " base = {\"One\": 2, \"Two\": 3, \"Three\": 3, \"Four\": 4, \"Five\": 4}.get(rating, 3)\n", " trend_factor = random.choices([-1, 0, 1], weights=[1, 3, 2])[0]\n", " return int(np.clip(base + trend_factor, 1, 5))" ] }, { "cell_type": "markdown", "metadata": { "id": "n4-TaNTFgPak" }, "source": [ "### *c. ✋🏻🛑⛔️ Run the function to create a \"popularity_score\" column from \"rating\"*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "V-G3OCUCgR07", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "12a853c6-7253-4892-ca16-a7e9587fe127" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Columns: Index(['title', 'price', 'rating', 'popularity_score', 'sentiment_label'], dtype='object')\n", "\n", "First 5 rows:\n", " title price rating popularity_score \\\n", "0 A Light in the Attic 51.77 3 3 \n", "1 Tipping the Velvet 53.74 1 1 \n", "2 Soumission 50.10 1 1 \n", "3 Sharp Objects 47.82 4 4 \n", "4 Sapiens: A Brief History of Humankind 54.23 5 4 \n", "\n", " sentiment_label \n", "0 neutral \n", "1 negative \n", "2 negative \n", "3 positive \n", "4 positive \n", "\n", "Distribution of popularity_score:\n", "popularity_score\n", "1 184\n", "2 196\n", "3 174\n", "4 179\n", "5 207\n", "Name: count, dtype: int64\n" ] } ], "source": [ "# --- Imports (if not already imported) ---\n", "import numpy as np\n", "import random\n", "\n", "# --- Reproducibility ---\n", "random.seed(2025)\n", "np.random.seed(2025)\n", "\n", "# --- Ensure column names have no hidden spaces ---\n", "df_books.columns = df_books.columns.str.strip()\n", "\n", "# --- Confirm rating column exists ---\n", "if \"rating\" not in df_books.columns:\n", " raise ValueError(\"Column 'rating' not found in df_books\")\n", "\n", "# --- Define popularity score generator (for numeric ratings 1–5) ---\n", "def generate_popularity_score(rating):\n", " trend_factor = random.choices([-1, 0, 1], weights=[1, 3, 2])[0]\n", " return int(np.clip(rating + trend_factor, 1, 5))\n", "\n", "# --- Create new column ---\n", "df_books[\"popularity_score\"] = df_books[\"rating\"].apply(generate_popularity_score)\n", "\n", "# --- Verify output ---\n", "print(\"Columns:\", df_books.columns)\n", "print(\"\\nFirst 5 rows:\")\n", "print(df_books.head())\n", "\n", "print(\"\\nDistribution of popularity_score:\")\n", "print(df_books[\"popularity_score\"].value_counts().sort_index())" ] }, { "cell_type": "markdown", "metadata": { "id": "HnngRNTgacYt" }, "source": [ "### *d. Decide on the sentiment_label based on the popularity score with a get_sentiment function*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "kUtWmr8maZLZ" }, "outputs": [], "source": [ "def get_sentiment(popularity_score):\n", " if popularity_score <= 2:\n", " return \"negative\"\n", " elif popularity_score == 3:\n", " return \"neutral\"\n", " else:\n", " return \"positive\"" ] }, { "cell_type": "markdown", "metadata": { "id": "HF9F9HIzgT7Z" }, "source": [ "### *e. ✋🏻🛑⛔️ Run the function to create a \"sentiment_label\" column from \"popularity_score\"*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tafQj8_7gYCG", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "80df923c-021f-4957-a774-98c1348d8f19" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Columns: Index(['title', 'price', 'rating', 'popularity_score', 'sentiment_label'], dtype='object')\n", "\n", "First 5 rows:\n", " title price rating popularity_score \\\n", "0 A Light in the Attic 51.77 3 3 \n", "1 Tipping the Velvet 53.74 1 1 \n", "2 Soumission 50.10 1 1 \n", "3 Sharp Objects 47.82 4 4 \n", "4 Sapiens: A Brief History of Humankind 54.23 5 4 \n", "\n", " sentiment_label \n", "0 neutral \n", "1 negative \n", "2 negative \n", "3 positive \n", "4 positive \n", "\n", "Sentiment distribution:\n", "sentiment_label\n", "positive 386\n", "negative 380\n", "neutral 174\n", "Name: count, dtype: int64\n" ] } ], "source": [ "# --- Ensure popularity_score exists ---\n", "if \"popularity_score\" not in df_books.columns:\n", " raise ValueError(\"Column 'popularity_score' not found in df_books\")\n", "\n", "# --- Create sentiment_label column ---\n", "df_books[\"sentiment_label\"] = df_books[\"popularity_score\"].apply(get_sentiment)\n", "\n", "# --- Verify results ---\n", "print(\"Columns:\", df_books.columns)\n", "\n", "print(\"\\nFirst 5 rows:\")\n", "print(df_books.head())\n", "\n", "print(\"\\nSentiment distribution:\")\n", "print(df_books[\"sentiment_label\"].value_counts())" ] }, { "cell_type": "markdown", "metadata": { "id": "T8AdKkmASq9a" }, "source": [ "## **4.** 📈 Generate synthetic book sales data of 18 months" ] }, { "cell_type": "markdown", "metadata": { "id": "OhXbdGD5fH0c" }, "source": [ "### *a. Create a generate_sales_profit function that would generate sales patterns based on sentiment_label (with some randomness)*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "qkVhYPXGbgEn" }, "outputs": [], "source": [ "def generate_sales_profile(sentiment):\n", " months = pd.date_range(end=datetime.today(), periods=18, freq=\"M\")\n", "\n", " if sentiment == \"positive\":\n", " base = random.randint(200, 300)\n", " trend = np.linspace(base, base + random.randint(20, 60), len(months))\n", " elif sentiment == \"negative\":\n", " base = random.randint(20, 80)\n", " trend = np.linspace(base, base - random.randint(10, 30), len(months))\n", " else: # neutral\n", " base = random.randint(80, 160)\n", " trend = np.full(len(months), base + random.randint(-10, 10))\n", "\n", " seasonality = 10 * np.sin(np.linspace(0, 3 * np.pi, len(months)))\n", " noise = np.random.normal(0, 5, len(months))\n", " monthly_sales = np.clip(trend + seasonality + noise, a_min=0, a_max=None).astype(int)\n", "\n", " return list(zip(months.strftime(\"%Y-%m\"), monthly_sales))" ] }, { "cell_type": "markdown", "metadata": { "id": "L2ak1HlcgoTe" }, "source": [ "### *b. Run the function as part of building sales_data*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "SlJ24AUafoDB" }, "outputs": [], "source": [ "sales_data = []\n", "for _, row in df_books.iterrows():\n", " records = generate_sales_profile(row[\"sentiment_label\"])\n", " for month, units in records:\n", " sales_data.append({\n", " \"title\": row[\"title\"],\n", " \"month\": month,\n", " \"units_sold\": units,\n", " \"sentiment_label\": row[\"sentiment_label\"]\n", " })" ] }, { "cell_type": "markdown", "metadata": { "id": "4IXZKcCSgxnq" }, "source": [ "### *c. ✋🏻🛑⛔️ Create a df_sales DataFrame from sales_data*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "wcN6gtiZg-ws", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "18516e9e-edbb-451c-e9d5-f1d7d9fd17ed" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Sales Data Shape: (16920, 4)\n", "\n", "First 5 rows:\n", " title month units_sold sentiment_label\n", "0 A Light in the Attic 2024-09 105 neutral\n", "1 A Light in the Attic 2024-10 103 neutral\n", "2 A Light in the Attic 2024-11 110 neutral\n", "3 A Light in the Attic 2024-12 117 neutral\n", "4 A Light in the Attic 2025-01 109 neutral\n", "\n", "Months per book (should be 18):\n", "title\n", "\"Most Blessed of the Patriarchs\": Thomas Jefferson and the Empire of the Imagination 18\n", "#GIRLBOSS 18\n", "#HigherSelfie: Wake Up Your Life. Free Your Soul. Find Your Tribe. 18\n", "'Salem's Lot 18\n", "(Un)Qualified: How God Uses Broken People to Do Big Things 18\n", "Name: month, dtype: int64\n" ] } ], "source": [ "# --- Required imports ---\n", "import pandas as pd\n", "import numpy as np\n", "import random\n", "from datetime import datetime\n", "\n", "# --- Ensure sentiment_label exists ---\n", "if \"sentiment_label\" not in df_books.columns:\n", " raise ValueError(\"Column 'sentiment_label' not found in df_books\")\n", "\n", "# --- Sales profile generator ---\n", "def generate_sales_profile(sentiment):\n", " months = pd.date_range(end=datetime.today(), periods=18, freq=\"M\")\n", "\n", " if sentiment == \"positive\":\n", " base = random.randint(200, 300)\n", " trend = np.linspace(base, base + random.randint(20, 60), len(months))\n", " elif sentiment == \"negative\":\n", " base = random.randint(20, 80)\n", " trend = np.linspace(base, base - random.randint(10, 30), len(months))\n", " else: # neutral\n", " base = random.randint(80, 160)\n", " trend = np.full(len(months), base + random.randint(-10, 10))\n", "\n", " seasonality = 10 * np.sin(np.linspace(0, 3 * np.pi, len(months)))\n", " noise = np.random.normal(0, 5, len(months))\n", "\n", " monthly_sales = np.clip(trend + seasonality + noise, a_min=0, a_max=None).astype(int)\n", "\n", " return list(zip(months.strftime(\"%Y-%m\"), monthly_sales))\n", "\n", "\n", "# --- Build sales_data list ---\n", "sales_data = []\n", "\n", "for _, row in df_books.iterrows():\n", " records = generate_sales_profile(row[\"sentiment_label\"])\n", " for month, units in records:\n", " sales_data.append({\n", " \"title\": row[\"title\"],\n", " \"month\": month,\n", " \"units_sold\": units,\n", " \"sentiment_label\": row[\"sentiment_label\"]\n", " })\n", "\n", "# --- Convert to DataFrame ---\n", "df_sales = pd.DataFrame(sales_data)\n", "\n", "# --- Verify ---\n", "print(\"Sales Data Shape:\", df_sales.shape)\n", "print(\"\\nFirst 5 rows:\")\n", "print(df_sales.head())\n", "\n", "print(\"\\nMonths per book (should be 18):\")\n", "print(df_sales.groupby(\"title\")[\"month\"].count().head())" ] }, { "cell_type": "markdown", "metadata": { "id": "EhIjz9WohAmZ" }, "source": [ "### *d. Save df_sales as synthetic_sales_data.csv & view first few lines*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "MzbZvLcAhGaH", "outputId": "4d8bd347-9ab8-4b3d-ea27-e9041b5a6f36" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " title month units_sold sentiment_label\n", "0 A Light in the Attic 2024-09 105 neutral\n", "1 A Light in the Attic 2024-10 103 neutral\n", "2 A Light in the Attic 2024-11 110 neutral\n", "3 A Light in the Attic 2024-12 117 neutral\n", "4 A Light in the Attic 2025-01 109 neutral\n" ] } ], "source": [ "df_sales.to_csv(\"synthetic_sales_data.csv\", index=False)\n", "\n", "print(df_sales.head())" ] }, { "cell_type": "markdown", "metadata": { "id": "7g9gqBgQMtJn" }, "source": [ "## **5.** 🎯 Generate synthetic customer reviews" ] }, { "cell_type": "markdown", "metadata": { "id": "Gi4y9M9KuDWx" }, "source": [ "### *a. ✋🏻🛑⛔️ Ask ChatGPT to create a list of 50 distinct generic book review texts for the sentiment labels \"positive\", \"neutral\", and \"negative\" called synthetic_reviews_by_sentiment*" ] }, { "cell_type": "code", "source": [ "import random\n", "import itertools\n", "\n", "random.seed(2025)\n", "\n", "def generate_reviews(sentiment, n=50):\n", " adjectives = {\n", " \"positive\": [\n", " \"excellent\", \"engaging\", \"compelling\", \"insightful\", \"captivating\",\n", " \"thought-provoking\", \"beautifully written\", \"masterfully crafted\",\n", " \"immersive\", \"inspiring\", \"memorable\", \"refreshing\", \"well-paced\"\n", " ],\n", " \"neutral\": [\n", " \"adequate\", \"acceptable\", \"moderate\", \"standard\", \"balanced\",\n", " \"straightforward\", \"predictable\", \"conventional\",\n", " \"reasonably structured\", \"competent\", \"simple\", \"average\", \"steady\"\n", " ],\n", " \"negative\": [\n", " \"disappointing\", \"confusing\", \"poorly written\", \"uninspired\",\n", " \"weak\", \"overly long\", \"underdeveloped\", \"inconsistent\",\n", " \"repetitive\", \"lackluster\", \"flat\", \"messy\", \"forgettable\"\n", " ]\n", " }\n", "\n", " templates = {\n", " \"positive\": [\n", " \"An {adj} read that exceeded expectations.\",\n", " \"A truly {adj} book that I would recommend.\",\n", " \"The story was {adj} and kept me interested throughout.\",\n", " \"Overall, a {adj} experience from beginning to end.\",\n", " \"A {adj} journey with strong moments and satisfying payoff.\"\n", " ],\n", " \"neutral\": [\n", " \"A fairly {adj} book overall.\",\n", " \"The reading experience was {adj} but not remarkable.\",\n", " \"An {adj} story that delivers what it promises.\",\n", " \"Overall, a {adj} and steady read.\",\n", " \"It felt {adj}: fine for passing time, but not standout.\"\n", " ],\n", " \"negative\": [\n", " \"A {adj} book that did not fully deliver.\",\n", " \"The story felt {adj} and difficult to enjoy.\",\n", " \"Overall, a rather {adj} experience.\",\n", " \"An unfortunately {adj} read.\",\n", " \"The pacing was {adj}, which made it hard to stay invested.\"\n", " ]\n", " }\n", "\n", " # Build ALL possible unique combinations, shuffle, then take n\n", " combos = [t.format(adj=a) for t, a in itertools.product(templates[sentiment], adjectives[sentiment])]\n", " random.shuffle(combos)\n", " return combos[:n]\n", "\n", "synthetic_reviews_by_sentiment = {\n", " \"positive\": generate_reviews(\"positive\", 50),\n", " \"neutral\": generate_reviews(\"neutral\", 50),\n", " \"negative\": generate_reviews(\"negative\", 50)\n", "}\n", "\n", "# Quick check\n", "for s, reviews in synthetic_reviews_by_sentiment.items():\n", " print(s, len(reviews), \"unique:\", len(set(reviews)))\n", " print(\"Sample:\", reviews[:3], \"\\n\")" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "NGOUIvP3Yhei", "outputId": "79c0dad9-0255-4e28-fbcd-7f110fcf1e5e" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "positive 50 unique: 50\n", "Sample: ['Overall, a engaging experience from beginning to end.', 'A truly masterfully crafted book that I would recommend.', 'A truly thought-provoking book that I would recommend.'] \n", "\n", "neutral 50 unique: 50\n", "Sample: ['A fairly straightforward book overall.', 'A fairly average book overall.', 'The reading experience was moderate but not remarkable.'] \n", "\n", "negative 50 unique: 50\n", "Sample: ['An unfortunately messy read.', 'An unfortunately overly long read.', 'A underdeveloped book that did not fully deliver.'] \n", "\n" ] } ] }, { "cell_type": "markdown", "metadata": { "id": "fQhfVaDmuULT" }, "source": [ "### *b. Generate 10 reviews per book using random sampling from the corresponding 50*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "l2SRc3PjuTGM" }, "outputs": [], "source": [ "review_rows = []\n", "for _, row in df_books.iterrows():\n", " title = row['title']\n", " sentiment_label = row['sentiment_label']\n", " review_pool = synthetic_reviews_by_sentiment[sentiment_label]\n", " sampled_reviews = random.sample(review_pool, 10)\n", " for review_text in sampled_reviews:\n", " review_rows.append({\n", " \"title\": title,\n", " \"sentiment_label\": sentiment_label,\n", " \"review_text\": review_text,\n", " \"rating\": row['rating'],\n", " \"popularity_score\": row['popularity_score']\n", " })" ] }, { "cell_type": "markdown", "metadata": { "id": "bmJMXF-Bukdm" }, "source": [ "### *c. Create the final dataframe df_reviews & save it as synthetic_book_reviews.csv*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ZUKUqZsuumsp" }, "outputs": [], "source": [ "df_reviews = pd.DataFrame(review_rows)\n", "df_reviews.to_csv(\"synthetic_book_reviews.csv\", index=False)" ] }, { "cell_type": "markdown", "source": [ "### *c. inputs for R*" ], "metadata": { "id": "_602pYUS3gY5" } }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "3946e521", "outputId": "e60ce6f6-3be2-46e4-8c2c-7377064b2f6c" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "✅ Wrote synthetic_title_level_features.csv\n", "✅ Wrote synthetic_monthly_revenue_series.csv\n" ] } ], "source": [ "import numpy as np\n", "\n", "def _safe_num(s):\n", " return pd.to_numeric(\n", " pd.Series(s).astype(str).str.replace(r\"[^0-9.]\", \"\", regex=True),\n", " errors=\"coerce\"\n", " )\n", "\n", "# --- Clean book metadata (price/rating) ---\n", "df_books_r = df_books.copy()\n", "if \"price\" in df_books_r.columns:\n", " df_books_r[\"price\"] = _safe_num(df_books_r[\"price\"])\n", "if \"rating\" in df_books_r.columns:\n", " df_books_r[\"rating\"] = _safe_num(df_books_r[\"rating\"])\n", "\n", "df_books_r[\"title\"] = df_books_r[\"title\"].astype(str).str.strip()\n", "\n", "# --- Clean sales ---\n", "df_sales_r = df_sales.copy()\n", "df_sales_r[\"title\"] = df_sales_r[\"title\"].astype(str).str.strip()\n", "df_sales_r[\"month\"] = pd.to_datetime(df_sales_r[\"month\"], errors=\"coerce\")\n", "df_sales_r[\"units_sold\"] = _safe_num(df_sales_r[\"units_sold\"])\n", "\n", "# --- Clean reviews ---\n", "df_reviews_r = df_reviews.copy()\n", "df_reviews_r[\"title\"] = df_reviews_r[\"title\"].astype(str).str.strip()\n", "df_reviews_r[\"sentiment_label\"] = df_reviews_r[\"sentiment_label\"].astype(str).str.lower().str.strip()\n", "if \"rating\" in df_reviews_r.columns:\n", " df_reviews_r[\"rating\"] = _safe_num(df_reviews_r[\"rating\"])\n", "if \"popularity_score\" in df_reviews_r.columns:\n", " df_reviews_r[\"popularity_score\"] = _safe_num(df_reviews_r[\"popularity_score\"])\n", "\n", "# --- Sentiment shares per title (from reviews) ---\n", "sent_counts = (\n", " df_reviews_r.groupby([\"title\", \"sentiment_label\"])\n", " .size()\n", " .unstack(fill_value=0)\n", ")\n", "for lab in [\"positive\", \"neutral\", \"negative\"]:\n", " if lab not in sent_counts.columns:\n", " sent_counts[lab] = 0\n", "\n", "sent_counts[\"total_reviews\"] = sent_counts[[\"positive\", \"neutral\", \"negative\"]].sum(axis=1)\n", "den = sent_counts[\"total_reviews\"].replace(0, np.nan)\n", "sent_counts[\"share_positive\"] = sent_counts[\"positive\"] / den\n", "sent_counts[\"share_neutral\"] = sent_counts[\"neutral\"] / den\n", "sent_counts[\"share_negative\"] = sent_counts[\"negative\"] / den\n", "sent_counts = sent_counts.reset_index()\n", "\n", "# --- Sales aggregation per title ---\n", "sales_by_title = (\n", " df_sales_r.dropna(subset=[\"title\"])\n", " .groupby(\"title\", as_index=False)\n", " .agg(\n", " months_observed=(\"month\", \"nunique\"),\n", " avg_units_sold=(\"units_sold\", \"mean\"),\n", " total_units_sold=(\"units_sold\", \"sum\"),\n", " )\n", ")\n", "\n", "# --- Title-level features (join sales + books + sentiment) ---\n", "df_title = (\n", " sales_by_title\n", " .merge(df_books_r[[\"title\", \"price\", \"rating\"]], on=\"title\", how=\"left\")\n", " .merge(sent_counts[[\"title\", \"share_positive\", \"share_neutral\", \"share_negative\", \"total_reviews\"]],\n", " on=\"title\", how=\"left\")\n", ")\n", "\n", "df_title[\"avg_revenue\"] = df_title[\"avg_units_sold\"] * df_title[\"price\"]\n", "df_title[\"total_revenue\"] = df_title[\"total_units_sold\"] * df_title[\"price\"]\n", "\n", "df_title.to_csv(\"synthetic_title_level_features.csv\", index=False)\n", "print(\"✅ Wrote synthetic_title_level_features.csv\")\n", "\n", "# --- Monthly revenue series (proxy: units_sold * price) ---\n", "monthly_rev = (\n", " df_sales_r.merge(df_books_r[[\"title\", \"price\"]], on=\"title\", how=\"left\")\n", ")\n", "monthly_rev[\"revenue\"] = monthly_rev[\"units_sold\"] * monthly_rev[\"price\"]\n", "\n", "df_monthly = (\n", " monthly_rev.dropna(subset=[\"month\"])\n", " .groupby(\"month\", as_index=False)[\"revenue\"]\n", " .sum()\n", " .rename(columns={\"revenue\": \"total_revenue\"})\n", " .sort_values(\"month\")\n", ")\n", "# if revenue is all NA (e.g., missing price), fallback to units_sold as a teaching proxy\n", "if df_monthly[\"total_revenue\"].notna().sum() == 0:\n", " df_monthly = (\n", " df_sales_r.dropna(subset=[\"month\"])\n", " .groupby(\"month\", as_index=False)[\"units_sold\"]\n", " .sum()\n", " .rename(columns={\"units_sold\": \"total_revenue\"})\n", " .sort_values(\"month\")\n", " )\n", "\n", "df_monthly[\"month\"] = pd.to_datetime(df_monthly[\"month\"], errors=\"coerce\").dt.strftime(\"%Y-%m-%d\")\n", "df_monthly.to_csv(\"synthetic_monthly_revenue_series.csv\", index=False)\n", "print(\"✅ Wrote synthetic_monthly_revenue_series.csv\")\n" ] }, { "cell_type": "markdown", "metadata": { "id": "RYvGyVfXuo54" }, "source": [ "### *d. ✋🏻🛑⛔️ View the first few lines*" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "xfE8NMqOurKo", "outputId": "191730ba-d5e2-4df7-97d2-99feb0b704af" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " title sentiment_label \\\n", "0 A Light in the Attic neutral \n", "1 A Light in the Attic neutral \n", "2 A Light in the Attic neutral \n", "3 A Light in the Attic neutral \n", "4 A Light in the Attic neutral \n", "\n", " review_text rating popularity_score \n", "0 Had potential that went unrealized. Three 3 \n", "1 The themes were solid, but not well explored. Three 3 \n", "2 It simply lacked that emotional punch. Three 3 \n", "3 Serviceable but not something I'd go out of my... Three 3 \n", "4 Standard fare with some promise. Three 3 \n" ] } ], "source": [] } ], "metadata": { "colab": { "collapsed_sections": [ "jpASMyIQMaAq", "lquNYCbfL9IM", "0IWuNpxxYDJF", "oCdTsin2Yfp3", "T0TOeRC4Yrnn", "duI5dv3CZYvF", "qMjRKMBQZlJi", "p-1Pr2szaqLk", "SIaJUGIpaH4V", "pY4yCoIuaQqp", "n4-TaNTFgPak", "HnngRNTgacYt", "HF9F9HIzgT7Z", "T8AdKkmASq9a", "OhXbdGD5fH0c", "L2ak1HlcgoTe", "4IXZKcCSgxnq", "EhIjz9WohAmZ", "Gi4y9M9KuDWx", "fQhfVaDmuULT", "bmJMXF-Bukdm", "RYvGyVfXuo54" ], "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 0 }