Constance2222 commited on
Commit
0183d19
·
verified ·
1 Parent(s): 2d61df2

Upload 2 files

Browse files
Files changed (2) hide show
  1. datacreation.ipynb +1343 -0
  2. pythonanalysis.ipynb +0 -0
datacreation.ipynb ADDED
@@ -0,0 +1,1343 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "metadata": {
6
+ "id": "4ba6aba8"
7
+ },
8
+ "source": [
9
+ "# 🤖 **Data Collection, Creation, Storage, and Processing**\n"
10
+ ]
11
+ },
12
+ {
13
+ "cell_type": "markdown",
14
+ "metadata": {
15
+ "id": "jpASMyIQMaAq"
16
+ },
17
+ "source": [
18
+ "## **1.** 📦 Install required packages"
19
+ ]
20
+ },
21
+ {
22
+ "cell_type": "code",
23
+ "execution_count": null,
24
+ "metadata": {
25
+ "colab": {
26
+ "base_uri": "https://localhost:8080/"
27
+ },
28
+ "id": "f48c8f8c",
29
+ "outputId": "457541ac-bf99-4803-fe35-142bcbc6b484"
30
+ },
31
+ "outputs": [
32
+ {
33
+ "output_type": "stream",
34
+ "name": "stdout",
35
+ "text": [
36
+ "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.12/dist-packages (4.13.5)\n",
37
+ "Requirement already satisfied: pandas in /usr/local/lib/python3.12/dist-packages (2.2.2)\n",
38
+ "Requirement already satisfied: matplotlib in /usr/local/lib/python3.12/dist-packages (3.10.0)\n",
39
+ "Requirement already satisfied: seaborn in /usr/local/lib/python3.12/dist-packages (0.13.2)\n",
40
+ "Requirement already satisfied: numpy in /usr/local/lib/python3.12/dist-packages (2.0.2)\n",
41
+ "Requirement already satisfied: textblob in /usr/local/lib/python3.12/dist-packages (0.19.0)\n",
42
+ "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.12/dist-packages (from beautifulsoup4) (2.8.3)\n",
43
+ "Requirement already satisfied: typing-extensions>=4.0.0 in /usr/local/lib/python3.12/dist-packages (from beautifulsoup4) (4.15.0)\n",
44
+ "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas) (2.9.0.post0)\n",
45
+ "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas) (2025.2)\n",
46
+ "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas) (2025.3)\n",
47
+ "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (1.3.3)\n",
48
+ "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (0.12.1)\n",
49
+ "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (4.61.1)\n",
50
+ "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (1.4.9)\n",
51
+ "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (26.0)\n",
52
+ "Requirement already satisfied: pillow>=8 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (11.3.0)\n",
53
+ "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (3.3.2)\n",
54
+ "Requirement already satisfied: nltk>=3.9 in /usr/local/lib/python3.12/dist-packages (from textblob) (3.9.1)\n",
55
+ "Requirement already satisfied: click in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (8.3.1)\n",
56
+ "Requirement already satisfied: joblib in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (1.5.3)\n",
57
+ "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (2025.11.3)\n",
58
+ "Requirement already satisfied: tqdm in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (4.67.3)\n",
59
+ "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n"
60
+ ]
61
+ }
62
+ ],
63
+ "source": [
64
+ "!pip install beautifulsoup4 pandas matplotlib seaborn numpy textblob"
65
+ ]
66
+ },
67
+ {
68
+ "cell_type": "markdown",
69
+ "metadata": {
70
+ "id": "lquNYCbfL9IM"
71
+ },
72
+ "source": [
73
+ "## **2.** ⛏ Web-scrape all book titles, prices, and ratings from books.toscrape.com"
74
+ ]
75
+ },
76
+ {
77
+ "cell_type": "markdown",
78
+ "metadata": {
79
+ "id": "0IWuNpxxYDJF"
80
+ },
81
+ "source": [
82
+ "### *a. Initial setup*\n",
83
+ "Define the base url of the website you will scrape as well as how and what you will scrape"
84
+ ]
85
+ },
86
+ {
87
+ "cell_type": "code",
88
+ "execution_count": null,
89
+ "metadata": {
90
+ "id": "91d52125"
91
+ },
92
+ "outputs": [],
93
+ "source": [
94
+ "import requests\n",
95
+ "from bs4 import BeautifulSoup\n",
96
+ "import pandas as pd\n",
97
+ "import time\n",
98
+ "\n",
99
+ "base_url = \"https://books.toscrape.com/catalogue/page-{}.html\"\n",
100
+ "headers = {\"User-Agent\": \"Mozilla/5.0\"}\n",
101
+ "\n",
102
+ "titles, prices, ratings = [], [], []"
103
+ ]
104
+ },
105
+ {
106
+ "cell_type": "markdown",
107
+ "metadata": {
108
+ "id": "oCdTsin2Yfp3"
109
+ },
110
+ "source": [
111
+ "### *b. Fill titles, prices, and ratings from the web pages*"
112
+ ]
113
+ },
114
+ {
115
+ "cell_type": "code",
116
+ "execution_count": null,
117
+ "metadata": {
118
+ "id": "xqO5Y3dnYhxt"
119
+ },
120
+ "outputs": [],
121
+ "source": [
122
+ "# Loop through all 50 pages\n",
123
+ "for page in range(1, 51):\n",
124
+ " url = base_url.format(page)\n",
125
+ " response = requests.get(url, headers=headers)\n",
126
+ " soup = BeautifulSoup(response.content, \"html.parser\")\n",
127
+ " books = soup.find_all(\"article\", class_=\"product_pod\")\n",
128
+ "\n",
129
+ " for book in books:\n",
130
+ " titles.append(book.h3.a[\"title\"])\n",
131
+ " prices.append(float(book.find(\"p\", class_=\"price_color\").text[1:]))\n",
132
+ " ratings.append(book.p.get(\"class\")[1])\n",
133
+ "\n",
134
+ " time.sleep(0.5) # polite scraping delay"
135
+ ]
136
+ },
137
+ {
138
+ "cell_type": "markdown",
139
+ "metadata": {
140
+ "id": "T0TOeRC4Yrnn"
141
+ },
142
+ "source": [
143
+ "### *c. ✋🏻🛑⛔️ Create a dataframe df_books that contains the now complete \"title\", \"price\", and \"rating\" objects*"
144
+ ]
145
+ },
146
+ {
147
+ "cell_type": "code",
148
+ "execution_count": null,
149
+ "metadata": {
150
+ "id": "l5FkkNhUYTHh",
151
+ "colab": {
152
+ "base_uri": "https://localhost:8080/"
153
+ },
154
+ "outputId": "85261ed4-9380-47d6-fa4c-8f29d4584e46"
155
+ },
156
+ "outputs": [
157
+ {
158
+ "output_type": "stream",
159
+ "name": "stdout",
160
+ "text": [
161
+ " title price rating\n",
162
+ "0 A Light in the Attic 51.77 Three\n",
163
+ "1 Tipping the Velvet 53.74 One\n",
164
+ "2 Soumission 50.10 One\n",
165
+ "3 Sharp Objects 47.82 Four\n",
166
+ "4 Sapiens: A Brief History of Humankind 54.23 Five\n"
167
+ ]
168
+ }
169
+ ],
170
+ "source": [
171
+ "# Create DataFrame\n",
172
+ "df_books = pd.DataFrame({\n",
173
+ " \"title\": titles,\n",
174
+ " \"price\": prices,\n",
175
+ " \"rating\": ratings\n",
176
+ "})\n",
177
+ "\n",
178
+ "# Display first few rows\n",
179
+ "print(df_books.head())\n"
180
+ ]
181
+ },
182
+ {
183
+ "cell_type": "markdown",
184
+ "metadata": {
185
+ "id": "duI5dv3CZYvF"
186
+ },
187
+ "source": [
188
+ "### *d. Save web-scraped dataframe either as a CSV or Excel file*"
189
+ ]
190
+ },
191
+ {
192
+ "cell_type": "code",
193
+ "execution_count": null,
194
+ "metadata": {
195
+ "id": "lC1U_YHtZifh"
196
+ },
197
+ "outputs": [],
198
+ "source": [
199
+ "# 💾 Save to CSV\n",
200
+ "df_books.to_csv(\"books_data.csv\", index=False)\n",
201
+ "\n",
202
+ "# 💾 Or save to Excel\n",
203
+ "# df_books.to_excel(\"books_data.xlsx\", index=False)"
204
+ ]
205
+ },
206
+ {
207
+ "cell_type": "markdown",
208
+ "metadata": {
209
+ "id": "qMjRKMBQZlJi"
210
+ },
211
+ "source": [
212
+ "### *e. ✋🏻🛑⛔️ View first fiew lines*"
213
+ ]
214
+ },
215
+ {
216
+ "cell_type": "code",
217
+ "execution_count": null,
218
+ "metadata": {
219
+ "colab": {
220
+ "base_uri": "https://localhost:8080/",
221
+ "height": 204
222
+ },
223
+ "id": "O_wIvTxYZqCK",
224
+ "outputId": "e5a35df9-ae0a-45d4-ba02-64dafc330674"
225
+ },
226
+ "outputs": [
227
+ {
228
+ "output_type": "execute_result",
229
+ "data": {
230
+ "text/plain": [
231
+ " title price rating\n",
232
+ "0 A Light in the Attic 51.77 Three\n",
233
+ "1 Tipping the Velvet 53.74 One\n",
234
+ "2 Soumission 50.10 One\n",
235
+ "3 Sharp Objects 47.82 Four\n",
236
+ "4 Sapiens: A Brief History of Humankind 54.23 Five"
237
+ ],
238
+ "text/html": [
239
+ "\n",
240
+ " <div id=\"df-3d5ac7f5-2143-4dab-8553-a7ecd7fbcb7d\" class=\"colab-df-container\">\n",
241
+ " <div>\n",
242
+ "<style scoped>\n",
243
+ " .dataframe tbody tr th:only-of-type {\n",
244
+ " vertical-align: middle;\n",
245
+ " }\n",
246
+ "\n",
247
+ " .dataframe tbody tr th {\n",
248
+ " vertical-align: top;\n",
249
+ " }\n",
250
+ "\n",
251
+ " .dataframe thead th {\n",
252
+ " text-align: right;\n",
253
+ " }\n",
254
+ "</style>\n",
255
+ "<table border=\"1\" class=\"dataframe\">\n",
256
+ " <thead>\n",
257
+ " <tr style=\"text-align: right;\">\n",
258
+ " <th></th>\n",
259
+ " <th>title</th>\n",
260
+ " <th>price</th>\n",
261
+ " <th>rating</th>\n",
262
+ " </tr>\n",
263
+ " </thead>\n",
264
+ " <tbody>\n",
265
+ " <tr>\n",
266
+ " <th>0</th>\n",
267
+ " <td>A Light in the Attic</td>\n",
268
+ " <td>51.77</td>\n",
269
+ " <td>Three</td>\n",
270
+ " </tr>\n",
271
+ " <tr>\n",
272
+ " <th>1</th>\n",
273
+ " <td>Tipping the Velvet</td>\n",
274
+ " <td>53.74</td>\n",
275
+ " <td>One</td>\n",
276
+ " </tr>\n",
277
+ " <tr>\n",
278
+ " <th>2</th>\n",
279
+ " <td>Soumission</td>\n",
280
+ " <td>50.10</td>\n",
281
+ " <td>One</td>\n",
282
+ " </tr>\n",
283
+ " <tr>\n",
284
+ " <th>3</th>\n",
285
+ " <td>Sharp Objects</td>\n",
286
+ " <td>47.82</td>\n",
287
+ " <td>Four</td>\n",
288
+ " </tr>\n",
289
+ " <tr>\n",
290
+ " <th>4</th>\n",
291
+ " <td>Sapiens: A Brief History of Humankind</td>\n",
292
+ " <td>54.23</td>\n",
293
+ " <td>Five</td>\n",
294
+ " </tr>\n",
295
+ " </tbody>\n",
296
+ "</table>\n",
297
+ "</div>\n",
298
+ " <div class=\"colab-df-buttons\">\n",
299
+ "\n",
300
+ " <div class=\"colab-df-container\">\n",
301
+ " <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3d5ac7f5-2143-4dab-8553-a7ecd7fbcb7d')\"\n",
302
+ " title=\"Convert this dataframe to an interactive table.\"\n",
303
+ " style=\"display:none;\">\n",
304
+ "\n",
305
+ " <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
306
+ " <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
307
+ " </svg>\n",
308
+ " </button>\n",
309
+ "\n",
310
+ " <style>\n",
311
+ " .colab-df-container {\n",
312
+ " display:flex;\n",
313
+ " gap: 12px;\n",
314
+ " }\n",
315
+ "\n",
316
+ " .colab-df-convert {\n",
317
+ " background-color: #E8F0FE;\n",
318
+ " border: none;\n",
319
+ " border-radius: 50%;\n",
320
+ " cursor: pointer;\n",
321
+ " display: none;\n",
322
+ " fill: #1967D2;\n",
323
+ " height: 32px;\n",
324
+ " padding: 0 0 0 0;\n",
325
+ " width: 32px;\n",
326
+ " }\n",
327
+ "\n",
328
+ " .colab-df-convert:hover {\n",
329
+ " background-color: #E2EBFA;\n",
330
+ " box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
331
+ " fill: #174EA6;\n",
332
+ " }\n",
333
+ "\n",
334
+ " .colab-df-buttons div {\n",
335
+ " margin-bottom: 4px;\n",
336
+ " }\n",
337
+ "\n",
338
+ " [theme=dark] .colab-df-convert {\n",
339
+ " background-color: #3B4455;\n",
340
+ " fill: #D2E3FC;\n",
341
+ " }\n",
342
+ "\n",
343
+ " [theme=dark] .colab-df-convert:hover {\n",
344
+ " background-color: #434B5C;\n",
345
+ " box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
346
+ " filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
347
+ " fill: #FFFFFF;\n",
348
+ " }\n",
349
+ " </style>\n",
350
+ "\n",
351
+ " <script>\n",
352
+ " const buttonEl =\n",
353
+ " document.querySelector('#df-3d5ac7f5-2143-4dab-8553-a7ecd7fbcb7d button.colab-df-convert');\n",
354
+ " buttonEl.style.display =\n",
355
+ " google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
356
+ "\n",
357
+ " async function convertToInteractive(key) {\n",
358
+ " const element = document.querySelector('#df-3d5ac7f5-2143-4dab-8553-a7ecd7fbcb7d');\n",
359
+ " const dataTable =\n",
360
+ " await google.colab.kernel.invokeFunction('convertToInteractive',\n",
361
+ " [key], {});\n",
362
+ " if (!dataTable) return;\n",
363
+ "\n",
364
+ " const docLinkHtml = 'Like what you see? Visit the ' +\n",
365
+ " '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
366
+ " + ' to learn more about interactive tables.';\n",
367
+ " element.innerHTML = '';\n",
368
+ " dataTable['output_type'] = 'display_data';\n",
369
+ " await google.colab.output.renderOutput(dataTable, element);\n",
370
+ " const docLink = document.createElement('div');\n",
371
+ " docLink.innerHTML = docLinkHtml;\n",
372
+ " element.appendChild(docLink);\n",
373
+ " }\n",
374
+ " </script>\n",
375
+ " </div>\n",
376
+ "\n",
377
+ "\n",
378
+ " </div>\n",
379
+ " </div>\n"
380
+ ],
381
+ "application/vnd.google.colaboratory.intrinsic+json": {
382
+ "type": "dataframe",
383
+ "variable_name": "df_books",
384
+ "summary": "{\n \"name\": \"df_books\",\n \"rows\": 1000,\n \"fields\": [\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 999,\n \"samples\": [\n \"The Grownup\",\n \"Persepolis: The Story of a Childhood (Persepolis #1-2)\",\n \"Ayumi's Violin\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"price\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 14.446689669952772,\n \"min\": 10.0,\n \"max\": 59.99,\n \"num_unique_values\": 903,\n \"samples\": [\n 19.73,\n 55.65,\n 46.31\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"rating\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 5,\n \"samples\": [\n \"One\",\n \"Two\",\n \"Four\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}"
385
+ }
386
+ },
387
+ "metadata": {},
388
+ "execution_count": 15
389
+ }
390
+ ],
391
+ "source": [
392
+ "df_books.head()"
393
+ ]
394
+ },
395
+ {
396
+ "cell_type": "markdown",
397
+ "metadata": {
398
+ "id": "p-1Pr2szaqLk"
399
+ },
400
+ "source": [
401
+ "## **3.** 🧩 Create a meaningful connection between real & synthetic datasets"
402
+ ]
403
+ },
404
+ {
405
+ "cell_type": "markdown",
406
+ "metadata": {
407
+ "id": "SIaJUGIpaH4V"
408
+ },
409
+ "source": [
410
+ "### *a. Initial setup*"
411
+ ]
412
+ },
413
+ {
414
+ "cell_type": "code",
415
+ "execution_count": null,
416
+ "metadata": {
417
+ "id": "-gPXGcRPuV_9"
418
+ },
419
+ "outputs": [],
420
+ "source": [
421
+ "import numpy as np\n",
422
+ "import random\n",
423
+ "from datetime import datetime\n",
424
+ "import warnings\n",
425
+ "\n",
426
+ "warnings.filterwarnings(\"ignore\")\n",
427
+ "random.seed(2025)\n",
428
+ "np.random.seed(2025)"
429
+ ]
430
+ },
431
+ {
432
+ "cell_type": "markdown",
433
+ "metadata": {
434
+ "id": "pY4yCoIuaQqp"
435
+ },
436
+ "source": [
437
+ "### *b. Generate popularity scores based on rating (with some randomness) with a generate_popularity_score function*"
438
+ ]
439
+ },
440
+ {
441
+ "cell_type": "code",
442
+ "execution_count": null,
443
+ "metadata": {
444
+ "id": "mnd5hdAbaNjz"
445
+ },
446
+ "outputs": [],
447
+ "source": [
448
+ "def generate_popularity_score(rating):\n",
449
+ " base = {\"One\": 2, \"Two\": 3, \"Three\": 3, \"Four\": 4, \"Five\": 4}.get(rating, 3)\n",
450
+ " trend_factor = random.choices([-1, 0, 1], weights=[1, 3, 2])[0]\n",
451
+ " return int(np.clip(base + trend_factor, 1, 5))"
452
+ ]
453
+ },
454
+ {
455
+ "cell_type": "markdown",
456
+ "metadata": {
457
+ "id": "n4-TaNTFgPak"
458
+ },
459
+ "source": [
460
+ "### *c. ✋🏻🛑⛔️ Run the function to create a \"popularity_score\" column from \"rating\"*"
461
+ ]
462
+ },
463
+ {
464
+ "cell_type": "code",
465
+ "execution_count": null,
466
+ "metadata": {
467
+ "id": "V-G3OCUCgR07",
468
+ "colab": {
469
+ "base_uri": "https://localhost:8080/"
470
+ },
471
+ "outputId": "7204adb1-c37a-4126-f53c-7ad2b40a1b8f"
472
+ },
473
+ "outputs": [
474
+ {
475
+ "output_type": "stream",
476
+ "name": "stdout",
477
+ "text": [
478
+ " title price rating popularity_score\n",
479
+ "0 A Light in the Attic 51.77 Three 3\n",
480
+ "1 Tipping the Velvet 53.74 One 2\n",
481
+ "2 Soumission 50.10 One 3\n",
482
+ "3 Sharp Objects 47.82 Four 4\n",
483
+ "4 Sapiens: A Brief History of Humankind 54.23 Five 4\n"
484
+ ]
485
+ }
486
+ ],
487
+ "source": [
488
+ "# Create popularity_score column from rating\n",
489
+ "df_books[\"popularity_score\"] = df_books[\"rating\"].apply(generate_popularity_score)\n",
490
+ "\n",
491
+ "# Display first rows to verify\n",
492
+ "print(df_books.head())\n"
493
+ ]
494
+ },
495
+ {
496
+ "cell_type": "markdown",
497
+ "metadata": {
498
+ "id": "HnngRNTgacYt"
499
+ },
500
+ "source": [
501
+ "### *d. Decide on the sentiment_label based on the popularity score with a get_sentiment function*"
502
+ ]
503
+ },
504
+ {
505
+ "cell_type": "code",
506
+ "execution_count": null,
507
+ "metadata": {
508
+ "id": "kUtWmr8maZLZ"
509
+ },
510
+ "outputs": [],
511
+ "source": [
512
+ "def get_sentiment(popularity_score):\n",
513
+ " if popularity_score <= 2:\n",
514
+ " return \"negative\"\n",
515
+ " elif popularity_score == 3:\n",
516
+ " return \"neutral\"\n",
517
+ " else:\n",
518
+ " return \"positive\""
519
+ ]
520
+ },
521
+ {
522
+ "cell_type": "markdown",
523
+ "metadata": {
524
+ "id": "HF9F9HIzgT7Z"
525
+ },
526
+ "source": [
527
+ "### *e. ✋🏻🛑⛔️ Run the function to create a \"sentiment_label\" column from \"popularity_score\"*"
528
+ ]
529
+ },
530
+ {
531
+ "cell_type": "code",
532
+ "execution_count": null,
533
+ "metadata": {
534
+ "id": "tafQj8_7gYCG",
535
+ "colab": {
536
+ "base_uri": "https://localhost:8080/"
537
+ },
538
+ "outputId": "c3c009fc-c7ee-4a31-b2bb-35c2a64cfddf"
539
+ },
540
+ "outputs": [
541
+ {
542
+ "output_type": "stream",
543
+ "name": "stdout",
544
+ "text": [
545
+ " title price rating popularity_score \\\n",
546
+ "0 A Light in the Attic 51.77 Three 3 \n",
547
+ "1 Tipping the Velvet 53.74 One 2 \n",
548
+ "2 Soumission 50.10 One 3 \n",
549
+ "3 Sharp Objects 47.82 Four 4 \n",
550
+ "4 Sapiens: A Brief History of Humankind 54.23 Five 4 \n",
551
+ "\n",
552
+ " sentiment_label \n",
553
+ "0 neutral \n",
554
+ "1 negative \n",
555
+ "2 neutral \n",
556
+ "3 positive \n",
557
+ "4 positive \n"
558
+ ]
559
+ }
560
+ ],
561
+ "source": [
562
+ "# Create sentiment_label column from popularity_score\n",
563
+ "df_books[\"sentiment_label\"] = df_books[\"popularity_score\"].apply(get_sentiment)\n",
564
+ "\n",
565
+ "# Display first rows to verify\n",
566
+ "print(df_books.head())"
567
+ ]
568
+ },
569
+ {
570
+ "cell_type": "markdown",
571
+ "metadata": {
572
+ "id": "T8AdKkmASq9a"
573
+ },
574
+ "source": [
575
+ "## **4.** 📈 Generate synthetic book sales data of 18 months"
576
+ ]
577
+ },
578
+ {
579
+ "cell_type": "markdown",
580
+ "metadata": {
581
+ "id": "OhXbdGD5fH0c"
582
+ },
583
+ "source": [
584
+ "### *a. Create a generate_sales_profit function that would generate sales patterns based on sentiment_label (with some randomness)*"
585
+ ]
586
+ },
587
+ {
588
+ "cell_type": "code",
589
+ "execution_count": null,
590
+ "metadata": {
591
+ "id": "qkVhYPXGbgEn"
592
+ },
593
+ "outputs": [],
594
+ "source": [
595
+ "def generate_sales_profile(sentiment):\n",
596
+ " months = pd.date_range(end=datetime.today(), periods=18, freq=\"M\")\n",
597
+ "\n",
598
+ " if sentiment == \"positive\":\n",
599
+ " base = random.randint(200, 300)\n",
600
+ " trend = np.linspace(base, base + random.randint(20, 60), len(months))\n",
601
+ " elif sentiment == \"negative\":\n",
602
+ " base = random.randint(20, 80)\n",
603
+ " trend = np.linspace(base, base - random.randint(10, 30), len(months))\n",
604
+ " else: # neutral\n",
605
+ " base = random.randint(80, 160)\n",
606
+ " trend = np.full(len(months), base + random.randint(-10, 10))\n",
607
+ "\n",
608
+ " seasonality = 10 * np.sin(np.linspace(0, 3 * np.pi, len(months)))\n",
609
+ " noise = np.random.normal(0, 5, len(months))\n",
610
+ " monthly_sales = np.clip(trend + seasonality + noise, a_min=0, a_max=None).astype(int)\n",
611
+ "\n",
612
+ " return list(zip(months.strftime(\"%Y-%m\"), monthly_sales))"
613
+ ]
614
+ },
615
+ {
616
+ "cell_type": "markdown",
617
+ "metadata": {
618
+ "id": "L2ak1HlcgoTe"
619
+ },
620
+ "source": [
621
+ "### *b. Run the function as part of building sales_data*"
622
+ ]
623
+ },
624
+ {
625
+ "cell_type": "code",
626
+ "execution_count": null,
627
+ "metadata": {
628
+ "id": "SlJ24AUafoDB"
629
+ },
630
+ "outputs": [],
631
+ "source": [
632
+ "sales_data = []\n",
633
+ "for _, row in df_books.iterrows():\n",
634
+ " records = generate_sales_profile(row[\"sentiment_label\"])\n",
635
+ " for month, units in records:\n",
636
+ " sales_data.append({\n",
637
+ " \"title\": row[\"title\"],\n",
638
+ " \"month\": month,\n",
639
+ " \"units_sold\": units,\n",
640
+ " \"sentiment_label\": row[\"sentiment_label\"]\n",
641
+ " })"
642
+ ]
643
+ },
644
+ {
645
+ "cell_type": "markdown",
646
+ "metadata": {
647
+ "id": "4IXZKcCSgxnq"
648
+ },
649
+ "source": [
650
+ "### *c. ✋🏻🛑⛔️ Create a df_sales DataFrame from sales_data*"
651
+ ]
652
+ },
653
+ {
654
+ "cell_type": "code",
655
+ "execution_count": null,
656
+ "metadata": {
657
+ "id": "wcN6gtiZg-ws",
658
+ "colab": {
659
+ "base_uri": "https://localhost:8080/"
660
+ },
661
+ "outputId": "2209d715-6c17-48cf-8b83-92487127ca35"
662
+ },
663
+ "outputs": [
664
+ {
665
+ "output_type": "stream",
666
+ "name": "stdout",
667
+ "text": [
668
+ " title month units_sold sentiment_label\n",
669
+ "0 A Light in the Attic 2024-09 130 neutral\n",
670
+ "1 A Light in the Attic 2024-10 139 neutral\n",
671
+ "2 A Light in the Attic 2024-11 132 neutral\n",
672
+ "3 A Light in the Attic 2024-12 137 neutral\n",
673
+ "4 A Light in the Attic 2025-01 138 neutral\n"
674
+ ]
675
+ }
676
+ ],
677
+ "source": [
678
+ "# Create df_sales DataFrame\n",
679
+ "df_sales = pd.DataFrame(sales_data)\n",
680
+ "\n",
681
+ "# Display first rows to verify\n",
682
+ "print(df_sales.head())"
683
+ ]
684
+ },
685
+ {
686
+ "cell_type": "markdown",
687
+ "metadata": {
688
+ "id": "EhIjz9WohAmZ"
689
+ },
690
+ "source": [
691
+ "### *d. Save df_sales as synthetic_sales_data.csv & view first few lines*"
692
+ ]
693
+ },
694
+ {
695
+ "cell_type": "code",
696
+ "execution_count": null,
697
+ "metadata": {
698
+ "colab": {
699
+ "base_uri": "https://localhost:8080/"
700
+ },
701
+ "id": "MzbZvLcAhGaH",
702
+ "outputId": "04b2820a-639e-422b-efb8-2a54ed85d89c"
703
+ },
704
+ "outputs": [
705
+ {
706
+ "output_type": "stream",
707
+ "name": "stdout",
708
+ "text": [
709
+ " title month units_sold sentiment_label\n",
710
+ "0 A Light in the Attic 2024-09 130 neutral\n",
711
+ "1 A Light in the Attic 2024-10 139 neutral\n",
712
+ "2 A Light in the Attic 2024-11 132 neutral\n",
713
+ "3 A Light in the Attic 2024-12 137 neutral\n",
714
+ "4 A Light in the Attic 2025-01 138 neutral\n"
715
+ ]
716
+ }
717
+ ],
718
+ "source": [
719
+ "df_sales.to_csv(\"synthetic_sales_data.csv\", index=False)\n",
720
+ "\n",
721
+ "print(df_sales.head())"
722
+ ]
723
+ },
724
+ {
725
+ "cell_type": "markdown",
726
+ "metadata": {
727
+ "id": "7g9gqBgQMtJn"
728
+ },
729
+ "source": [
730
+ "## **5.** 🎯 Generate synthetic customer reviews"
731
+ ]
732
+ },
733
+ {
734
+ "cell_type": "markdown",
735
+ "metadata": {
736
+ "id": "Gi4y9M9KuDWx"
737
+ },
738
+ "source": [
739
+ "### *a. ✋🏻🛑⛔️ Ask ChatGPT to create a list of 50 distinct generic book review texts for the sentiment labels \"positive\", \"neutral\", and \"negative\" called synthetic_reviews_by_sentiment*"
740
+ ]
741
+ },
742
+ {
743
+ "cell_type": "code",
744
+ "execution_count": null,
745
+ "metadata": {
746
+ "id": "b3cd2a50"
747
+ },
748
+ "outputs": [],
749
+ "source": [
750
+ "synthetic_reviews_by_sentiment = {\n",
751
+ " \"positive\": [\n",
752
+ " \"A compelling and heartwarming read that stayed with me long after I finished.\",\n",
753
+ " \"Brilliantly written with unforgettable characters and a gripping storyline.\",\n",
754
+ " \"An inspiring story that was both emotionally rich and beautifully told.\",\n",
755
+ " \"Absolutely loved it — engaging from the first page to the last.\",\n",
756
+ " \"A masterpiece of storytelling with depth and authenticity.\",\n",
757
+ " \"Thought-provoking and wonderfully crafted.\",\n",
758
+ " \"A delightful surprise that exceeded all my expectations.\",\n",
759
+ " \"An uplifting and powerful narrative.\",\n",
760
+ " \"Rich in detail and full of memorable moments.\",\n",
761
+ " \"A captivating journey that I didn’t want to end.\",\n",
762
+ " \"Emotionally resonant and skillfully written.\",\n",
763
+ " \"An immersive experience with vivid world-building.\",\n",
764
+ " \"Highly entertaining and deeply satisfying.\",\n",
765
+ " \"A truly rewarding and unforgettable book.\",\n",
766
+ " \"Compelling characters and a beautifully paced plot.\",\n",
767
+ " \"A fantastic read that I would highly recommend.\",\n",
768
+ " \"Creative, engaging, and full of heart.\",\n",
769
+ " \"An exceptional story told with elegance.\",\n",
770
+ " \"Full of charm and meaningful insights.\",\n",
771
+ " \"A page-turner that kept me hooked.\",\n",
772
+ " \"Incredibly well-written and thoughtfully structured.\",\n",
773
+ " \"A brilliant balance of emotion and action.\",\n",
774
+ " \"Engaging from start to finish.\",\n",
775
+ " \"A beautifully imagined and executed novel.\",\n",
776
+ " \"Remarkably insightful and moving.\",\n",
777
+ " \"An outstanding literary achievement.\",\n",
778
+ " \"Deeply satisfying and emotionally powerful.\",\n",
779
+ " \"A vibrant and compelling story.\",\n",
780
+ " \"Wonderfully developed characters and setting.\",\n",
781
+ " \"An absolute joy to read.\",\n",
782
+ " \"Intriguing, inspiring, and unforgettable.\",\n",
783
+ " \"A strong and confident narrative voice.\",\n",
784
+ " \"A moving story with lasting impact.\",\n",
785
+ " \"Expertly crafted and engaging.\",\n",
786
+ " \"A must-read for fans of the genre.\",\n",
787
+ " \"Heartfelt and beautifully expressed.\",\n",
788
+ " \"Smart, engaging, and emotionally rich.\",\n",
789
+ " \"A creative and immersive adventure.\",\n",
790
+ " \"Thoughtful and brilliantly executed.\",\n",
791
+ " \"A satisfying and well-rounded story.\",\n",
792
+ " \"Powerful themes handled with care.\",\n",
793
+ " \"Engrossing and masterfully written.\",\n",
794
+ " \"A rich and layered narrative.\",\n",
795
+ " \"Truly captivating and inspiring.\",\n",
796
+ " \"An enjoyable and rewarding read.\",\n",
797
+ " \"A standout book that deserves praise.\",\n",
798
+ " \"Fresh, engaging, and compelling.\",\n",
799
+ " \"An emotionally gripping experience.\",\n",
800
+ " \"Well-paced and beautifully detailed.\",\n",
801
+ " \"A remarkable and touching story.\"\n",
802
+ " ],\n",
803
+ " \"neutral\": [\n",
804
+ " \"An average book — not particularly memorable, but not bad either.\",\n",
805
+ " \"Some parts were enjoyable, others less so.\",\n",
806
+ " \"It was okay overall — a fairly standard read.\",\n",
807
+ " \"Decent story, though nothing groundbreaking.\",\n",
808
+ " \"A mixed experience with highs and lows.\",\n",
809
+ " \"Readable, but it didn’t leave a strong impression.\",\n",
810
+ " \"Fairly predictable, though competently written.\",\n",
811
+ " \"An acceptable way to spend a few hours.\",\n",
812
+ " \"Some interesting ideas, but uneven execution.\",\n",
813
+ " \"Neither exciting nor disappointing.\",\n",
814
+ " \"A serviceable story with modest impact.\",\n",
815
+ " \"Moderately engaging, but not outstanding.\",\n",
816
+ " \"It had its moments, though it felt average.\",\n",
817
+ " \"Solid writing, but the plot was familiar.\",\n",
818
+ " \"An alright read with limited surprises.\",\n",
819
+ " \"Pleasant enough, though somewhat forgettable.\",\n",
820
+ " \"Reasonably entertaining but lacked depth.\",\n",
821
+ " \"It met expectations without exceeding them.\",\n",
822
+ " \"A straightforward and simple narrative.\",\n",
823
+ " \"Balanced between interesting and ordinary.\",\n",
824
+ " \"A fairly typical example of the genre.\",\n",
825
+ " \"Engaging in parts, slow in others.\",\n",
826
+ " \"Competent but not particularly exciting.\",\n",
827
+ " \"Some strong scenes mixed with weaker ones.\",\n",
828
+ " \"An easy read that didn’t challenge much.\",\n",
829
+ " \"Predictable yet somewhat enjoyable.\",\n",
830
+ " \"A standard storyline executed adequately.\",\n",
831
+ " \"Neither captivating nor frustrating.\",\n",
832
+ " \"It had potential, though not fully realized.\",\n",
833
+ " \"A neutral reading experience overall.\",\n",
834
+ " \"Fairly consistent but not memorable.\",\n",
835
+ " \"An average plot with steady pacing.\",\n",
836
+ " \"Readable but lacking standout elements.\",\n",
837
+ " \"Moderately satisfying but not impactful.\",\n",
838
+ " \"Fine for casual reading.\",\n",
839
+ " \"Some creative ideas, but uneven delivery.\",\n",
840
+ " \"An ordinary story told competently.\",\n",
841
+ " \"It was fine, just not remarkable.\",\n",
842
+ " \"A decent but unremarkable book.\",\n",
843
+ " \"Balanced but somewhat flat.\",\n",
844
+ " \"An adequate narrative without surprises.\",\n",
845
+ " \"Some enjoyable passages throughout.\",\n",
846
+ " \"A predictable but steady storyline.\",\n",
847
+ " \"Not bad, just not exceptional.\",\n",
848
+ " \"Mildly engaging overall.\",\n",
849
+ " \"An average addition to the genre.\",\n",
850
+ " \"Reasonably structured but not gripping.\",\n",
851
+ " \"It held my attention at times.\",\n",
852
+ " \"A passable and straightforward read.\",\n",
853
+ " \"Acceptable, though not memorable.\"\n",
854
+ " ],\n",
855
+ " \"negative\": [\n",
856
+ " \"I struggled to stay engaged throughout the book.\",\n",
857
+ " \"The plot felt confusing and poorly developed.\",\n",
858
+ " \"Disappointing — it failed to meet expectations.\",\n",
859
+ " \"The characters lacked depth and authenticity.\",\n",
860
+ " \"Difficult to finish due to slow pacing.\",\n",
861
+ " \"The storyline felt disjointed and unclear.\",\n",
862
+ " \"Not as compelling as I had hoped.\",\n",
863
+ " \"Underwhelming and forgettable.\",\n",
864
+ " \"A frustrating reading experience overall.\",\n",
865
+ " \"The writing style didn’t resonate with me.\",\n",
866
+ " \"It lacked originality and direction.\",\n",
867
+ " \"Predictable and uninspired.\",\n",
868
+ " \"The narrative felt forced and unnatural.\",\n",
869
+ " \"I found it hard to connect with the characters.\",\n",
870
+ " \"The ending was unsatisfying.\",\n",
871
+ " \"Overly complicated without purpose.\",\n",
872
+ " \"Flat dialogue and weak character development.\",\n",
873
+ " \"It didn’t hold my interest.\",\n",
874
+ " \"Repetitive and slow-moving.\",\n",
875
+ " \"The plot twists felt unconvincing.\",\n",
876
+ " \"An underdeveloped and confusing storyline.\",\n",
877
+ " \"The pacing made it difficult to enjoy.\",\n",
878
+ " \"Not engaging enough to recommend.\",\n",
879
+ " \"A missed opportunity with little impact.\",\n",
880
+ " \"The writing felt rushed and inconsistent.\",\n",
881
+ " \"Uninspiring and dull overall.\",\n",
882
+ " \"It failed to deliver on its premise.\",\n",
883
+ " \"Weak character arcs and predictable events.\",\n",
884
+ " \"The story lacked cohesion.\",\n",
885
+ " \"I expected much more from this book.\",\n",
886
+ " \"The concept was interesting but poorly executed.\",\n",
887
+ " \"It felt longer than it needed to be.\",\n",
888
+ " \"Hard to follow and emotionally flat.\",\n",
889
+ " \"A disappointing attempt at storytelling.\",\n",
890
+ " \"The themes were not explored deeply.\",\n",
891
+ " \"It lacked tension and engagement.\",\n",
892
+ " \"Unclear motivations and weak dialogue.\",\n",
893
+ " \"The narrative didn’t flow smoothly.\",\n",
894
+ " \"More frustrating than enjoyable.\",\n",
895
+ " \"A bland and forgettable experience.\",\n",
896
+ " \"The plot progression was uneven.\",\n",
897
+ " \"Characters felt one-dimensional.\",\n",
898
+ " \"It didn’t live up to its potential.\",\n",
899
+ " \"Confusing structure and pacing issues.\",\n",
900
+ " \"A tedious and uninspiring read.\",\n",
901
+ " \"The storytelling felt disconnected.\",\n",
902
+ " \"Not immersive or compelling.\",\n",
903
+ " \"The writing lacked clarity.\",\n",
904
+ " \"An overall disappointing book.\",\n",
905
+ " \"It simply didn’t work for me.\"\n",
906
+ " ]\n",
907
+ "}"
908
+ ]
909
+ },
910
+ {
911
+ "cell_type": "markdown",
912
+ "metadata": {
913
+ "id": "fQhfVaDmuULT"
914
+ },
915
+ "source": [
916
+ "### *b. Generate 10 reviews per book using random sampling from the corresponding 50*"
917
+ ]
918
+ },
919
+ {
920
+ "cell_type": "code",
921
+ "execution_count": null,
922
+ "metadata": {
923
+ "id": "l2SRc3PjuTGM"
924
+ },
925
+ "outputs": [],
926
+ "source": [
927
+ "review_rows = []\n",
928
+ "for _, row in df_books.iterrows():\n",
929
+ " title = row['title']\n",
930
+ " sentiment_label = row['sentiment_label']\n",
931
+ " review_pool = synthetic_reviews_by_sentiment[sentiment_label]\n",
932
+ " sampled_reviews = random.sample(review_pool, 10)\n",
933
+ " for review_text in sampled_reviews:\n",
934
+ " review_rows.append({\n",
935
+ " \"title\": title,\n",
936
+ " \"sentiment_label\": sentiment_label,\n",
937
+ " \"review_text\": review_text,\n",
938
+ " \"rating\": row['rating'],\n",
939
+ " \"popularity_score\": row['popularity_score']\n",
940
+ " })"
941
+ ]
942
+ },
943
+ {
944
+ "cell_type": "markdown",
945
+ "metadata": {
946
+ "id": "bmJMXF-Bukdm"
947
+ },
948
+ "source": [
949
+ "### *c. Create the final dataframe df_reviews & save it as synthetic_book_reviews.csv*"
950
+ ]
951
+ },
952
+ {
953
+ "cell_type": "code",
954
+ "execution_count": null,
955
+ "metadata": {
956
+ "id": "ZUKUqZsuumsp"
957
+ },
958
+ "outputs": [],
959
+ "source": [
960
+ "df_reviews = pd.DataFrame(review_rows)\n",
961
+ "df_reviews.to_csv(\"synthetic_book_reviews.csv\", index=False)"
962
+ ]
963
+ },
964
+ {
965
+ "cell_type": "markdown",
966
+ "source": [
967
+ "### *c. inputs for R*"
968
+ ],
969
+ "metadata": {
970
+ "id": "_602pYUS3gY5"
971
+ }
972
+ },
973
+ {
974
+ "cell_type": "code",
975
+ "execution_count": null,
976
+ "metadata": {
977
+ "colab": {
978
+ "base_uri": "https://localhost:8080/"
979
+ },
980
+ "id": "3946e521",
981
+ "outputId": "514d7bef-0488-4933-b03c-953b9e8a7f66"
982
+ },
983
+ "outputs": [
984
+ {
985
+ "output_type": "stream",
986
+ "name": "stdout",
987
+ "text": [
988
+ "✅ Wrote synthetic_title_level_features.csv\n",
989
+ "✅ Wrote synthetic_monthly_revenue_series.csv\n"
990
+ ]
991
+ }
992
+ ],
993
+ "source": [
994
+ "import numpy as np\n",
995
+ "\n",
996
+ "def _safe_num(s):\n",
997
+ " return pd.to_numeric(\n",
998
+ " pd.Series(s).astype(str).str.replace(r\"[^0-9.]\", \"\", regex=True),\n",
999
+ " errors=\"coerce\"\n",
1000
+ " )\n",
1001
+ "\n",
1002
+ "# --- Clean book metadata (price/rating) ---\n",
1003
+ "df_books_r = df_books.copy()\n",
1004
+ "if \"price\" in df_books_r.columns:\n",
1005
+ " df_books_r[\"price\"] = _safe_num(df_books_r[\"price\"])\n",
1006
+ "if \"rating\" in df_books_r.columns:\n",
1007
+ " df_books_r[\"rating\"] = _safe_num(df_books_r[\"rating\"])\n",
1008
+ "\n",
1009
+ "df_books_r[\"title\"] = df_books_r[\"title\"].astype(str).str.strip()\n",
1010
+ "\n",
1011
+ "# --- Clean sales ---\n",
1012
+ "df_sales_r = df_sales.copy()\n",
1013
+ "df_sales_r[\"title\"] = df_sales_r[\"title\"].astype(str).str.strip()\n",
1014
+ "df_sales_r[\"month\"] = pd.to_datetime(df_sales_r[\"month\"], errors=\"coerce\")\n",
1015
+ "df_sales_r[\"units_sold\"] = _safe_num(df_sales_r[\"units_sold\"])\n",
1016
+ "\n",
1017
+ "# --- Clean reviews ---\n",
1018
+ "df_reviews_r = df_reviews.copy()\n",
1019
+ "df_reviews_r[\"title\"] = df_reviews_r[\"title\"].astype(str).str.strip()\n",
1020
+ "df_reviews_r[\"sentiment_label\"] = df_reviews_r[\"sentiment_label\"].astype(str).str.lower().str.strip()\n",
1021
+ "if \"rating\" in df_reviews_r.columns:\n",
1022
+ " df_reviews_r[\"rating\"] = _safe_num(df_reviews_r[\"rating\"])\n",
1023
+ "if \"popularity_score\" in df_reviews_r.columns:\n",
1024
+ " df_reviews_r[\"popularity_score\"] = _safe_num(df_reviews_r[\"popularity_score\"])\n",
1025
+ "\n",
1026
+ "# --- Sentiment shares per title (from reviews) ---\n",
1027
+ "sent_counts = (\n",
1028
+ " df_reviews_r.groupby([\"title\", \"sentiment_label\"])\n",
1029
+ " .size()\n",
1030
+ " .unstack(fill_value=0)\n",
1031
+ ")\n",
1032
+ "for lab in [\"positive\", \"neutral\", \"negative\"]:\n",
1033
+ " if lab not in sent_counts.columns:\n",
1034
+ " sent_counts[lab] = 0\n",
1035
+ "\n",
1036
+ "sent_counts[\"total_reviews\"] = sent_counts[[\"positive\", \"neutral\", \"negative\"]].sum(axis=1)\n",
1037
+ "den = sent_counts[\"total_reviews\"].replace(0, np.nan)\n",
1038
+ "sent_counts[\"share_positive\"] = sent_counts[\"positive\"] / den\n",
1039
+ "sent_counts[\"share_neutral\"] = sent_counts[\"neutral\"] / den\n",
1040
+ "sent_counts[\"share_negative\"] = sent_counts[\"negative\"] / den\n",
1041
+ "sent_counts = sent_counts.reset_index()\n",
1042
+ "\n",
1043
+ "# --- Sales aggregation per title ---\n",
1044
+ "sales_by_title = (\n",
1045
+ " df_sales_r.dropna(subset=[\"title\"])\n",
1046
+ " .groupby(\"title\", as_index=False)\n",
1047
+ " .agg(\n",
1048
+ " months_observed=(\"month\", \"nunique\"),\n",
1049
+ " avg_units_sold=(\"units_sold\", \"mean\"),\n",
1050
+ " total_units_sold=(\"units_sold\", \"sum\"),\n",
1051
+ " )\n",
1052
+ ")\n",
1053
+ "\n",
1054
+ "# --- Title-level features (join sales + books + sentiment) ---\n",
1055
+ "df_title = (\n",
1056
+ " sales_by_title\n",
1057
+ " .merge(df_books_r[[\"title\", \"price\", \"rating\"]], on=\"title\", how=\"left\")\n",
1058
+ " .merge(sent_counts[[\"title\", \"share_positive\", \"share_neutral\", \"share_negative\", \"total_reviews\"]],\n",
1059
+ " on=\"title\", how=\"left\")\n",
1060
+ ")\n",
1061
+ "\n",
1062
+ "df_title[\"avg_revenue\"] = df_title[\"avg_units_sold\"] * df_title[\"price\"]\n",
1063
+ "df_title[\"total_revenue\"] = df_title[\"total_units_sold\"] * df_title[\"price\"]\n",
1064
+ "\n",
1065
+ "df_title.to_csv(\"synthetic_title_level_features.csv\", index=False)\n",
1066
+ "print(\"✅ Wrote synthetic_title_level_features.csv\")\n",
1067
+ "\n",
1068
+ "# --- Monthly revenue series (proxy: units_sold * price) ---\n",
1069
+ "monthly_rev = (\n",
1070
+ " df_sales_r.merge(df_books_r[[\"title\", \"price\"]], on=\"title\", how=\"left\")\n",
1071
+ ")\n",
1072
+ "monthly_rev[\"revenue\"] = monthly_rev[\"units_sold\"] * monthly_rev[\"price\"]\n",
1073
+ "\n",
1074
+ "df_monthly = (\n",
1075
+ " monthly_rev.dropna(subset=[\"month\"])\n",
1076
+ " .groupby(\"month\", as_index=False)[\"revenue\"]\n",
1077
+ " .sum()\n",
1078
+ " .rename(columns={\"revenue\": \"total_revenue\"})\n",
1079
+ " .sort_values(\"month\")\n",
1080
+ ")\n",
1081
+ "# if revenue is all NA (e.g., missing price), fallback to units_sold as a teaching proxy\n",
1082
+ "if df_monthly[\"total_revenue\"].notna().sum() == 0:\n",
1083
+ " df_monthly = (\n",
1084
+ " df_sales_r.dropna(subset=[\"month\"])\n",
1085
+ " .groupby(\"month\", as_index=False)[\"units_sold\"]\n",
1086
+ " .sum()\n",
1087
+ " .rename(columns={\"units_sold\": \"total_revenue\"})\n",
1088
+ " .sort_values(\"month\")\n",
1089
+ " )\n",
1090
+ "\n",
1091
+ "df_monthly[\"month\"] = pd.to_datetime(df_monthly[\"month\"], errors=\"coerce\").dt.strftime(\"%Y-%m-%d\")\n",
1092
+ "df_monthly.to_csv(\"synthetic_monthly_revenue_series.csv\", index=False)\n",
1093
+ "print(\"✅ Wrote synthetic_monthly_revenue_series.csv\")\n"
1094
+ ]
1095
+ },
1096
+ {
1097
+ "cell_type": "markdown",
1098
+ "metadata": {
1099
+ "id": "RYvGyVfXuo54"
1100
+ },
1101
+ "source": [
1102
+ "### *d. ✋🏻🛑⛔️ View the first few lines*"
1103
+ ]
1104
+ },
1105
+ {
1106
+ "cell_type": "code",
1107
+ "execution_count": null,
1108
+ "metadata": {
1109
+ "colab": {
1110
+ "base_uri": "https://localhost:8080/",
1111
+ "height": 289
1112
+ },
1113
+ "id": "xfE8NMqOurKo",
1114
+ "outputId": "7415ff40-a5d2-42ed-f763-975b3abceff9"
1115
+ },
1116
+ "outputs": [
1117
+ {
1118
+ "output_type": "execute_result",
1119
+ "data": {
1120
+ "text/plain": [
1121
+ " title sentiment_label \\\n",
1122
+ "0 A Light in the Attic neutral \n",
1123
+ "1 A Light in the Attic neutral \n",
1124
+ "2 A Light in the Attic neutral \n",
1125
+ "3 A Light in the Attic neutral \n",
1126
+ "4 A Light in the Attic neutral \n",
1127
+ "\n",
1128
+ " review_text rating popularity_score \n",
1129
+ "0 Readable, but it didn’t leave a strong impress... Three 3 \n",
1130
+ "1 Fine for casual reading. Three 3 \n",
1131
+ "2 An alright read with limited surprises. Three 3 \n",
1132
+ "3 Some strong scenes mixed with weaker ones. Three 3 \n",
1133
+ "4 An average book — not particularly memorable, ... Three 3 "
1134
+ ],
1135
+ "text/html": [
1136
+ "\n",
1137
+ " <div id=\"df-69a79cc9-9362-484c-ae80-78e414306d48\" class=\"colab-df-container\">\n",
1138
+ " <div>\n",
1139
+ "<style scoped>\n",
1140
+ " .dataframe tbody tr th:only-of-type {\n",
1141
+ " vertical-align: middle;\n",
1142
+ " }\n",
1143
+ "\n",
1144
+ " .dataframe tbody tr th {\n",
1145
+ " vertical-align: top;\n",
1146
+ " }\n",
1147
+ "\n",
1148
+ " .dataframe thead th {\n",
1149
+ " text-align: right;\n",
1150
+ " }\n",
1151
+ "</style>\n",
1152
+ "<table border=\"1\" class=\"dataframe\">\n",
1153
+ " <thead>\n",
1154
+ " <tr style=\"text-align: right;\">\n",
1155
+ " <th></th>\n",
1156
+ " <th>title</th>\n",
1157
+ " <th>sentiment_label</th>\n",
1158
+ " <th>review_text</th>\n",
1159
+ " <th>rating</th>\n",
1160
+ " <th>popularity_score</th>\n",
1161
+ " </tr>\n",
1162
+ " </thead>\n",
1163
+ " <tbody>\n",
1164
+ " <tr>\n",
1165
+ " <th>0</th>\n",
1166
+ " <td>A Light in the Attic</td>\n",
1167
+ " <td>neutral</td>\n",
1168
+ " <td>Readable, but it didn’t leave a strong impress...</td>\n",
1169
+ " <td>Three</td>\n",
1170
+ " <td>3</td>\n",
1171
+ " </tr>\n",
1172
+ " <tr>\n",
1173
+ " <th>1</th>\n",
1174
+ " <td>A Light in the Attic</td>\n",
1175
+ " <td>neutral</td>\n",
1176
+ " <td>Fine for casual reading.</td>\n",
1177
+ " <td>Three</td>\n",
1178
+ " <td>3</td>\n",
1179
+ " </tr>\n",
1180
+ " <tr>\n",
1181
+ " <th>2</th>\n",
1182
+ " <td>A Light in the Attic</td>\n",
1183
+ " <td>neutral</td>\n",
1184
+ " <td>An alright read with limited surprises.</td>\n",
1185
+ " <td>Three</td>\n",
1186
+ " <td>3</td>\n",
1187
+ " </tr>\n",
1188
+ " <tr>\n",
1189
+ " <th>3</th>\n",
1190
+ " <td>A Light in the Attic</td>\n",
1191
+ " <td>neutral</td>\n",
1192
+ " <td>Some strong scenes mixed with weaker ones.</td>\n",
1193
+ " <td>Three</td>\n",
1194
+ " <td>3</td>\n",
1195
+ " </tr>\n",
1196
+ " <tr>\n",
1197
+ " <th>4</th>\n",
1198
+ " <td>A Light in the Attic</td>\n",
1199
+ " <td>neutral</td>\n",
1200
+ " <td>An average book — not particularly memorable, ...</td>\n",
1201
+ " <td>Three</td>\n",
1202
+ " <td>3</td>\n",
1203
+ " </tr>\n",
1204
+ " </tbody>\n",
1205
+ "</table>\n",
1206
+ "</div>\n",
1207
+ " <div class=\"colab-df-buttons\">\n",
1208
+ "\n",
1209
+ " <div class=\"colab-df-container\">\n",
1210
+ " <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-69a79cc9-9362-484c-ae80-78e414306d48')\"\n",
1211
+ " title=\"Convert this dataframe to an interactive table.\"\n",
1212
+ " style=\"display:none;\">\n",
1213
+ "\n",
1214
+ " <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
1215
+ " <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
1216
+ " </svg>\n",
1217
+ " </button>\n",
1218
+ "\n",
1219
+ " <style>\n",
1220
+ " .colab-df-container {\n",
1221
+ " display:flex;\n",
1222
+ " gap: 12px;\n",
1223
+ " }\n",
1224
+ "\n",
1225
+ " .colab-df-convert {\n",
1226
+ " background-color: #E8F0FE;\n",
1227
+ " border: none;\n",
1228
+ " border-radius: 50%;\n",
1229
+ " cursor: pointer;\n",
1230
+ " display: none;\n",
1231
+ " fill: #1967D2;\n",
1232
+ " height: 32px;\n",
1233
+ " padding: 0 0 0 0;\n",
1234
+ " width: 32px;\n",
1235
+ " }\n",
1236
+ "\n",
1237
+ " .colab-df-convert:hover {\n",
1238
+ " background-color: #E2EBFA;\n",
1239
+ " box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
1240
+ " fill: #174EA6;\n",
1241
+ " }\n",
1242
+ "\n",
1243
+ " .colab-df-buttons div {\n",
1244
+ " margin-bottom: 4px;\n",
1245
+ " }\n",
1246
+ "\n",
1247
+ " [theme=dark] .colab-df-convert {\n",
1248
+ " background-color: #3B4455;\n",
1249
+ " fill: #D2E3FC;\n",
1250
+ " }\n",
1251
+ "\n",
1252
+ " [theme=dark] .colab-df-convert:hover {\n",
1253
+ " background-color: #434B5C;\n",
1254
+ " box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
1255
+ " filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
1256
+ " fill: #FFFFFF;\n",
1257
+ " }\n",
1258
+ " </style>\n",
1259
+ "\n",
1260
+ " <script>\n",
1261
+ " const buttonEl =\n",
1262
+ " document.querySelector('#df-69a79cc9-9362-484c-ae80-78e414306d48 button.colab-df-convert');\n",
1263
+ " buttonEl.style.display =\n",
1264
+ " google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
1265
+ "\n",
1266
+ " async function convertToInteractive(key) {\n",
1267
+ " const element = document.querySelector('#df-69a79cc9-9362-484c-ae80-78e414306d48');\n",
1268
+ " const dataTable =\n",
1269
+ " await google.colab.kernel.invokeFunction('convertToInteractive',\n",
1270
+ " [key], {});\n",
1271
+ " if (!dataTable) return;\n",
1272
+ "\n",
1273
+ " const docLinkHtml = 'Like what you see? Visit the ' +\n",
1274
+ " '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
1275
+ " + ' to learn more about interactive tables.';\n",
1276
+ " element.innerHTML = '';\n",
1277
+ " dataTable['output_type'] = 'display_data';\n",
1278
+ " await google.colab.output.renderOutput(dataTable, element);\n",
1279
+ " const docLink = document.createElement('div');\n",
1280
+ " docLink.innerHTML = docLinkHtml;\n",
1281
+ " element.appendChild(docLink);\n",
1282
+ " }\n",
1283
+ " </script>\n",
1284
+ " </div>\n",
1285
+ "\n",
1286
+ "\n",
1287
+ " </div>\n",
1288
+ " </div>\n"
1289
+ ],
1290
+ "application/vnd.google.colaboratory.intrinsic+json": {
1291
+ "type": "dataframe",
1292
+ "variable_name": "df_reviews",
1293
+ "summary": "{\n \"name\": \"df_reviews\",\n \"rows\": 10000,\n \"fields\": [\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 999,\n \"samples\": [\n \"The Grownup\",\n \"Persepolis: The Story of a Childhood (Persepolis #1-2)\",\n \"Ayumi's Violin\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"sentiment_label\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"neutral\",\n \"negative\",\n \"positive\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"review_text\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 150,\n \"samples\": [\n \"Difficult to finish due to slow pacing.\",\n \"The storyline felt disjointed and unclear.\",\n \"The writing style didn\\u2019t resonate with me.\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"rating\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 5,\n \"samples\": [\n \"One\",\n \"Two\",\n \"Four\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"popularity_score\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 1,\n \"min\": 1,\n \"max\": 5,\n \"num_unique_values\": 5,\n \"samples\": [\n 2,\n 1,\n 4\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}"
1294
+ }
1295
+ },
1296
+ "metadata": {},
1297
+ "execution_count": 34
1298
+ }
1299
+ ],
1300
+ "source": [
1301
+ "df_reviews.head()\n"
1302
+ ]
1303
+ }
1304
+ ],
1305
+ "metadata": {
1306
+ "colab": {
1307
+ "collapsed_sections": [
1308
+ "jpASMyIQMaAq",
1309
+ "lquNYCbfL9IM",
1310
+ "0IWuNpxxYDJF",
1311
+ "oCdTsin2Yfp3",
1312
+ "T0TOeRC4Yrnn",
1313
+ "duI5dv3CZYvF",
1314
+ "qMjRKMBQZlJi",
1315
+ "p-1Pr2szaqLk",
1316
+ "SIaJUGIpaH4V",
1317
+ "pY4yCoIuaQqp",
1318
+ "n4-TaNTFgPak",
1319
+ "HnngRNTgacYt",
1320
+ "HF9F9HIzgT7Z",
1321
+ "T8AdKkmASq9a",
1322
+ "OhXbdGD5fH0c",
1323
+ "L2ak1HlcgoTe",
1324
+ "4IXZKcCSgxnq",
1325
+ "EhIjz9WohAmZ",
1326
+ "Gi4y9M9KuDWx",
1327
+ "fQhfVaDmuULT",
1328
+ "bmJMXF-Bukdm",
1329
+ "RYvGyVfXuo54"
1330
+ ],
1331
+ "provenance": []
1332
+ },
1333
+ "kernelspec": {
1334
+ "display_name": "Python 3",
1335
+ "name": "python3"
1336
+ },
1337
+ "language_info": {
1338
+ "name": "python"
1339
+ }
1340
+ },
1341
+ "nbformat": 4,
1342
+ "nbformat_minor": 0
1343
+ }
pythonanalysis.ipynb ADDED
The diff for this file is too large to render. See raw diff