{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import spotipy\n",
    "import spotipy.oauth2 as oauth2\n",
    "from spotipy.oauth2 import SpotifyOAuth,SpotifyClientCredentials\n",
    "import yaml\n",
    "import re\n",
    "from tqdm import tqdm\n",
    "import multiprocessing as mp\n",
    "import time\n",
    "import random\n",
    "import datetime\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "stream= open(\"spotify/spotify2.yaml\")\n",
    "spotify_details = yaml.safe_load(stream)\n",
    "auth_manager = SpotifyClientCredentials(client_id=spotify_details['Client_id'],\n",
    "                                        client_secret=spotify_details['client_secret'])\n",
    "sp = spotipy.client.Spotify(auth_manager=auth_manager)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('Data/1mV3.csv')\n",
    "artist_features=pd.read_csv('Data/artist_features.csv')\n",
    "audio_features=pd.read_csv('Data/audio_features.csv')\n",
    "track_features=pd.read_csv('Data/track_features.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merging all dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(df,audio_features, left_on = \"track_uri\", right_on= \"id\",how = 'outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(df,track_features, left_on = \"track_uri\", right_on= \"Track_uri\",how = 'outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(df,artist_features, left_on = \"artist_uri\", right_on= \"Artist_uri\",how = 'outer')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Handling missing data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "track_uri               0\n",
       "artist_uri              0\n",
       "album_uri               0\n",
       "danceability          101\n",
       "energy                101\n",
       "key                   101\n",
       "loudness              101\n",
       "mode                  101\n",
       "speechiness           101\n",
       "acousticness          101\n",
       "instrumentalness      101\n",
       "liveness              101\n",
       "valence               101\n",
       "tempo                 101\n",
       "type                  101\n",
       "id                    101\n",
       "uri                   101\n",
       "track_href            101\n",
       "analysis_url          101\n",
       "duration_ms           101\n",
       "time_signature        101\n",
       "Track_uri             576\n",
       "Track_release_date    576\n",
       "Track_pop             576\n",
       "Artist_uri              0\n",
       "Artist_pop              0\n",
       "Artist_genres           0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handling audio_features missing From extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "missing_t_uri=df.track_uri[df.id.isna()]\n",
    "missing_t_uri=missing_t_uri.unique()\n",
    "random.shuffle(missing_t_uri)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('data/audio_features.csv','a')\n",
    "for i in tqdm(range(0,len(missing_t_uri),1)):\n",
    "    try:\n",
    "     track_feature = sp.audio_features(missing_t_uri[i:i+1])\n",
    "     track_df = pd.DataFrame(track_feature)\n",
    "     csv_data = track_df.to_csv(header=False,index=False)\n",
    "     f.write(csv_data)\n",
    "    except Exception as e:\n",
    "        r = open(\"extract_log0.txt\", \"a\")\n",
    "        r.write(datetime.datetime.now().strftime(\"%d.%b %Y %H:%M:%S\")+\": \"+str(e)+'\\n')\n",
    "        r.close()\n",
    "        time.sleep(1)\n",
    "        continue\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handling track_features missing From extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "missing_t_uri=df.track_uri[df.Track_uri.isna()]\n",
    "missing_t_uri=missing_t_uri.unique()\n",
    "random.shuffle(missing_t_uri)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('data/track_features.csv','a')\n",
    "for i in tqdm(range(0,len(missing_t_uri),1)):\n",
    "    try:\n",
    "        track_features = sp.tracks(missing_t_uri[i:i+1])\n",
    "        for x in range(1):\n",
    "            track_pop=pd.DataFrame([missing_t_uri[i+x]])\n",
    "            track_pop['release_date']=track_features['tracks'][x]['album']['release_date']\n",
    "            track_pop['pop'] = track_features['tracks'][x][\"popularity\"]\n",
    "            csv_data = track_pop.to_csv(header=False,index=False)\n",
    "            f.write(csv_data)\n",
    "    except Exception as e:\n",
    "        r = open(\"extract_log.txt\", \"a\")\n",
    "        r.write(datetime.datetime.now().strftime(\"%d.%b %Y %H:%M:%S\")+\": \"+str(e)+'\\n')\n",
    "        r.close()\n",
    "        time.sleep(1)\n",
    "        continue\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Droping Unwanted Columns Save Space"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There were still 101 from audio_features and 576 from track_features extraction that were missing from the soptify api, so I had to drop them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(axis=0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isna().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['track_uri', 'artist_uri', 'album_uri', 'danceability', 'energy', 'key',\n",
       "       'loudness', 'mode', 'speechiness', 'acousticness', 'instrumentalness',\n",
       "       'liveness', 'valence', 'tempo', 'type', 'id', 'uri', 'track_href',\n",
       "       'analysis_url', 'duration_ms', 'time_signature', 'Track_uri',\n",
       "       'Track_release_date', 'Track_pop', 'Artist_uri', 'Artist_pop',\n",
       "       'Artist_genres'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(columns=['Track_uri','Artist_uri','type','id','uri','track_href','analysis_url'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>track_uri</th>\n",
       "      <th>artist_uri</th>\n",
       "      <th>album_uri</th>\n",
       "      <th>danceability</th>\n",
       "      <th>energy</th>\n",
       "      <th>key</th>\n",
       "      <th>loudness</th>\n",
       "      <th>mode</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>acousticness</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>liveness</th>\n",
       "      <th>valence</th>\n",
       "      <th>tempo</th>\n",
       "      <th>duration_ms</th>\n",
       "      <th>time_signature</th>\n",
       "      <th>Track_release_date</th>\n",
       "      <th>Track_pop</th>\n",
       "      <th>Artist_pop</th>\n",
       "      <th>Artist_genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0UaMYEvWZi0ZqiDOoHU3YI</td>\n",
       "      <td>2wIVse2owClT7go1WT98tk</td>\n",
       "      <td>6vV5UrXcfyQD1wu4Qo2I9K</td>\n",
       "      <td>0.904</td>\n",
       "      <td>0.813</td>\n",
       "      <td>4.0</td>\n",
       "      <td>-7.105</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.121</td>\n",
       "      <td>0.0311</td>\n",
       "      <td>0.00697</td>\n",
       "      <td>0.0471</td>\n",
       "      <td>0.81</td>\n",
       "      <td>125.461</td>\n",
       "      <td>226864.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2005-07-04</td>\n",
       "      <td>67.0</td>\n",
       "      <td>71</td>\n",
       "      <td>dance_pop hip_hop hip_pop pop_rap r&amp;b rap urba...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                track_uri              artist_uri               album_uri  \\\n",
       "0  0UaMYEvWZi0ZqiDOoHU3YI  2wIVse2owClT7go1WT98tk  6vV5UrXcfyQD1wu4Qo2I9K   \n",
       "\n",
       "   danceability  energy  key  loudness  mode  speechiness  acousticness  \\\n",
       "0         0.904   0.813  4.0    -7.105   0.0        0.121        0.0311   \n",
       "\n",
       "   instrumentalness  liveness  valence    tempo  duration_ms  time_signature  \\\n",
       "0           0.00697    0.0471     0.81  125.461     226864.0             4.0   \n",
       "\n",
       "  Track_release_date  Track_pop  Artist_pop  \\\n",
       "0         2005-07-04       67.0          71   \n",
       "\n",
       "                                       Artist_genres  \n",
       "0  dance_pop hip_hop hip_pop pop_rap r&b rap urba...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create five point buckets for track and artist popularity .\n",
    "\n",
    "and 50 point buckets for the track release date."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Track_pop'] = df['Track_pop'].apply(lambda x: int(x/5))\n",
    "df['Artist_pop'] = df['Artist_pop'].apply(lambda x: int(x/5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Track_release_date'] = df['Track_release_date'].apply(lambda x: x.split('-')[0])\n",
    "df['Track_release_date']=df['Track_release_date'].astype('int16')\n",
    "df['Track_release_date'] = df['Track_release_date'].apply(lambda x: int(x/50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>track_uri</th>\n",
       "      <th>artist_uri</th>\n",
       "      <th>album_uri</th>\n",
       "      <th>danceability</th>\n",
       "      <th>energy</th>\n",
       "      <th>key</th>\n",
       "      <th>loudness</th>\n",
       "      <th>mode</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>acousticness</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>liveness</th>\n",
       "      <th>valence</th>\n",
       "      <th>tempo</th>\n",
       "      <th>duration_ms</th>\n",
       "      <th>time_signature</th>\n",
       "      <th>Track_release_date</th>\n",
       "      <th>Track_pop</th>\n",
       "      <th>Artist_pop</th>\n",
       "      <th>Artist_genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0UaMYEvWZi0ZqiDOoHU3YI</td>\n",
       "      <td>2wIVse2owClT7go1WT98tk</td>\n",
       "      <td>6vV5UrXcfyQD1wu4Qo2I9K</td>\n",
       "      <td>0.904</td>\n",
       "      <td>0.813</td>\n",
       "      <td>4.0</td>\n",
       "      <td>-7.105</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.121</td>\n",
       "      <td>0.0311</td>\n",
       "      <td>0.00697</td>\n",
       "      <td>0.0471</td>\n",
       "      <td>0.81</td>\n",
       "      <td>125.461</td>\n",
       "      <td>226864.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>dance_pop hip_hop hip_pop pop_rap r&amp;b rap urba...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                track_uri              artist_uri               album_uri  \\\n",
       "0  0UaMYEvWZi0ZqiDOoHU3YI  2wIVse2owClT7go1WT98tk  6vV5UrXcfyQD1wu4Qo2I9K   \n",
       "\n",
       "   danceability  energy  key  loudness  mode  speechiness  acousticness  \\\n",
       "0         0.904   0.813  4.0    -7.105   0.0        0.121        0.0311   \n",
       "\n",
       "   instrumentalness  liveness  valence    tempo  duration_ms  time_signature  \\\n",
       "0           0.00697    0.0471     0.81  125.461     226864.0             4.0   \n",
       "\n",
       "   Track_release_date  Track_pop  Artist_pop  \\\n",
       "0                  40         13          14   \n",
       "\n",
       "                                       Artist_genres  \n",
       "0  dance_pop hip_hop hip_pop pop_rap r&b rap urba...  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/1M_unique_processed_data.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "e246d2215c418239c9316a1ebf2d8abb44dc50b2e5b0e29defd87143398aa387"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}