diff --git "a/recommendation_module_project/model_(2).ipynb" "b/recommendation_module_project/model_(2).ipynb"
new file mode 100644--- /dev/null
+++ "b/recommendation_module_project/model_(2).ipynb"
@@ -0,0 +1,936 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "id": "uq9k8YYUKjnp"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import urllib.request\n",
+ "import zipfile\n",
+ "import json\n",
+ "import pandas as pd\n",
+ "import time\n",
+ "import torch\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import torch.nn as nn\n",
+ "import torch.nn.functional as F\n",
+ "import torch.optim as optim\n",
+ "from torch.utils.data import DataLoader, TensorDataset\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn.preprocessing import LabelEncoder"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "y1pGv3um_VAV",
+ "outputId": "64ee7998-f542-4477-a6c3-7444a04a42c8"
+ },
+ "outputs": [],
+ "source": [
+ "# from google.colab import drive\n",
+ "# drive.mount('/content/drive')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {
+ "id": "MrmOyqSn_Y7C"
+ },
+ "outputs": [],
+ "source": [
+ "# prompt: copy a file from another directory to current directory in python code and create folders if needed\n",
+ "\n",
+ "import shutil\n",
+ "import os\n",
+ "\n",
+ "def copy_file(src, dst):\n",
+ " \"\"\"\n",
+ " Copies a file from src to dst, creating any necessary directories.\n",
+ "\n",
+ " Args:\n",
+ " src: The path to the source file.\n",
+ " dst: The path to the destination file.\n",
+ " \"\"\"\n",
+ " # Create the destination directory if it doesn't exist.\n",
+ " dst_dir = os.path.dirname(dst)\n",
+ " if not os.path.exists(dst_dir):\n",
+ " os.makedirs(dst_dir)\n",
+ "\n",
+ " # Copy the file.\n",
+ " shutil.copy2(src, dst)\n",
+ "\n",
+ "# copy_file('/content/drive/MyDrive/rec_data/spotify_million_playlist_dataset.zip', os.getcwd() + '/data/raw/spotify_million_playlist_dataset.zip')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "id": "L5h3Tsa0LIoo"
+ },
+ "outputs": [],
+ "source": [
+ "def unzip_archive(filepath, dir_path):\n",
+ " with zipfile.ZipFile(f\"{filepath}\", 'r') as zip_ref:\n",
+ " zip_ref.extractall(dir_path)\n",
+ "\n",
+ "unzip_archive(os.getcwd() + '/data/raw/spotify_million_playlist_dataset.zip', os.getcwd() + '/data/raw/playlists')\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {
+ "id": "JcLT9U2Q_LJw"
+ },
+ "outputs": [],
+ "source": [
+ "import shutil\n",
+ "\n",
+ "def make_dir(directory):\n",
+ " if os.path.exists(directory):\n",
+ " shutil.rmtree(directory)\n",
+ " os.makedirs(directory)\n",
+ " else:\n",
+ " os.makedirs(directory)\n",
+ "\n",
+ "directory = os.getcwd() + '/data/raw/data'\n",
+ "make_dir(directory)\n",
+ "directory = os.getcwd() + '/data/processed'\n",
+ "make_dir(directory)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {
+ "id": "fC-0iP1L_LJx"
+ },
+ "outputs": [],
+ "source": [
+ "cols = [\n",
+ " 'name',\n",
+ " 'pid',\n",
+ " 'num_followers',\n",
+ " 'pos',\n",
+ " 'artist_name',\n",
+ " 'track_name',\n",
+ " 'album_name'\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "qyCujIu8cDGg",
+ "outputId": "f3b21394-acbc-40ab-d70a-b666acd985e7"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "mpd.slice.278000-278999.json\t200/1000\t20.0%"
+ ]
+ }
+ ],
+ "source": [
+ "directory = os.getcwd() + '/data/raw/playlists/data'\n",
+ "df = pd.DataFrame()\n",
+ "index = 0\n",
+ "# Loop through all files in the directory\n",
+ "for filename in os.listdir(directory):\n",
+ " # Check if the item is a file (not a subdirectory)\n",
+ " if os.path.isfile(os.path.join(directory, filename)):\n",
+ " if filename.find('.json') != -1 :\n",
+ " index += 1\n",
+ "\n",
+ " # Print the filename or perform operations on the file\n",
+ " print(f'\\r{filename}\\t{index}/1000\\t{((index/1000)*100):.1f}%', end='')\n",
+ "\n",
+ " # If you need the full file path, you can use:\n",
+ " full_path = os.path.join(directory, filename)\n",
+ "\n",
+ " with open(full_path, 'r') as file:\n",
+ " json_data = json.load(file)\n",
+ "\n",
+ " temp = pd.DataFrame(json_data['playlists'])\n",
+ " expanded_df = temp.explode('tracks').reset_index(drop=True)\n",
+ "\n",
+ " # Normalize the JSON data\n",
+ " json_normalized = pd.json_normalize(expanded_df['tracks'])\n",
+ "\n",
+ " # Concatenate the original DataFrame with the normalized JSON data\n",
+ " result = pd.concat([expanded_df.drop(columns=['tracks']), json_normalized], axis=1)\n",
+ "\n",
+ " result = result[cols]\n",
+ "\n",
+ " df = pd.concat([df, result], axis=0, ignore_index=True)\n",
+ "\n",
+ " if index % 50 == 0:\n",
+ " df.to_parquet(f'{os.getcwd()}/data/raw/data/playlists_{index % 1000}.parquet')\n",
+ " del df\n",
+ " df = pd.DataFrame()\n",
+ " if index % 200 == 0:\n",
+ " break"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {
+ "id": "unZ418pc_LJy"
+ },
+ "outputs": [],
+ "source": [
+ "import pyarrow.parquet as pq\n",
+ "\n",
+ "def read_parquet_folder(folder_path):\n",
+ " dataframes = []\n",
+ " for file in os.listdir(folder_path):\n",
+ " if file.endswith('.parquet'):\n",
+ " file_path = os.path.join(folder_path, file)\n",
+ " df = pd.read_parquet(file_path)\n",
+ " dataframes.append(df)\n",
+ "\n",
+ " return pd.concat(dataframes, ignore_index=True)\n",
+ "\n",
+ "folder_path = os.getcwd() + '/data/raw/data'\n",
+ "df = read_parquet_folder(folder_path)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {
+ "id": "es6n8S3a_LJz"
+ },
+ "outputs": [],
+ "source": [
+ "directory = os.getcwd() + '/data/raw/mappings'\n",
+ "make_dir(directory)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {
+ "id": "Rc2JtdBR_LJz"
+ },
+ "outputs": [],
+ "source": [
+ "def create_ids(df, col, name):\n",
+ " # Create a dictionary mapping unique values to IDs\n",
+ " value_to_id = {val: i for i, val in enumerate(df[col].unique())}\n",
+ "\n",
+ " # Create a new column with the IDs\n",
+ " df[f'{name}_id'] = df[col].map(value_to_id)\n",
+ " df[[f'{name}_id', col]].drop_duplicates().to_csv(os.getcwd() + f'/data/processed/{name}.csv')\n",
+ "\n",
+ " return df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {
+ "id": "O6aZ566R_LJ0"
+ },
+ "outputs": [],
+ "source": [
+ "# df = create_ids(df, 'artist_name', 'artist')\n",
+ "df = create_ids(df, 'pid', 'playlist')\n",
+ "# df = create_ids(df, 'track_name', 'track')\n",
+ "# df = create_ids(df, 'album_name', 'album')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {
+ "id": "pWWICQvh03KH"
+ },
+ "outputs": [],
+ "source": [
+ "df['song_count'] = df.groupby(['pid','artist_name','album_name'])['track_name'].transform('nunique')\n",
+ "\n",
+ "df['playlist_songs'] = df.groupby(['pid'])['pos'].transform('max')\n",
+ "df['playlist_songs'] += 1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {
+ "id": "F-S7j-gI4I6W"
+ },
+ "outputs": [],
+ "source": [
+ "df['artist_album'] = df[['artist_name', 'album_name']].agg('::'.join, axis=1)\n",
+ "\n",
+ "# Step 2: Create a dictionary mapping unique combined values to IDs\n",
+ "value_to_id = {val: i for i, val in enumerate(df['artist_album'].unique())}\n",
+ "\n",
+ "# Step 3: Map these IDs back to the DataFrame\n",
+ "df['artist_album_id'] = df['artist_album'].map(value_to_id)\n",
+ "\n",
+ "df[[f'artist_album_id', 'artist_album', 'artist_name', 'album_name', 'track_name']].drop_duplicates().to_csv(os.getcwd() + f'/data/processed/artist_album.csv')\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {
+ "id": "q6KHerHG6xZF"
+ },
+ "outputs": [],
+ "source": [
+ "# df = df.groupby(['playlist_id','artist_album','artist_album_id','playlist_songs']).agg({\n",
+ "# 'song_count': 'sum',\n",
+ "# 'track_name': '|'.join,\n",
+ "# 'track_name': '|'.join,\n",
+ "# }).reset_index()\n",
+ "df['song_count'] = df.groupby(['playlist_id','artist_album_id'])['song_count'].transform('sum')\n",
+ "\n",
+ "# Encode the genres data\n",
+ "encoder = LabelEncoder()\n",
+ "encoder.fit(df['track_name'])\n",
+ "df['track_id'] = encoder.transform(df['track_name'])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "r0YprWVe_LJ0"
+ },
+ "outputs": [],
+ "source": [
+ "# df['artist_count'] = df.groupby(['playlist_id','artist_id'])['song_id'].transform('nunique')\n",
+ "# df['album_count'] = df.groupby(['playlist_id','artist_id','album_id'])['song_id'].transform('nunique')\n",
+ "# df['song_count'] = df.groupby(['artist_id'])['song_id'].transform('count')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {
+ "id": "D0IkRvv6_LJ1"
+ },
+ "outputs": [],
+ "source": [
+ "# df['artist_percent'] = df['artist_count'] / df['playlist_songs']\n",
+ "df['song_percent'] = df['song_count'] / df['playlist_songs']\n",
+ "# df['album_percent'] = df['album_count'] / df['playlist_songs']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {
+ "id": "TnFfvqoSxtW3"
+ },
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "\n",
+ "# Assuming you have a DataFrame 'df' with a column 'column_name'\n",
+ "df['song_percent'] = 1 / (1 + np.exp(-df['song_percent']))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 206
+ },
+ "id": "XyURi3ZQ_LJ1",
+ "outputId": "70e3d126-ab5c-490d-a92e-030f32348969"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " playlist_id | \n",
+ " artist_album_id | \n",
+ " song_percent | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 0 | \n",
+ " 0 | \n",
+ " 0.612336 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 0 | \n",
+ " 1 | \n",
+ " 0.861591 | \n",
+ "
\n",
+ " \n",
+ " | 12 | \n",
+ " 0 | \n",
+ " 2 | \n",
+ " 0.528540 | \n",
+ "
\n",
+ " \n",
+ " | 13 | \n",
+ " 0 | \n",
+ " 3 | \n",
+ " 0.507142 | \n",
+ "
\n",
+ " \n",
+ " | 14 | \n",
+ " 0 | \n",
+ " 4 | \n",
+ " 0.612336 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "