Create recommend.py

recommend.py

@@ -0,0 +1,154 @@
+import pandas as pd
+import numpy as np
+import requests
+from bs4 import BeautifulSoup
+import urllib.parse
+
+import streamlit as st
+import uuid
+
+import warnings
+
+warnings.filterwarnings('ignore')
+
+# Importing Custom Modules
+from folder_management import create_folder, remove_files_folder
+from sqlite_database import create_insert_table, query_table
+
+'''
+This Module will, for a particular keyword and publisherid combo query data from bizrate.com store it
+in bizrate.db. And then generate Top 50 recommendation from it and Store it in RecSysData.db
+
+'''
+
+
+# Function Definitions
+def tag_to_list(tag_name, soup):
+    if tag_name == 'Image':
+        tag_list = soup.find_all(tag_name, {'xsize': '400'})
+    else:
+        tag_list = soup.find_all(tag_name)
+
+    return pd.Series([item.text for item in tag_list])
+
+
+def loadRSS(filepath, keyword, publisherid):
+    url = filepath
+    resp = requests.get(url)
+    with open('bizrate/' + keyword + "_" + publisherid + '.xml', 'wb') as f:
+        f.write(resp.content)
+
+
+def remove_dollar_comma(row):
+    try:
+        row = row.split('$')[1]
+        row = row.replace(',', '')
+        row = float(row)
+    except AttributeError:
+        return row
+    return row
+
+
+def url_decode(url):
+    return urllib.parse.unquote(url.split('?t=')[1])
+
+
+def query_bizrate(keyword, publisherid='725895', search_results=500):
+    file_path = 'http://catalog.bizrate.com/services/catalog/v1/api/product?apiKey=c942e4e24d0859a748b4d1c07c1c3df1' \
+                '&publisherId={}&placementId=1&categoryId=&keyword={' \
+                '}&productId=&productIdType=&offersOnly=true&merchantId=&brandId=&biddedOnly=&minPrice=&maxPrice' \
+                '=&minMarkdown=&zipCode=&freeShipping=&start=0&results={' \
+                '}&startOffers=0&resultsOffers=0&sort=relevancy_desc&attFilter=&attWeights=&attributeId' \
+                '=&resultsAttribute=10&resultsAttributeValues=10&showAttributes=&showProductAttributes' \
+                '=&minRelevancyScore=1000&maxAge=&showRawUrl=&showUnitPricing=&useSecureImageDomain' \
+                '=&useSecureLinkDomain=&reviews=none&format=xml&callback=callback'.format(publisherid,
+                                                                                          keyword,
+                                                                                          search_results)
+
+    loadRSS(file_path, keyword, publisherid)  # stores data as xml file in RecSysData Folder
+
+    with open('bizrate/' + keyword + "_" + publisherid + '.xml', 'r', errors='ignore') as f:
+        file = f.read()
+
+    soup = BeautifulSoup(file, 'xml')
+
+    # cols = ['title', 'Brand', 'mature', 'description', 'manufacturer', 'url', 'Image', 'Skus', 'upc', 'gtin', 'ean13',
+    #         'detailUrl',
+    #         'price', 'originalPrice', 'markdownPercent', 'totalPrice', 'bidded', 'merchantProductId',
+    #         'merchantName', 'merchantLogoUrl', 'condition', 'stock', 'shipAmount', 'shipType', 'relevancy']
+
+    cols = ['title', 'Brand', 'url', 'Image', 'Skus', 'price', 'originalPrice', 'markdownPercent', 'totalPrice',
+            'condition', 'stock', 'relevancy']
+
+    df_product = pd.DataFrame({'title': pd.Series(dtype='object'),
+                               'Brand': pd.Series(dtype='object'),
+                               'url': pd.Series(dtype='object'),
+                               'Image': pd.Series(dtype='object'),
+                               'Skus': pd.Series(dtype='object'),
+                               'price': pd.Series(dtype='object'),
+                               'originalPrice': pd.Series(dtype='object'),
+                               'markdownPercent': pd.Series(dtype='object'),
+                               'totalPrice': pd.Series(dtype='object'),
+                               'condition': pd.Series(dtype='object'),
+                               'stock': pd.Series(dtype='object'),
+                               'relevancy': pd.Series(dtype='object')})
+    for col in cols:
+        df_product[col] = tag_to_list(col, soup)
+    df_product.dropna(subset=['Skus'], inplace=True)
+    df_product['Skus'] = df_product['Skus'].astype('object')
+
+    # Inserting the DataFrame into the bizrate DB into the corresponding table
+    create_insert_table(db_name='bizrate', table_name=keyword + "_" + publisherid, df=df_product)
+
+
+def recommend(keyword, publisherid='725895', relevancy_filter=False, price_filter=True, discount_filter=False,
+              condition_filter='NEW', stock_filter='IN', n_rec=50):
+
+    # Querying required data from the bizrate database
+    df_product = query_table(db_name='bizrate', table_name=keyword + "_" + publisherid)
+
+    '''df_sku = pd.read_excel('df_sku.xlsx')
+    sku_list = list(df_product['Skus'].unique())
+    df_session = pd.read_excel('df_session.xlsx')
+    session_id = list(df_session['ID'])[-1]
+    for sku in sku_list:
+        df_row = pd.DataFrame({'SessionID': session_id, 'Keyword': selected_keyword, 'Skus': sku, 'Count': 0}, index=[0])
+        df_sku = pd.concat([df_sku, df_row], ignore_index=True)
+    df_sku.to_excel('df_sku.xlsx', index = False)'''
+
+    df_product['price'] = df_product['price'].map(remove_dollar_comma)
+    df_product['originalPrice'] = df_product['originalPrice'].map(remove_dollar_comma)
+    df_product['totalPrice'] = df_product['totalPrice'].map(remove_dollar_comma)
+    # df_product['shipAmount'] = df_product['shipAmount'].map(remove_dollar_comma)
+    df_product['markdownPercent'] = df_product['markdownPercent'].astype('float')
+    df_product['relevancy'] = df_product['relevancy'].astype('float')
+
+    # Sorting the Results
+    df_product = df_product.sort_values(by=['relevancy', 'price', 'markdownPercent'],
+                                        ascending=[relevancy_filter, price_filter, discount_filter], na_position='last')
+
+    # Filtering the Results
+    # condition_filter = st.sidebar.selectbox("Condition of the Product: ", list(df_product['condition'].unique()))
+    # stock_filter = st.sidebar.selectbox("Stock of Products: ", list(df_product['stock'].unique()))
+
+    filter_condition = (df_product['condition'] == condition_filter) & (df_product['stock'] == stock_filter)
+    df_product = df_product[filter_condition]
+
+    # Top N Recommendations
+    top_n_rec = df_product.head(n_rec)
+
+    # Inserting the DataFrame into the bizrate DB into the corresponding table
+    create_insert_table(db_name='RecSysData', table_name=keyword + "_" + publisherid, df=top_n_rec)
+
+
+# Main Program
+#selected_keyword = 'aquaman'
+#publisher_id = '725895'
+#query_bizrate(selected_keyword, publisher_id, 100)
+#recommend(selected_keyword, publisher_id)
+
+# list_keywords = ['aquaman', 'superman', 'batman', 'shoes', 'electronics', 'wallet', 'movies', 'books']
+#
+# for selected_keyword in list_keywords:
+#     query_bizrate(selected_keyword)
+#     recommend(selected_keyword)