File size: 6,855 Bytes
fa3c63f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | import pandas as pd
import numpy as np
import requests
from bs4 import BeautifulSoup
import urllib.parse
import streamlit as st
import uuid
import warnings
warnings.filterwarnings('ignore')
# Importing Custom Modules
from folder_management import create_folder, remove_files_folder
from sqlite_database import create_insert_table, query_table
'''
This Module will, for a particular keyword and publisherid combo query data from bizrate.com store it
in bizrate.db. And then generate Top 50 recommendation from it and Store it in RecSysData.db
'''
# Function Definitions
def tag_to_list(tag_name, soup):
if tag_name == 'Image':
tag_list = soup.find_all(tag_name, {'xsize': '400'})
else:
tag_list = soup.find_all(tag_name)
return pd.Series([item.text for item in tag_list])
def loadRSS(filepath, keyword, publisherid):
url = filepath
resp = requests.get(url)
with open('bizrate/' + keyword + "_" + publisherid + '.xml', 'wb') as f:
f.write(resp.content)
def remove_dollar_comma(row):
try:
row = row.split('$')[1]
row = row.replace(',', '')
row = float(row)
except AttributeError:
return row
return row
def url_decode(url):
return urllib.parse.unquote(url.split('?t=')[1])
def query_bizrate(keyword, publisherid='725895', search_results=500):
file_path = 'http://catalog.bizrate.com/services/catalog/v1/api/product?apiKey=c942e4e24d0859a748b4d1c07c1c3df1' \
'&publisherId={}&placementId=1&categoryId=&keyword={' \
'}&productId=&productIdType=&offersOnly=true&merchantId=&brandId=&biddedOnly=&minPrice=&maxPrice' \
'=&minMarkdown=&zipCode=&freeShipping=&start=0&results={' \
'}&startOffers=0&resultsOffers=0&sort=relevancy_desc&attFilter=&attWeights=&attributeId' \
'=&resultsAttribute=10&resultsAttributeValues=10&showAttributes=&showProductAttributes' \
'=&minRelevancyScore=1000&maxAge=&showRawUrl=&showUnitPricing=&useSecureImageDomain' \
'=&useSecureLinkDomain=&reviews=none&format=xml&callback=callback'.format(publisherid,
keyword,
search_results)
loadRSS(file_path, keyword, publisherid) # stores data as xml file in RecSysData Folder
with open('bizrate/' + keyword + "_" + publisherid + '.xml', 'r', errors='ignore') as f:
file = f.read()
soup = BeautifulSoup(file, 'xml')
# cols = ['title', 'Brand', 'mature', 'description', 'manufacturer', 'url', 'Image', 'Skus', 'upc', 'gtin', 'ean13',
# 'detailUrl',
# 'price', 'originalPrice', 'markdownPercent', 'totalPrice', 'bidded', 'merchantProductId',
# 'merchantName', 'merchantLogoUrl', 'condition', 'stock', 'shipAmount', 'shipType', 'relevancy']
cols = ['title', 'Brand', 'url', 'Image', 'Skus', 'price', 'originalPrice', 'markdownPercent', 'totalPrice',
'condition', 'stock', 'relevancy']
df_product = pd.DataFrame({'title': pd.Series(dtype='object'),
'Brand': pd.Series(dtype='object'),
'url': pd.Series(dtype='object'),
'Image': pd.Series(dtype='object'),
'Skus': pd.Series(dtype='object'),
'price': pd.Series(dtype='object'),
'originalPrice': pd.Series(dtype='object'),
'markdownPercent': pd.Series(dtype='object'),
'totalPrice': pd.Series(dtype='object'),
'condition': pd.Series(dtype='object'),
'stock': pd.Series(dtype='object'),
'relevancy': pd.Series(dtype='object')})
for col in cols:
df_product[col] = tag_to_list(col, soup)
df_product.dropna(subset=['Skus'], inplace=True)
df_product['Skus'] = df_product['Skus'].astype('object')
# Inserting the DataFrame into the bizrate DB into the corresponding table
create_insert_table(db_name='bizrate', table_name=keyword + "_" + publisherid, df=df_product)
def recommend(keyword, publisherid='725895', relevancy_filter=False, price_filter=True, discount_filter=False,
condition_filter='NEW', stock_filter='IN', n_rec=50):
# Querying required data from the bizrate database
df_product = query_table(db_name='bizrate', table_name=keyword + "_" + publisherid)
'''df_sku = pd.read_excel('df_sku.xlsx')
sku_list = list(df_product['Skus'].unique())
df_session = pd.read_excel('df_session.xlsx')
session_id = list(df_session['ID'])[-1]
for sku in sku_list:
df_row = pd.DataFrame({'SessionID': session_id, 'Keyword': selected_keyword, 'Skus': sku, 'Count': 0}, index=[0])
df_sku = pd.concat([df_sku, df_row], ignore_index=True)
df_sku.to_excel('df_sku.xlsx', index = False)'''
df_product['price'] = df_product['price'].map(remove_dollar_comma)
df_product['originalPrice'] = df_product['originalPrice'].map(remove_dollar_comma)
df_product['totalPrice'] = df_product['totalPrice'].map(remove_dollar_comma)
# df_product['shipAmount'] = df_product['shipAmount'].map(remove_dollar_comma)
df_product['markdownPercent'] = df_product['markdownPercent'].astype('float')
df_product['relevancy'] = df_product['relevancy'].astype('float')
# Sorting the Results
df_product = df_product.sort_values(by=['relevancy', 'price', 'markdownPercent'],
ascending=[relevancy_filter, price_filter, discount_filter], na_position='last')
# Filtering the Results
# condition_filter = st.sidebar.selectbox("Condition of the Product: ", list(df_product['condition'].unique()))
# stock_filter = st.sidebar.selectbox("Stock of Products: ", list(df_product['stock'].unique()))
filter_condition = (df_product['condition'] == condition_filter) & (df_product['stock'] == stock_filter)
df_product = df_product[filter_condition]
# Top N Recommendations
top_n_rec = df_product.head(n_rec)
# Inserting the DataFrame into the bizrate DB into the corresponding table
create_insert_table(db_name='RecSysData', table_name=keyword + "_" + publisherid, df=top_n_rec)
# Main Program
#selected_keyword = 'aquaman'
#publisher_id = '725895'
#query_bizrate(selected_keyword, publisher_id, 100)
#recommend(selected_keyword, publisher_id)
# list_keywords = ['aquaman', 'superman', 'batman', 'shoes', 'electronics', 'wallet', 'movies', 'books']
#
# for selected_keyword in list_keywords:
# query_bizrate(selected_keyword)
# recommend(selected_keyword)
|