import pandas as pd import numpy as np import requests from bs4 import BeautifulSoup import urllib.parse import streamlit as st import uuid import warnings warnings.filterwarnings('ignore') # Importing Custom Modules from folder_management import create_folder, remove_files_folder from sqlite_database import create_insert_table, query_table ''' This Module will, for a particular keyword and publisherid combo query data from bizrate.com store it in bizrate.db. And then generate Top 50 recommendation from it and Store it in RecSysData.db ''' # Function Definitions def tag_to_list(tag_name, soup): if tag_name == 'Image': tag_list = soup.find_all(tag_name, {'xsize': '400'}) else: tag_list = soup.find_all(tag_name) return pd.Series([item.text for item in tag_list]) def loadRSS(filepath, keyword, publisherid): url = filepath resp = requests.get(url) with open('bizrate/' + keyword + "_" + publisherid + '.xml', 'wb') as f: f.write(resp.content) def remove_dollar_comma(row): try: row = row.split('$')[1] row = row.replace(',', '') row = float(row) except AttributeError: return row return row def url_decode(url): return urllib.parse.unquote(url.split('?t=')[1]) def query_bizrate(keyword, publisherid='725895', search_results=500): file_path = 'http://catalog.bizrate.com/services/catalog/v1/api/product?apiKey=c942e4e24d0859a748b4d1c07c1c3df1' \ '&publisherId={}&placementId=1&categoryId=&keyword={' \ '}&productId=&productIdType=&offersOnly=true&merchantId=&brandId=&biddedOnly=&minPrice=&maxPrice' \ '=&minMarkdown=&zipCode=&freeShipping=&start=0&results={' \ '}&startOffers=0&resultsOffers=0&sort=relevancy_desc&attFilter=&attWeights=&attributeId' \ '=&resultsAttribute=10&resultsAttributeValues=10&showAttributes=&showProductAttributes' \ '=&minRelevancyScore=1000&maxAge=&showRawUrl=&showUnitPricing=&useSecureImageDomain' \ '=&useSecureLinkDomain=&reviews=none&format=xml&callback=callback'.format(publisherid, keyword, search_results) loadRSS(file_path, keyword, publisherid) # stores data as xml file in RecSysData Folder with open('bizrate/' + keyword + "_" + publisherid + '.xml', 'r', errors='ignore') as f: file = f.read() soup = BeautifulSoup(file, 'xml') # cols = ['title', 'Brand', 'mature', 'description', 'manufacturer', 'url', 'Image', 'Skus', 'upc', 'gtin', 'ean13', # 'detailUrl', # 'price', 'originalPrice', 'markdownPercent', 'totalPrice', 'bidded', 'merchantProductId', # 'merchantName', 'merchantLogoUrl', 'condition', 'stock', 'shipAmount', 'shipType', 'relevancy'] cols = ['title', 'Brand', 'url', 'Image', 'Skus', 'price', 'originalPrice', 'markdownPercent', 'totalPrice', 'condition', 'stock', 'relevancy'] df_product = pd.DataFrame({'title': pd.Series(dtype='object'), 'Brand': pd.Series(dtype='object'), 'url': pd.Series(dtype='object'), 'Image': pd.Series(dtype='object'), 'Skus': pd.Series(dtype='object'), 'price': pd.Series(dtype='object'), 'originalPrice': pd.Series(dtype='object'), 'markdownPercent': pd.Series(dtype='object'), 'totalPrice': pd.Series(dtype='object'), 'condition': pd.Series(dtype='object'), 'stock': pd.Series(dtype='object'), 'relevancy': pd.Series(dtype='object')}) for col in cols: df_product[col] = tag_to_list(col, soup) df_product.dropna(subset=['Skus'], inplace=True) df_product['Skus'] = df_product['Skus'].astype('object') # Inserting the DataFrame into the bizrate DB into the corresponding table create_insert_table(db_name='bizrate', table_name=keyword + "_" + publisherid, df=df_product) def recommend(keyword, publisherid='725895', relevancy_filter=False, price_filter=True, discount_filter=False, condition_filter='NEW', stock_filter='IN', n_rec=50): # Querying required data from the bizrate database df_product = query_table(db_name='bizrate', table_name=keyword + "_" + publisherid) '''df_sku = pd.read_excel('df_sku.xlsx') sku_list = list(df_product['Skus'].unique()) df_session = pd.read_excel('df_session.xlsx') session_id = list(df_session['ID'])[-1] for sku in sku_list: df_row = pd.DataFrame({'SessionID': session_id, 'Keyword': selected_keyword, 'Skus': sku, 'Count': 0}, index=[0]) df_sku = pd.concat([df_sku, df_row], ignore_index=True) df_sku.to_excel('df_sku.xlsx', index = False)''' df_product['price'] = df_product['price'].map(remove_dollar_comma) df_product['originalPrice'] = df_product['originalPrice'].map(remove_dollar_comma) df_product['totalPrice'] = df_product['totalPrice'].map(remove_dollar_comma) # df_product['shipAmount'] = df_product['shipAmount'].map(remove_dollar_comma) df_product['markdownPercent'] = df_product['markdownPercent'].astype('float') df_product['relevancy'] = df_product['relevancy'].astype('float') # Sorting the Results df_product = df_product.sort_values(by=['relevancy', 'price', 'markdownPercent'], ascending=[relevancy_filter, price_filter, discount_filter], na_position='last') # Filtering the Results # condition_filter = st.sidebar.selectbox("Condition of the Product: ", list(df_product['condition'].unique())) # stock_filter = st.sidebar.selectbox("Stock of Products: ", list(df_product['stock'].unique())) filter_condition = (df_product['condition'] == condition_filter) & (df_product['stock'] == stock_filter) df_product = df_product[filter_condition] # Top N Recommendations top_n_rec = df_product.head(n_rec) # Inserting the DataFrame into the bizrate DB into the corresponding table create_insert_table(db_name='RecSysData', table_name=keyword + "_" + publisherid, df=top_n_rec) # Main Program #selected_keyword = 'aquaman' #publisher_id = '725895' #query_bizrate(selected_keyword, publisher_id, 100) #recommend(selected_keyword, publisher_id) # list_keywords = ['aquaman', 'superman', 'batman', 'shoes', 'electronics', 'wallet', 'movies', 'books'] # # for selected_keyword in list_keywords: # query_bizrate(selected_keyword) # recommend(selected_keyword)