app.py

import streamlit as st
import re
import altair as alt
from folder_management import create_folder, remove_files_folder

st.set_page_config('Product Recommendation Application', layout='wide', page_icon='airtory_logo.PNG')


# Functions/Pages
def intro():
    import streamlit as st

    st.markdown("## Welcome to the Product Recommendation Application! 👋")
    st.sidebar.success("Select a page from above.")

    st.markdown(
        """
        **👈 Select a page from the navigation bar to the left**

###  Functionality of Pages
#### Generate Recommendations Page
- This page allows you to generate recommendations for a particular keyword/publisherid combo.
- The recommendations for each keyword/publisherid combo, are stored in a separate SQLite Database Table.
- Ideally, for each keyword/publisherid combo, the recommendations should only be generated once per day. To mock this, you are only allowed to generate recommendations for a specific keyword/publisherid combo once.
#### Product Recommendations Page
- This page displays the Product Recommendations for each keyword/publisherid combo for which the recommendations were generated in the previous page.
- You can currently select the available keywords from the dropdown provided in the side bar.
- For the time being, the Publisher ID is restricted to a single known value. This can be easily updated later.
- Note: Since the click tracker I was using had issues, as a workaround I had to use a Button instead. So now, to register a click against a product, you need to click on the 'Select' button provided below each product. 
- For each session, once a product is clicked on, it is no longer displayed.
- You can choose the number of recommendations to be displayed, between 1 & 5.
- The maximum number of recommendations that are available for display for each keyword/publisherid combo is restricted to 50.
- The minumum number depends on the products available from bizrate.
- If/when all available recommendations for a keyword/publisherid combo has been clicked on, you can either choose a different keyword, reload the application to create a new session.
#### Analytics Page
- Displays a Line Chart and a Bar Chart, to show the distribution of clicks across Session ID, Keyword, Publisher ID, SKU or Date.
- Displays a table of the most clicked SKU's
    """
    )


def generate_recommendations():
    import os
    import string
    import random
    import streamlit as st
    import warnings
    warnings.filterwarnings('ignore')

    # custom module
    # from clickcounter import clickcounter
    from folder_management import create_folder, remove_files_folder
    from sqlite_database import create_insert_table, query_table, insert_clickdata_table, check_for_table
    from recommend import query_bizrate, recommend, generate_clickreport

    # Functions
    def random_string(N=7):
        res = ''.join(random.choices(string.ascii_lowercase + string.digits, k=N))
        return str(res)

    def query_and_recommend(keyword, publisherid):
        query_bizrate(keyword, publisherid)
        recommend(keyword, publisherid)
        generate_clickreport()

    # Main Program

    # st.set_page_config('Generate Recommendations', layout='wide')

    st.title('Generate Recommendations')
    st.markdown('''<pre style="text-align:center">
    <strong><span style="color:#6a8759">
    Generate Recommendations for any Keyword - Publisher ID Combo. <br><br> </span></strong></pre>''',
                unsafe_allow_html=True)

    col1, col2 = st.columns(2)

    selected_keyword = ''
    publisher_id = ''

    with col1:
        selected_keyword = st.text_input('Enter a single word as keyword:', 'bed')
        selected_keyword = selected_keyword.lower()
        selected_keyword = selected_keyword.replace('-', '')
        selected_keyword = ''.join(re.split(r"[ \|\\\/,.]", selected_keyword))
        # selected_keyword = ''.join(selected_keyword.split())
        st.write(selected_keyword)

    with col2:
        publisher_id = st.text_input("Enter a Publisher ID: ", '726189')
        # st.write(publisher_id)

    keyword_pubid_list = []
    for file in os.listdir('bizrate'):
        keyword_pubid_list.append(file.replace(".xml", ""))

    if selected_keyword + "_" + publisher_id in keyword_pubid_list:
        st.error('Keyword - Publisher ID Combo Exists!')
    else:
        st.info("Keyword - Publisher ID Combo doesn't exist. Must be queried")
        st.button('Generate Recommendations', key=random_string(), on_click=query_and_recommend,
                  args=([selected_keyword, publisher_id]))


def display_recommendations():
    import pandas as pd
    import numpy as np
    import requests
    from bs4 import BeautifulSoup
    import urllib.parse
    import os
    import string
    import random

    import time

    import streamlit as st
    import uuid

    import warnings
    import sys

    warnings.filterwarnings('ignore')

    # custom module
    # from clickcounter import clickcounter
    from folder_management import create_folder, remove_files_folder
    from sqlite_database import create_insert_table, query_table, insert_clickdata_table, check_for_table

    # Functions
    def random_string(N=7):
        res = ''.join(random.choices(string.ascii_lowercase + string.digits, k=N))
        return str(res)

    # def display_products(dfp):
    #     product_list = list(dfp['title'])
    #     image_list = list(dfp['Image'])
    #     price_list = list(dfp['price'])
    #     org_price = list(dfp['originalPrice'])
    #     disc_list = list(dfp['markdownPercent'])
    #     dfp['Skus'] = dfp['Skus'].astype('object')
    #     sku_list = list(dfp['Skus'])
    #     url_list = list(dfp['url'])
    #     clicked_sku, counter_dict = clickcounter(image_list, sku_list, price_list, disc_list, org_price, url_list, product_list)
    #     return clicked_sku, counter_dict

    # Main Program
    # st.set_page_config('Product Recommender System', layout='wide')
    st.title('Product Recommender System')
    st.markdown('''<pre style="text-align:center">
    <strong><span style="color:#6a8759">
    It will take as inputs: Keyword and Publisher ID. 
    Displays (upto) Top 5 Recommendations &amp; Collects Click Information. <br><br> </span></strong></pre>''',
                unsafe_allow_html=True)

    # Generating Session ID:
    if 'store' not in st.session_state:
        st.session_state.store = False

        try:
            df_session = pd.read_excel('df_session.xlsx')
            df_row = pd.DataFrame({'ID': uuid.uuid4()}, index=[0])
            df_session = pd.concat([df_session, df_row], ignore_index=True)
            df_session.to_excel('df_session.xlsx', index=False)
            df_sku = pd.DataFrame({'SessionID': pd.Series(dtype='object'), 'Keyword': pd.Series(dtype='object'),
                                   'Skus': pd.Series(dtype='object'),
                                   'Count': pd.Series(dtype='int')})  # Initializing the SKU-Count DataFrame
            df_sku.to_excel('df_sku.xlsx', index=False)

        except FileNotFoundError:
            df_session = pd.DataFrame({'ID': uuid.uuid4()}, index=[0])  # Initializing the SKU-Count DataFrame
            df_session.to_excel('df_session.xlsx', index=False)

    df_session = pd.read_excel('df_session.xlsx')
    session_id = list(df_session['ID'])[-1]
    # st.sidebar.write(session_id) # Uncomment to display the Session ID

    # Inputs

    # selected_keyword = st.sidebar.text_input("Enter a single word as Keyword: ", 'aquaman')
    # publisher_id = st.sidebar.text_input("Enter Publisher ID: ", '725895')
    list_keywords = []
    list_publisherid = []
    for file in os.listdir('bizrate'):
        keyword_publisherid = file.replace(".xml", "")
        list_keywords.append(keyword_publisherid.split("_")[0])
        list_publisherid.append(keyword_publisherid.split("_")[1])
    # list_keywords = ['aquaman', 'superman', 'batman', 'shoes', 'electronics', 'wallet', 'movies', 'books']  # Temporary
    list_publisherid = ['726189']  # For the time being Publisher ID is being HardCoded.
    selected_keyword = st.sidebar.selectbox("Select a Keyword:", set(sorted(list_keywords)))
    publisher_id = st.sidebar.selectbox("Select a Publisher ID:", set(list_publisherid))

    # st.write(selected_keyword)
    # st.write(publisher_id)

    # Query clickReport.db, aggregate clicks and sort most clicked
    clickreport_df = query_table('clickReport', 'clickReport')
    clicked_df = clickreport_df[clickreport_df['keyword'] == selected_keyword]
    clicked_df = clicked_df[['keyword', 'Skus', 'clicks']].groupby(['keyword', 'Skus']).sum().reset_index()
    clicked_df = clicked_df.sort_values(by='clicks', ascending=False)
    to_display_df = clicked_df.copy()
    clicked_df = clicked_df[['Skus', 'clicks']]
    print("LINE 202: Most Clicked: \n {}".format(clicked_df.head()))
    # Query Recommended Data as per Keyword and Publisher ID

    try:
        rec_df = query_table('RecSysData', selected_keyword + "_" + publisher_id)
        rec_df = pd.merge(clicked_df, rec_df, how='right', on=['Skus'])
        rec_df = rec_df.sort_values(by='clicks', ascending=False)
        print("LINE 209: \n {}".format(rec_df.head()))
        print("LINE 210: \n {}".format(rec_df.columns))
    except pd.errors.DatabaseError:
        # st.error('This Keyword Publisher ID Combo Doesnt Exist!')
        rec_df = pd.DataFrame({}, columns=['title', 'Brand', 'url', 'Image', 'Skus', 'price', 'originalPrice',
                                           'markdownPercent', 'totalPrice', 'condition', 'stock', 'relevancy'])
        rec_df = pd.merge(clicked_df, rec_df, how='left', on=['Skus'])
        rec_df = rec_df.sort_values(by='clicks', ascending=False)

    # Logic to Decide which SKU's to Display

    # 1. If a SKU has already been clicked on. It cannot be displayed again.

    try:
        click_data_df = query_table('session_data', 'session_data')

        if click_data_df.empty:
            # df_top_rec = rec_df.head(5)
            rec_df = rec_df.head(5)
        else:
            displayed_skus = list(click_data_df[click_data_df['session_id'] == session_id][
                                      'Skus'])  # List of SKU's already displayed in the current session
            # st.write("Clicked SKUs in this Session: ")
            # st.write(displayed_skus)
            rec_df = rec_df[~rec_df['Skus'].isin(displayed_skus)]
    except pd.errors.DatabaseError as e:
        print(e)

    # Top 5 Recommendations
    # top_df = rec_df.head(5).reset_index()
    try:
        top_df = rec_df.head().sample(5).reset_index()  # random 5 recommendations
    except Exception as e:
        # st.error(e)
        top_df = rec_df.head(5).reset_index()

    recs_to_display = 0
    if len(top_df) > 5:
        recs_to_display = st.sidebar.slider('Recommendations to Display', 1, 5)
    elif len(top_df) > 1:
        recs_to_display = st.sidebar.slider('Recommendations to Display', 1, len(top_df), len(top_df))
    else:
        recs_to_display = 1

    # recs_to_display image width dictionary
    image_width_dictionary = {
        5: 200,
        4: 240,
        3: 280,
        2: 320,
        1: 400}

    if recs_to_display > 0:
        idx = 0
        cols = st.columns(recs_to_display)
        for col in cols:
            with col:
                try:
                    title = top_df['title'][idx]
                    img_link = top_df['url'][idx]
                    sku = top_df['Skus'][idx]
                    list_price = top_df['originalPrice'][idx]
                    selling_price = top_df['price'][idx]
                    discount = top_df['markdownPercent'][idx]
                    st.image(top_df['Image'][idx], width=image_width_dictionary[recs_to_display])
                    # st.write('SKU: {}'.format(sku))
                    # st.write('SKU: {}'.format(sku))
                    # st.write('SKU: {}'.format(sku))
                    content = '''<p><strong> <a href={}>{}</a> </strong> <br>
                                    <strong>SKU:</strong> {} <br>
                                    <strong>S.P:</strong> $ {}<br>
                                    <strong>Discount:</strong> % {} <br>
                                    <strong>L.P:</strong> $ {} <br>
                                    </p>'''.format(img_link, title, sku, selling_price, discount, list_price)
                    st.markdown(content, unsafe_allow_html=True)
                    st.button('Select', key=random_string(), on_click=insert_clickdata_table,
                              args=([session_id, selected_keyword, publisher_id, sku, 1]))
                except KeyError:
                    st.info('No more recommendations to display for this Keyword-PublisherID Combo!')
                    st.info('You can search for another keyword or reload the page!')
            idx += 1
    else:
        st.text(
            'No more recommendations to display for this Keyword-PublisherID Combo! You can search for another keyword or reload the page!')

    # Display Click Table showing total clicks and clicks in current session
    st.write("\n")
    st.write("\n")
    st.write("Most Clicked SKUs for keyword : {}".format(selected_keyword))
    st.dataframe(to_display_df.reset_index(drop=True).head(10))
    st.write("Clicked SKUs in this Session: ")
    st.dataframe(click_data_df[click_data_df['session_id'] == session_id].reset_index(drop=True))


def display_analytics():
    import pandas as pd
    import numpy as np
    import requests
    from bs4 import BeautifulSoup
    import urllib.parse
    import os
    import string
    import random

    import time

    import streamlit as st
    import uuid

    import warnings

    warnings.filterwarnings('ignore')

    # custom module
    # from clickcounter import clickcounter
    from folder_management import create_folder, remove_files_folder
    from sqlite_database import create_insert_table, query_table, insert_clickdata_table, check_for_table

    # Main Program
    # st.set_page_config('Analytics', layout='wide')
    st.title('Analytics')

    df = query_table('session_data', 'session_data')
    # print(pd.Timestamp(df['clicked_at'][0]).date())
    df['date'] = df['clicked_at'].map(lambda x: pd.Timestamp(x).date())
    x_col_list = list(df.columns)
    x_col_list.remove('count')
    x_col_list.remove('clicked_at')
    print(x_col_list)
    column = st.selectbox('Select a Dimension:', x_col_list)

    # Grouping the DataFrame
    if column:
        grouped_df = df.groupby([column]).sum().reset_index()
        if column != 'date':
            grouped_df = grouped_df.sort_values(by='count', ascending=False)
        grouped_df.rename(columns={'count': 'clicks'}, inplace=True)
        # st.dataframe(grouped_df.head())
        col1, col2 = st.columns(2)
        with col1:
            # st.line_chart(grouped_df, x=column, y='clicks')
            alt_line_chart = alt.Chart(grouped_df).mark_line(color='#3ac81e').encode(x=column, y='clicks')
            st.altair_chart(alt_line_chart, use_container_width=True)

        with col2:
            # st.bar_chart(grouped_df, x=column, y='clicks')
            alt_bar_chart = alt.Chart(grouped_df).mark_bar(color='#3ac81e').encode(x=column, y='clicks')
            st.altair_chart(alt_bar_chart, use_container_width=True)

    st.title("Top Products")
    agg_df = df[['keyword', 'publisherid', 'Skus', 'count']]
    agg_df.rename(columns={'keyword': 'Keywords', 'publisherid': 'PublisherID', 'count': 'Clicks'}, inplace=True)
    agg_df = agg_df.groupby(['Skus', 'Keywords', 'PublisherID']).sum().reset_index()
    agg_df = agg_df.sort_values(by='Clicks', ascending=False).reset_index(drop=True)
    st.dataframe(agg_df.head(20), width=1000)


create_folder('bizrate')
create_folder('sqlite_databases')

page_names_to_funcs = {
    "—": intro,
    "Generate Recommendations": generate_recommendations,
    "Product Recommendations": display_recommendations,
    "Analytics": display_analytics
}

page_name = st.sidebar.selectbox("Choose a Page", page_names_to_funcs.keys())
page_names_to_funcs[page_name]()