app.py

import streamlit as st
import pandas as pd
import psycopg2
import re
import nltk
from nltk.tokenize import word_tokenize
from nltk.tag import pos_tag
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import unicodedata

nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('stopwords')

stop_words = set(stopwords.words('english'))

def get_paragraph(row, index):
    ans = ''
    for x in row[index]:
        ans = ans + ' ' + x.lower()
    return ans

def remove_accents(text):
    text = unicodedata.normalize('NFKD', text).encode('ASCII', 'ignore').decode('utf-8')
    return text

def get_clean_text(row, index):
    if not isinstance(row[index], str):
        return ''
    if row[index] == "NULL":
        return ''
    clean_text = ''
    words = word_tokenize(row[index].lower())
    for word in words:
        word = word.replace(',', ' ')
        word = remove_accents(word)
        if re.match(r'^[a-zA-Z]+$', word) and word not in stop_words and len(word) > 1 and word[1] != '.':
            clean_text += ' ' + word
    return clean_text

def combine(row, indices):
    ans = ''
    for i in indices:
        ans = ans + ' ' + row[i]
    return ans
# Connect to the PostgreSQL database

# Read the data from the PostgreSQL table
query = "SELECT * FROM base_springerdata"
# st cache
@st.cache_data  # 👈 Add the caching decorator
def load_data(query):
    conn = psycopg2.connect(
    host="ep-soft-art-878483.ap-southeast-1.aws.neon.tech",
    database="neondb",
    user="Raghuveer22",
    password="pw3tvedja4XU"
    )
    df =pd.read_sql_query(query, conn)
    df = df.drop(['item_doi'], axis=1)
    conn.close()
    return df
# main_df
main_df = load_data(query)
# Close the database connection

@st.cache_data
def get_journal_df(df):
    journal_art = df.groupby('publication_title')['item_title'].apply(list).reset_index(name='Articles')
    journal_art.set_index(['publication_title'], inplace=True)

    journal_auth = df.groupby('publication_title')['authors'].apply(list).reset_index(name='authors')
    journal_auth.set_index('publication_title', inplace=True)

    journal_key = df.drop_duplicates(subset=["publication_title", "keywords"], keep='first')
    journal_key = journal_key.drop(['item_title', 'authors', 'publication_year', 'url'], axis=1)
    journal_key.set_index(['publication_title'], inplace=True)

    journal_main = journal_art.join([journal_key, journal_auth])
    print('journal_main intial')
    journal_main.reset_index(inplace=True)
    journal_main['Articles'] = journal_main.apply(get_paragraph, index='Articles', axis=1)
    journal_main['Articles'] = journal_main.apply(get_clean_text, index='Articles', axis=1)
    journal_main['authors'] = journal_main.apply(get_paragraph, index='authors', axis=1)
    journal_main['authors'] = journal_main.apply(get_clean_text, index='authors', axis=1)
    journal_main['keywords'] = journal_main.apply(get_clean_text, index='keywords', axis=1)

    journal_main['Tags'] = journal_main.apply(combine, indices=['keywords', 'Articles', 'authors'], axis=1)
    journal_main['Tags'] = journal_main.apply(get_clean_text, index='Tags', axis=1)
    return journal_main

journal_main=get_journal_df(main_df)
print('journal_main processed')
# Journal Dataframe


@st.cache_data
def get_tfidfs(journal_main):
    vectorizer = TfidfVectorizer(decode_error='ignore', strip_accents='ascii')
    journal_tfidf_matrix = vectorizer.fit_transform(journal_main['Tags'])
    return vectorizer,journal_tfidf_matrix

vectorizer,journal_tfidf_matrix  = get_tfidfs(journal_main)
print('tfids and vectorizer for journals completed')
journal_threshold = 4

def get_journal_index(user_input):
    user_tfidf = vectorizer.transform([user_input])
    cosine_similarities = cosine_similarity(user_tfidf, journal_tfidf_matrix).flatten()
    indices = cosine_similarities.argsort()[::-1]
    top_recommendations = [i for i in indices if cosine_similarities[i] > 0][:min(journal_threshold, len(indices))]
    return top_recommendations

def get_article_df(row):
    article = main_df.loc[main_df['publication_title'] == journal_main['publication_title'][row.name]].copy()
    article['item_title'] = article.apply(get_clean_text, index='item_title', axis=1)
    article['authors'] = article.apply(get_clean_text, index='authors', axis=1)
    article['Tokenized'] = article['item_title'].apply(word_tokenize)
    article['Tagged'] = article['Tokenized'].apply(pos_tag)
    article['Tags'] = article['Tagged'].apply(lambda x: [word for word, tag in x if
                                                         tag.startswith('NN') or tag.startswith('JJ') and word.lower() not in stop_words])
    article['Tags'] = article.apply(get_paragraph, index='Tags', axis=1)
    article['Tags'] = article.apply(lambda x: x['Tags'] + ' ' + x['authors'] + ' ' + str(x['publication_year']), axis=1)
    article = article.drop(['keywords', 'publication_title', 'Tokenized', 'Tagged', 'authors', 'publication_year'], axis=1)
    article.reset_index(inplace=True)
    article.set_index('index', inplace=True)
    return article



def get_vectorizer(row):
    vectorizer = TfidfVectorizer(decode_error='ignore', strip_accents='ascii')
    return vectorizer


def get_tfidf_matrix(row):
    tfidf_matrix = row['article_vectorizer'].fit_transform(row['article_df']['Tags'])
    return tfidf_matrix

@st.cache_data
def article_preprocessing(df):
    df['article_df'] = df.apply(get_article_df, axis=1)
    df['article_vectorizer'] = df.apply(get_vectorizer, axis=1)
    df['article_matrix'] = df.apply(get_tfidf_matrix, axis=1)
    return df

journal_main=article_preprocessing(journal_main)
print('done')


article_threshold = 10


def get_article_recommendations(user_input):
    recommended_journals = get_journal_index(user_input)
    recommendations = []
    for journal_id in recommended_journals:
        user_tfidf = journal_main['article_vectorizer'][journal_id].transform([user_input])
        cosine_similarities = cosine_similarity(user_tfidf, journal_main['article_matrix'][journal_id]).flatten()
        indices = cosine_similarities.argsort()[::-1]
        top_recommendation_articles = [(cosine_similarities[i], i, journal_id) for i in indices if
                                       cosine_similarities[i] > 0][:min(article_threshold, len(indices))]
        recommendations += top_recommendation_articles
    recommendations.sort(reverse=True)
    return recommendations


def get_links(user_input):
    recommendations = get_article_recommendations(user_input)
    print(recommendations)
    links = []
    for article in recommendations:
        cosine_similarity, article_id, journal_id = article
        links.append((
            journal_main['article_df'][journal_id].iloc[article_id, 0],
            journal_main['article_df'][journal_id].iloc[article_id, 1],
            article_id,
            journal_id
        ))
    print(links)
    return links

# Define the Streamlit app
def main():
    st.title("Article Recommendation System")
    st.subheader("Testing application of ScholarSync™")
    
    # Get user input
    user_input = st.text_area("Enter your interests", height=100)

    # Recommendation button
    if st.button("Recommend Articles"):
        recommendations = get_links(user_input)
        for recommendation in recommendations:
            article_title, article_link, article_id, journal_id = recommendation
            st.markdown(f"**Article Title:** {article_title}")
            st.markdown(f"**Article Link:** {article_link}")
            st.markdown(f"**Article ID:** {article_id}")
            st.markdown(f"**Journal ID:** {journal_id}")
            st.markdown("---")

if __name__ == '__main__':
    main()