File size: 7,006 Bytes
d62ad66
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
import streamlit as st
import pandas as pd
import datetime
from top2vec import Top2Vec
from matplotlib import pyplot as plt

st.set_page_config(layout='wide')

def create_frequency_dataframe(dataframe, default, specific, year_start=1976, year_stop=2000):
    '''Produces a dataframe with the number of articles in each topic in a each year.
    
    Parameters
        dataframe: Pandas dataframe containing the articles, abstracts, and dates.
        default: String indicating which topics to include in the dataframe. Selected by user.
        specific: String of additional topic numbers to include in the dataframe. Selected by user. Added to default.
        year_start: Integer indicating the first year to include in the dataframe. Selected by user.
        year_stop: Integer indicating the last year to include in the dataframe. Selected by user.
        
    Returns
        date_df: Pandas dataframe with the number of articles in each topic in a each year.
        topic_columns: List of topic numbers included in the dataframe.
    '''
    topic_sizes, topic_nums = model.get_topic_sizes(reduced=True)
    topic_columns = []
    if default == 'Top 5':
        topic_columns = topic_nums[:5]
    elif default == 'Top 10':
        topic_columns = topic_nums[:10]
    elif default == 'Bottom 5':
        topic_columns = topic_nums[-5:]
    elif default == 'Bottom 10':
        topic_columns = topic_nums[-10:]

    topic_columns= list(topic_columns) + [int(i) for i in specific.split() if len(i) > 0] # Break input text into list of ints then merge lists
    
    # Format df for number of articles in each topic in a given year
    date_df = pd.DataFrame(index=range(year_start, year_stop + 1), columns=topic_columns)
    for col in date_df.columns:
        date_df[col].values[:] = 0  # Covert values from null to 0

    #Year/index num correspondence
    # index_dict = {}
    # for i, year in enumerate(date_df.index.values):
    #     index_dict[year] = i

    # Get doc_ids for each requested topic
    for topic in topic_columns:
        _, document_ids = model.search_documents_by_topic(topic_num=topic, reduced=True, num_docs=topic_sizes[topic]) # Works because topic number corresponds to its order in the topic size list
        for id in document_ids:
            if dataframe.at[id, 'DP'] >= datetime.date(year_start,1,1).year and dataframe.at[id, 'DP'] < datetime.date(year_stop + 1,1,1).year:
                date_df.at[dataframe.at[id, 'DP'], topic] += 1

    return date_df, topic_columns

def create_prevalence_dataframe(date_df):
    date_df = date_df.join(date_df.apply(sum, axis=1).reindex_like(date_df.index.to_series()).rename('Sums'))  # Sum the total number of articles in each year
    # Find the percentage of total number of articles represented by each topic in each year
    for i, row in zip(date_df.index.values, date_df['Sums']):
        if row != 0:
            date_df.loc[i, :] = (date_df.loc[i, :] / date_df.loc[i, 'Sums']) * 100
    date_df = date_df.drop(columns='Sums')
    return date_df

def create_plot(date_df, ylabel, title):
    fig, ax = plt.subplots(figsize=(10, 5))
    for topic in date_df.columns:
        ax.plot(date_df.index.values, date_df.loc[:, topic].values, label=f'Topic {topic}')
    ax.xaxis.set_major_locator(plt.MaxNLocator(integer=True)) # Only show whole numbers on x-axis
    ax.set_xlabel('Year')
    ax.set_ylabel(ylabel)
    ax.set_title(title)
    ax.legend(bbox_to_anchor=(1, 1))
    ax.grid(True)
    return fig

@st.cache_resource
def load_model():
    name = '2024-02-14_22_19_09_reduced__model_medline0[test,mincount_250,topicmergedelta_0.15,ngram_True,ngramvocab_args_250_.4_80M_npmi_connector].json'
    with open(name, 'rb') as f:
        return Top2Vec.load(f)

@st.cache_data
def load_data():
    df = pd.read_pickle('medline_0_df_nlp.pkl.bz2', compression='bz2')
    return df

# Notification of loading
start_container = st.empty()
if 'first' not in st.session_state:
    st.session_state['first'] = True
    
if st.session_state['first'] == True:
    start_container.text('Loading model and data... This may take two to three minutes on the initial run.')
    st.session_state['first'] = False

# Load the model and data
model = load_model()
df = load_data()

# Remove loading notification
start_container.empty()

# User Interface
with st.sidebar.form('Topic Prevalence', clear_on_submit=False):
    default = st.radio('Default Topics', ['Top 5', 'Top 10', 'Bottom 5', 'Bottom 10', 'None'])
    specific = st.text_input('Topic Numbers', help='''Enter a series of topic numbers to graph. 
                                                    The numbers should be separated by a space.
                                                    Topic numbers are assigned in order of decreasing frequency across the corpus.
                                                    In other words, Topic 0 is the most frequent topic, Topic 1 is the second most frequent, etc.''')
    year_start_tp, year_stop_tp = st.slider('Year Range', 
                                    min_value=1975, 
                                    max_value=2000, 
                                    value=[1975, 2000])
    submitted_ed = st.form_submit_button("Submit")

    st.markdown('### Glossary')
    st.markdown('**Proportion of Articles:** The percentage of the total number of articles displayed that was in a given topic.')
    st.markdown('**Topic Frequency**: The number of articles in a given topic.')
    
if submitted_ed:
    frequency_df, topics = create_frequency_dataframe(dataframe = df, 
                                                    default = default, 
                                                    specific = specific, 
                                                    year_start = year_start_tp, 
                                                    year_stop = year_stop_tp)
    frequency_plot = create_plot(frequency_df, 'Number of Articles in Topic', 'Topic Frequency by Year')
    prevalence_df = create_prevalence_dataframe(frequency_df)
    prevalence_plot = create_plot(prevalence_df, ylabel='Percent of all Articles (%)', title='Proportion of Articles in Topics by Year')
    
    # Display data
    col1, col2 = st.columns([3, 1])
    
    with col1:
        st.header('Proportion of Articles in Topics by Year - Graph')
        st.pyplot(prevalence_plot)
        st.divider()

        st.header('Proportion of Articles in Topics by Year - Table')
        st.dataframe(prevalence_df.T)
        st.divider()
    
        st.header('Topic Frequency by Year - Graph')
        st.pyplot(frequency_plot)
        st.divider()
        
        st.header('Topic Frequency by Year - Table')
        st.dataframe(frequency_df.T)
        st.divider()
    
    with col2:
        st.header('Topic Information')
        topic_words, _, topic_nums = model.get_topics(reduced=True)
        for topic in topics:
            st.write(f'Topic {topic}:', ', '.join(topic_words[topic][:10]))
            st.divider()