import streamlit as st import pandas as pd import datetime from top2vec import Top2Vec from matplotlib import pyplot as plt st.set_page_config(layout='wide') def create_frequency_dataframe(dataframe, default, specific, year_start=1976, year_stop=2000): '''Produces a dataframe with the number of articles in each topic in a each year. Parameters dataframe: Pandas dataframe containing the articles, abstracts, and dates. default: String indicating which topics to include in the dataframe. Selected by user. specific: String of additional topic numbers to include in the dataframe. Selected by user. Added to default. year_start: Integer indicating the first year to include in the dataframe. Selected by user. year_stop: Integer indicating the last year to include in the dataframe. Selected by user. Returns date_df: Pandas dataframe with the number of articles in each topic in a each year. topic_columns: List of topic numbers included in the dataframe. ''' topic_sizes, topic_nums = model.get_topic_sizes(reduced=True) topic_columns = [] if default == 'Top 5': topic_columns = topic_nums[:5] elif default == 'Top 10': topic_columns = topic_nums[:10] elif default == 'Bottom 5': topic_columns = topic_nums[-5:] elif default == 'Bottom 10': topic_columns = topic_nums[-10:] topic_columns= list(topic_columns) + [int(i) for i in specific.split() if len(i) > 0] # Break input text into list of ints then merge lists # Format df for number of articles in each topic in a given year date_df = pd.DataFrame(index=range(year_start, year_stop + 1), columns=topic_columns) for col in date_df.columns: date_df[col].values[:] = 0 # Covert values from null to 0 #Year/index num correspondence # index_dict = {} # for i, year in enumerate(date_df.index.values): # index_dict[year] = i # Get doc_ids for each requested topic for topic in topic_columns: _, document_ids = model.search_documents_by_topic(topic_num=topic, reduced=True, num_docs=topic_sizes[topic]) # Works because topic number corresponds to its order in the topic size list for id in document_ids: if dataframe.at[id, 'DP'] >= datetime.date(year_start,1,1).year and dataframe.at[id, 'DP'] < datetime.date(year_stop + 1,1,1).year: date_df.at[dataframe.at[id, 'DP'], topic] += 1 return date_df, topic_columns def create_prevalence_dataframe(date_df): date_df = date_df.join(date_df.apply(sum, axis=1).reindex_like(date_df.index.to_series()).rename('Sums')) # Sum the total number of articles in each year # Find the percentage of total number of articles represented by each topic in each year for i, row in zip(date_df.index.values, date_df['Sums']): if row != 0: date_df.loc[i, :] = (date_df.loc[i, :] / date_df.loc[i, 'Sums']) * 100 date_df = date_df.drop(columns='Sums') return date_df def create_plot(date_df, ylabel, title): fig, ax = plt.subplots(figsize=(10, 5)) for topic in date_df.columns: ax.plot(date_df.index.values, date_df.loc[:, topic].values, label=f'Topic {topic}') ax.xaxis.set_major_locator(plt.MaxNLocator(integer=True)) # Only show whole numbers on x-axis ax.set_xlabel('Year') ax.set_ylabel(ylabel) ax.set_title(title) ax.legend(bbox_to_anchor=(1, 1)) ax.grid(True) return fig @st.cache_resource def load_model(): name = '2024-02-14_22_19_09_reduced__model_medline0[test,mincount_250,topicmergedelta_0.15,ngram_True,ngramvocab_args_250_.4_80M_npmi_connector].json' with open(name, 'rb') as f: return Top2Vec.load(f) @st.cache_data def load_data(): df = pd.read_pickle('medline_0_df_nlp.pkl.bz2', compression='bz2') return df # Notification of loading start_container = st.empty() if 'first' not in st.session_state: st.session_state['first'] = True if st.session_state['first'] == True: start_container.text('Loading model and data... This may take two to three minutes on the initial run.') st.session_state['first'] = False # Load the model and data model = load_model() df = load_data() # Remove loading notification start_container.empty() # User Interface with st.sidebar.form('Topic Prevalence', clear_on_submit=False): default = st.radio('Default Topics', ['Top 5', 'Top 10', 'Bottom 5', 'Bottom 10', 'None']) specific = st.text_input('Topic Numbers', help='''Enter a series of topic numbers to graph. The numbers should be separated by a space. Topic numbers are assigned in order of decreasing frequency across the corpus. In other words, Topic 0 is the most frequent topic, Topic 1 is the second most frequent, etc.''') year_start_tp, year_stop_tp = st.slider('Year Range', min_value=1975, max_value=2000, value=[1975, 2000]) submitted_ed = st.form_submit_button("Submit") st.markdown('### Glossary') st.markdown('**Proportion of Articles:** The percentage of the total number of articles displayed that was in a given topic.') st.markdown('**Topic Frequency**: The number of articles in a given topic.') if submitted_ed: frequency_df, topics = create_frequency_dataframe(dataframe = df, default = default, specific = specific, year_start = year_start_tp, year_stop = year_stop_tp) frequency_plot = create_plot(frequency_df, 'Number of Articles in Topic', 'Topic Frequency by Year') prevalence_df = create_prevalence_dataframe(frequency_df) prevalence_plot = create_plot(prevalence_df, ylabel='Percent of all Articles (%)', title='Proportion of Articles in Topics by Year') # Display data col1, col2 = st.columns([3, 1]) with col1: st.header('Proportion of Articles in Topics by Year - Graph') st.pyplot(prevalence_plot) st.divider() st.header('Proportion of Articles in Topics by Year - Table') st.dataframe(prevalence_df.T) st.divider() st.header('Topic Frequency by Year - Graph') st.pyplot(frequency_plot) st.divider() st.header('Topic Frequency by Year - Table') st.dataframe(frequency_df.T) st.divider() with col2: st.header('Topic Information') topic_words, _, topic_nums = model.get_topics(reduced=True) for topic in topics: st.write(f'Topic {topic}:', ', '.join(topic_words[topic][:10])) st.divider()