File size: 7,006 Bytes
d62ad66 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 | import streamlit as st
import pandas as pd
import datetime
from top2vec import Top2Vec
from matplotlib import pyplot as plt
st.set_page_config(layout='wide')
def create_frequency_dataframe(dataframe, default, specific, year_start=1976, year_stop=2000):
'''Produces a dataframe with the number of articles in each topic in a each year.
Parameters
dataframe: Pandas dataframe containing the articles, abstracts, and dates.
default: String indicating which topics to include in the dataframe. Selected by user.
specific: String of additional topic numbers to include in the dataframe. Selected by user. Added to default.
year_start: Integer indicating the first year to include in the dataframe. Selected by user.
year_stop: Integer indicating the last year to include in the dataframe. Selected by user.
Returns
date_df: Pandas dataframe with the number of articles in each topic in a each year.
topic_columns: List of topic numbers included in the dataframe.
'''
topic_sizes, topic_nums = model.get_topic_sizes(reduced=True)
topic_columns = []
if default == 'Top 5':
topic_columns = topic_nums[:5]
elif default == 'Top 10':
topic_columns = topic_nums[:10]
elif default == 'Bottom 5':
topic_columns = topic_nums[-5:]
elif default == 'Bottom 10':
topic_columns = topic_nums[-10:]
topic_columns= list(topic_columns) + [int(i) for i in specific.split() if len(i) > 0] # Break input text into list of ints then merge lists
# Format df for number of articles in each topic in a given year
date_df = pd.DataFrame(index=range(year_start, year_stop + 1), columns=topic_columns)
for col in date_df.columns:
date_df[col].values[:] = 0 # Covert values from null to 0
#Year/index num correspondence
# index_dict = {}
# for i, year in enumerate(date_df.index.values):
# index_dict[year] = i
# Get doc_ids for each requested topic
for topic in topic_columns:
_, document_ids = model.search_documents_by_topic(topic_num=topic, reduced=True, num_docs=topic_sizes[topic]) # Works because topic number corresponds to its order in the topic size list
for id in document_ids:
if dataframe.at[id, 'DP'] >= datetime.date(year_start,1,1).year and dataframe.at[id, 'DP'] < datetime.date(year_stop + 1,1,1).year:
date_df.at[dataframe.at[id, 'DP'], topic] += 1
return date_df, topic_columns
def create_prevalence_dataframe(date_df):
date_df = date_df.join(date_df.apply(sum, axis=1).reindex_like(date_df.index.to_series()).rename('Sums')) # Sum the total number of articles in each year
# Find the percentage of total number of articles represented by each topic in each year
for i, row in zip(date_df.index.values, date_df['Sums']):
if row != 0:
date_df.loc[i, :] = (date_df.loc[i, :] / date_df.loc[i, 'Sums']) * 100
date_df = date_df.drop(columns='Sums')
return date_df
def create_plot(date_df, ylabel, title):
fig, ax = plt.subplots(figsize=(10, 5))
for topic in date_df.columns:
ax.plot(date_df.index.values, date_df.loc[:, topic].values, label=f'Topic {topic}')
ax.xaxis.set_major_locator(plt.MaxNLocator(integer=True)) # Only show whole numbers on x-axis
ax.set_xlabel('Year')
ax.set_ylabel(ylabel)
ax.set_title(title)
ax.legend(bbox_to_anchor=(1, 1))
ax.grid(True)
return fig
@st.cache_resource
def load_model():
name = '2024-02-14_22_19_09_reduced__model_medline0[test,mincount_250,topicmergedelta_0.15,ngram_True,ngramvocab_args_250_.4_80M_npmi_connector].json'
with open(name, 'rb') as f:
return Top2Vec.load(f)
@st.cache_data
def load_data():
df = pd.read_pickle('medline_0_df_nlp.pkl.bz2', compression='bz2')
return df
# Notification of loading
start_container = st.empty()
if 'first' not in st.session_state:
st.session_state['first'] = True
if st.session_state['first'] == True:
start_container.text('Loading model and data... This may take two to three minutes on the initial run.')
st.session_state['first'] = False
# Load the model and data
model = load_model()
df = load_data()
# Remove loading notification
start_container.empty()
# User Interface
with st.sidebar.form('Topic Prevalence', clear_on_submit=False):
default = st.radio('Default Topics', ['Top 5', 'Top 10', 'Bottom 5', 'Bottom 10', 'None'])
specific = st.text_input('Topic Numbers', help='''Enter a series of topic numbers to graph.
The numbers should be separated by a space.
Topic numbers are assigned in order of decreasing frequency across the corpus.
In other words, Topic 0 is the most frequent topic, Topic 1 is the second most frequent, etc.''')
year_start_tp, year_stop_tp = st.slider('Year Range',
min_value=1975,
max_value=2000,
value=[1975, 2000])
submitted_ed = st.form_submit_button("Submit")
st.markdown('### Glossary')
st.markdown('**Proportion of Articles:** The percentage of the total number of articles displayed that was in a given topic.')
st.markdown('**Topic Frequency**: The number of articles in a given topic.')
if submitted_ed:
frequency_df, topics = create_frequency_dataframe(dataframe = df,
default = default,
specific = specific,
year_start = year_start_tp,
year_stop = year_stop_tp)
frequency_plot = create_plot(frequency_df, 'Number of Articles in Topic', 'Topic Frequency by Year')
prevalence_df = create_prevalence_dataframe(frequency_df)
prevalence_plot = create_plot(prevalence_df, ylabel='Percent of all Articles (%)', title='Proportion of Articles in Topics by Year')
# Display data
col1, col2 = st.columns([3, 1])
with col1:
st.header('Proportion of Articles in Topics by Year - Graph')
st.pyplot(prevalence_plot)
st.divider()
st.header('Proportion of Articles in Topics by Year - Table')
st.dataframe(prevalence_df.T)
st.divider()
st.header('Topic Frequency by Year - Graph')
st.pyplot(frequency_plot)
st.divider()
st.header('Topic Frequency by Year - Table')
st.dataframe(frequency_df.T)
st.divider()
with col2:
st.header('Topic Information')
topic_words, _, topic_nums = model.get_topics(reduced=True)
for topic in topics:
st.write(f'Topic {topic}:', ', '.join(topic_words[topic][:10]))
st.divider()
|