Download pages/04_Topic_Prevalence.py from Array3022/PubMedTM: direct link, hf CLI and curl.
- Browser
- Download file 7.01 kB
-
https://huggingface.co/spaces/Array3022/PubMedTM/resolve/main/pages/04_Topic_Prevalence.py
- Command line
-
hf download hf://spaces/Array3022/PubMedTM/pages/04_Topic_Prevalence.py
-
curl -L -o 04_Topic_Prevalence.py https://huggingface.co/spaces/Array3022/PubMedTM/resolve/main/pages/04_Topic_Prevalence.py
7.01 kB
| import streamlit as st | |
| import pandas as pd | |
| import datetime | |
| from top2vec import Top2Vec | |
| from matplotlib import pyplot as plt | |
| st.set_page_config(layout='wide') | |
| def create_frequency_dataframe(dataframe, default, specific, year_start=1976, year_stop=2000): | |
| '''Produces a dataframe with the number of articles in each topic in a each year. | |
| Parameters | |
| dataframe: Pandas dataframe containing the articles, abstracts, and dates. | |
| default: String indicating which topics to include in the dataframe. Selected by user. | |
| specific: String of additional topic numbers to include in the dataframe. Selected by user. Added to default. | |
| year_start: Integer indicating the first year to include in the dataframe. Selected by user. | |
| year_stop: Integer indicating the last year to include in the dataframe. Selected by user. | |
| Returns | |
| date_df: Pandas dataframe with the number of articles in each topic in a each year. | |
| topic_columns: List of topic numbers included in the dataframe. | |
| ''' | |
| topic_sizes, topic_nums = model.get_topic_sizes(reduced=True) | |
| topic_columns = [] | |
| if default == 'Top 5': | |
| topic_columns = topic_nums[:5] | |
| elif default == 'Top 10': | |
| topic_columns = topic_nums[:10] | |
| elif default == 'Bottom 5': | |
| topic_columns = topic_nums[-5:] | |
| elif default == 'Bottom 10': | |
| topic_columns = topic_nums[-10:] | |
| topic_columns= list(topic_columns) + [int(i) for i in specific.split() if len(i) > 0] # Break input text into list of ints then merge lists | |
| # Format df for number of articles in each topic in a given year | |
| date_df = pd.DataFrame(index=range(year_start, year_stop + 1), columns=topic_columns) | |
| for col in date_df.columns: | |
| date_df[col].values[:] = 0 # Covert values from null to 0 | |
| #Year/index num correspondence | |
| # index_dict = {} | |
| # for i, year in enumerate(date_df.index.values): | |
| # index_dict[year] = i | |
| # Get doc_ids for each requested topic | |
| for topic in topic_columns: | |
| _, document_ids = model.search_documents_by_topic(topic_num=topic, reduced=True, num_docs=topic_sizes[topic]) # Works because topic number corresponds to its order in the topic size list | |
| for id in document_ids: | |
| if dataframe.at[id, 'DP'] >= datetime.date(year_start,1,1).year and dataframe.at[id, 'DP'] < datetime.date(year_stop + 1,1,1).year: | |
| date_df.at[dataframe.at[id, 'DP'], topic] += 1 | |
| return date_df, topic_columns | |
| def create_prevalence_dataframe(date_df): | |
| date_df = date_df.join(date_df.apply(sum, axis=1).reindex_like(date_df.index.to_series()).rename('Sums')) # Sum the total number of articles in each year | |
| # Find the percentage of total number of articles represented by each topic in each year | |
| for i, row in zip(date_df.index.values, date_df['Sums']): | |
| if row != 0: | |
| date_df.loc[i, :] = (date_df.loc[i, :] / date_df.loc[i, 'Sums']) * 100 | |
| date_df = date_df.drop(columns='Sums') | |
| return date_df | |
| def create_plot(date_df, ylabel, title): | |
| fig, ax = plt.subplots(figsize=(10, 5)) | |
| for topic in date_df.columns: | |
| ax.plot(date_df.index.values, date_df.loc[:, topic].values, label=f'Topic {topic}') | |
| ax.xaxis.set_major_locator(plt.MaxNLocator(integer=True)) # Only show whole numbers on x-axis | |
| ax.set_xlabel('Year') | |
| ax.set_ylabel(ylabel) | |
| ax.set_title(title) | |
| ax.legend(bbox_to_anchor=(1, 1)) | |
| ax.grid(True) | |
| return fig | |
| def load_model(): | |
| name = '2024-02-14_22_19_09_reduced__model_medline0[test,mincount_250,topicmergedelta_0.15,ngram_True,ngramvocab_args_250_.4_80M_npmi_connector].json' | |
| with open(name, 'rb') as f: | |
| return Top2Vec.load(f) | |
| def load_data(): | |
| df = pd.read_pickle('medline_0_df_nlp.pkl.bz2', compression='bz2') | |
| return df | |
| # Notification of loading | |
| start_container = st.empty() | |
| if 'first' not in st.session_state: | |
| st.session_state['first'] = True | |
| if st.session_state['first'] == True: | |
| start_container.text('Loading model and data... This may take two to three minutes on the initial run.') | |
| st.session_state['first'] = False | |
| # Load the model and data | |
| model = load_model() | |
| df = load_data() | |
| # Remove loading notification | |
| start_container.empty() | |
| # User Interface | |
| with st.sidebar.form('Topic Prevalence', clear_on_submit=False): | |
| default = st.radio('Default Topics', ['Top 5', 'Top 10', 'Bottom 5', 'Bottom 10', 'None']) | |
| specific = st.text_input('Topic Numbers', help='''Enter a series of topic numbers to graph. | |
| The numbers should be separated by a space. | |
| Topic numbers are assigned in order of decreasing frequency across the corpus. | |
| In other words, Topic 0 is the most frequent topic, Topic 1 is the second most frequent, etc.''') | |
| year_start_tp, year_stop_tp = st.slider('Year Range', | |
| min_value=1975, | |
| max_value=2000, | |
| value=[1975, 2000]) | |
| submitted_ed = st.form_submit_button("Submit") | |
| st.markdown('### Glossary') | |
| st.markdown('**Proportion of Articles:** The percentage of the total number of articles displayed that was in a given topic.') | |
| st.markdown('**Topic Frequency**: The number of articles in a given topic.') | |
| if submitted_ed: | |
| frequency_df, topics = create_frequency_dataframe(dataframe = df, | |
| default = default, | |
| specific = specific, | |
| year_start = year_start_tp, | |
| year_stop = year_stop_tp) | |
| frequency_plot = create_plot(frequency_df, 'Number of Articles in Topic', 'Topic Frequency by Year') | |
| prevalence_df = create_prevalence_dataframe(frequency_df) | |
| prevalence_plot = create_plot(prevalence_df, ylabel='Percent of all Articles (%)', title='Proportion of Articles in Topics by Year') | |
| # Display data | |
| col1, col2 = st.columns([3, 1]) | |
| with col1: | |
| st.header('Proportion of Articles in Topics by Year - Graph') | |
| st.pyplot(prevalence_plot) | |
| st.divider() | |
| st.header('Proportion of Articles in Topics by Year - Table') | |
| st.dataframe(prevalence_df.T) | |
| st.divider() | |
| st.header('Topic Frequency by Year - Graph') | |
| st.pyplot(frequency_plot) | |
| st.divider() | |
| st.header('Topic Frequency by Year - Table') | |
| st.dataframe(frequency_df.T) | |
| st.divider() | |
| with col2: | |
| st.header('Topic Information') | |
| topic_words, _, topic_nums = model.get_topics(reduced=True) | |
| for topic in topics: | |
| st.write(f'Topic {topic}:', ', '.join(topic_words[topic][:10])) | |
| st.divider() | |