Download pages/05_Text_Analysis.py from Array3022/PubMedTM: direct link, hf CLI and curl.
- Browser
- Download file 26.3 kB
-
https://huggingface.co/spaces/Array3022/PubMedTM/resolve/main/pages/05_Text_Analysis.py
- Command line
-
hf download hf://spaces/Array3022/PubMedTM/pages/05_Text_Analysis.py
-
curl -L -o 05_Text_Analysis.py https://huggingface.co/spaces/Array3022/PubMedTM/resolve/main/pages/05_Text_Analysis.py
26.3 kB
| import streamlit as st | |
| import pandas as pd | |
| from matplotlib import pyplot as plt | |
| import stanza | |
| from stanza import DownloadMethod | |
| # Load data | |
| def load_data(): | |
| df = pd.read_pickle('medline_0_df_nlp.pkl.bz2', compression='bz2') | |
| return df | |
| # Notification of loading | |
| start_container = st.empty() | |
| if 'first' not in st.session_state: | |
| st.session_state['first'] = True | |
| if st.session_state['first'] == True: | |
| start_container.text('Loading model and data... This may take two to three minutes on the initial run.') | |
| st.session_state['first'] = False | |
| # Load the model and data | |
| df = load_data() | |
| # Remove loading notification | |
| start_container.empty() | |
| def load_nlp(): | |
| # Download and initialize a genia pipeline with an anatem NER model | |
| nlp = stanza.Pipeline('en', package='genia', processors=['tokenize', 'pos', 'lemma'], | |
| use_gpu=True, ner_batch_size=128, tokenize_batch_size=128, | |
| download_method=DownloadMethod.REUSE_RESOURCES, logging_level='WARN') | |
| return nlp | |
| ###### Helper Functions ###### | |
| def filter_by_year(data, year_start, year_end): | |
| if year_start == None and year_end == None: | |
| return data | |
| elif year_start == None: | |
| return data[data['DP'] <= year_end] | |
| elif year_end == None: | |
| return data[data['DP'] >= year_start] | |
| else: | |
| return data.loc[(data['DP'] >= year_start) & (data['DP'] <= year_end)] | |
| def filter_tuples_by_entity_type(data, entity_type): | |
| '''Filters out tuples in the NER column based on the specified entity type.''' | |
| if entity_type == 'ALL': # Return all data | |
| return data | |
| data['NER'] = data['NER'].apply(lambda ner_list: [tup for tup in ner_list if tup[1] == entity_type]) # Return only tuples with the specified entity type | |
| return data | |
| def filter_by_country(data, country): | |
| data = data[data['PL'] == country] | |
| return data | |
| def group_entity_by_year(data): | |
| data_explode = data.explode('NER') | |
| data_explode['NER'] = data_explode['NER'].apply(lambda x: x[0] if not type(x)==float else None) # Return none if the entity is NaN (which is seen as a float) | |
| data_explode = data_explode.dropna(subset=['NER']) # Drop rows with None as the entity | |
| records = zip(data_explode['DP'], data_explode['NER']) | |
| entitiesDict = {} # {NER: [{year: count}, total_count] | |
| for yr, name in records: | |
| if name not in entitiesDict: | |
| entitiesDict[name] = [{}, 0] | |
| entityDict = entitiesDict[name][0] | |
| if yr not in entityDict: | |
| entityDict[yr] = 0 | |
| entityDict[yr] += 1 | |
| entitiesDict[name][1] += 1 | |
| return entitiesDict | |
| def group_journal_by_year(data): | |
| records = zip(data['DP'], data['JID'], data['JT']) | |
| journalsDict = {} # {JID: [{DP: count}, total_count, JT] | |
| for yr, jid, title in records: | |
| if jid not in journalsDict: | |
| journalsDict[jid] = [{}, 0, title] | |
| journalDict = journalsDict[jid][0] | |
| if yr not in journalDict: | |
| journalDict[yr] = 0 | |
| journalDict[yr] += 1 | |
| journalsDict[jid][1] += 1 | |
| return journalsDict | |
| def group_mesh_by_year(data): | |
| data_explode = data.explode('MH') | |
| data_explode = data_explode.dropna(subset=['MH']) # Drop rows with NaN as the entity | |
| records = zip(data_explode['DP'], data_explode['MH']) | |
| meshesDict = {} | |
| for yr, word in records: | |
| # Add mesh term if it does not exist in the dictionary | |
| if word not in meshesDict: | |
| meshesDict[word] = [{}, 0] # Create new entry prepared for the year count. {word: [{year: count}, total_count]} | |
| # Update the year count | |
| single_meshDict = meshesDict[word][0] # Access the subdictionary in each entry | |
| if yr not in single_meshDict: | |
| single_meshDict[yr] = 0 | |
| single_meshDict[yr] += 1 | |
| meshesDict[word][1] += 1 # Update the total count | |
| return meshesDict | |
| # def find_occurrence(list, target): # Finds the index of the first occurrence of an element in a list which contains tuples | |
| # index = 0 | |
| # for element in list: | |
| # if element[0] == target: | |
| # return index | |
| # index += 1 | |
| ###### Collocation Functions ###### | |
| def find_collocation(word, texts, n): | |
| # Could work on allowing user to enter list of words. | |
| # Could remove small strings (e.g. "'s") from ngrams. | |
| # 'America' and 'American' are different words, but 'America' and 'Americans' lemmatize to different words, which is somewhat unexpected behavior to me. | |
| ''' | |
| Finds neighboring words/n-grams of given word in text. | |
| Parameters: | |
| word (str): Word to find n-grams of. | |
| texts (list): List of strings to search for n-grams. | |
| n (int): Number of words to include in n-gram. | |
| Returns: | |
| (dict): Dictionary of n-grams with word as key and count as value. | |
| ''' | |
| # Find n-grams and create dictionary of n-grams | |
| ngrams = {} | |
| for text in texts: | |
| if word in text: | |
| i = text.index(word) | |
| for j in range(1, n+1): # Get required number of words before and after word based on n-gram count specified | |
| if i-j >= 0: | |
| if text[i-j] in ngrams: | |
| ngrams[text[i-j]] += 1 | |
| else: | |
| ngrams[text[i-j]] = 1 | |
| try: | |
| if text[i+j] in ngrams: | |
| ngrams[text[i+j]] += 1 | |
| else: | |
| ngrams[text[i+j]] = 1 | |
| except IndexError: | |
| pass | |
| return dict(sorted(ngrams.items(), key=lambda x:x[1], reverse=True)) | |
| # def create_plot(dictionary, word, number_of_items=10): | |
| # '''Creates a bar chart of the frequency of the items in the dictionary''' | |
| # fig, ax = plt.subplots() | |
| # ax.bar(x=list(dictionary.keys())[:number_of_items], height=list(dictionary.values())[:number_of_items], align='center') | |
| # # ax.set_xticklabels(labels=list(dictionary.keys())[:number_of_items], rotation=45) | |
| # ax.tick_params(axis='x', rotation=45) # Rotates x-axis labels | |
| # ax.set_xlabel('Words') | |
| # ax.set_ylabel('Frequency') | |
| # ax.set_title('Frequency of Words Surrounding ' + '"' + word + '"') | |
| # return fig | |
| def run_collocation(word, degree, dataframe=df, number_of_items=10): | |
| nlp = load_nlp() | |
| # Lemmatize input word | |
| process_word = nlp(word) | |
| lemma = process_word.sentences[0].words[0].lemma | |
| # Find ngrams | |
| ngrams = find_collocation(word=lemma, texts=dataframe['lemma_AB'], n=degree) | |
| # Plot ngrams | |
| fig, ax = plt.subplots() | |
| ax.bar(x=list(ngrams.keys())[:number_of_items], height=list(ngrams.values())[:number_of_items], align='center') | |
| # ax.set_xticklabels(labels=list(dictionary.keys())[:number_of_items], rotation=45) | |
| ax.tick_params(axis='x', rotation=45) # Rotates x-axis labels | |
| ax.set_xlabel('Words') | |
| ax.set_ylabel('Frequency') | |
| ax.set_title('Frequency of Words Surrounding ' + '"' + word + '"') | |
| return fig | |
| # return create_plot(ngrams, word) | |
| ###### Journal Analytics Functions ###### | |
| def jid_frequency(data=df, year_start=1975, year_end=2000, n=15): | |
| ''' | |
| Returns the frequency of the top n most common journal IDs in a given year range. | |
| ''' | |
| jid_dict = {} | |
| added = set() # List used to keep track of all of the items which have already been added to the jid_dict | |
| dates = data['DP'].tolist() | |
| jid = data['JID'].tolist() | |
| index = 0 | |
| for item in jid: | |
| if dates[index] >= year_start and dates[index] <= year_end: | |
| if item in added: # jid is already in the dictionary | |
| jid_dict[item] += 1 | |
| else: | |
| jid_dict[item] = 1 | |
| added.add(item) | |
| index += 1 | |
| # display(jid_dict) # Used for testing and debugging | |
| jid_df = pd.DataFrame.from_dict(jid_dict, orient='index', columns=['Frequency']) | |
| jid_df.sort_values(by=['Frequency'], inplace=True, ascending=False) | |
| jid_df = jid_df.head(n) | |
| # Create matplotlib figure from dataframe | |
| fig, ax = plt.subplots() | |
| ax.barh(y=jid_df.index, width=jid_df['Frequency'], align='center') | |
| ax.bar_label(ax.containers[0], fmt='%d', label_type='edge') | |
| ax.set_xlim(right=jid_df['Frequency'].max() + jid_df['Frequency'].max()*0.08) # Prevents labels from overlapping with border in most cases; increase the number if needed | |
| ax.set_title('Top {} Most Common JID\'s of Publications'.format(n)) | |
| ax.set_xlabel('Number of publications') | |
| ax.set_ylabel('Journal NlmId/JID') | |
| return fig | |
| # jid_df.plot(kind='barh', figsize=(10, 10), | |
| # title='Top {} Most Common JID\'s of Publications'.format(n), | |
| # legend=True, | |
| # xlabel='Number of publications', | |
| # ylabel='Journal NlmId/JID', | |
| # ) | |
| # # display(jid_df) | |
| # return(jid_df) | |
| def jid_frequency_by_year(data=df, year_start=1975, year_end=2000, n=15): | |
| ''' | |
| Creates a line graph showing the frequency at which the most common n JIDs appear in the dataset for each year in given year range. | |
| Parameters | |
| data: the medline dataframe | |
| n: int. The number of journals to display. | |
| year_start: int. The start of the year range to display. Years for each data point are taken from the 'DP' column. | |
| year_end: int. The end of the year range to display. Can be the same as year_start. | |
| Returns | |
| Matplotib figure object. | |
| ''' | |
| filtered_by_year = filter_by_year(data, year_start, year_end) | |
| journalsDict = group_journal_by_year(filtered_by_year) | |
| sortedJournals = sorted(journalsDict.items(), key = lambda x: x[1][1], reverse = True) | |
| topJournals = sortedJournals[0:n] | |
| df = pd.DataFrame() | |
| # Convert the dictionary to a Series and assign it to a new column in the DataFrame | |
| for jid, data in topJournals: | |
| yearly_data = data[0] | |
| # title = data[2] | |
| df[jid] = pd.Series(yearly_data) | |
| # Fill NaN values with 0, in case some entities don't have data for all years | |
| df = df.fillna(0) | |
| # Debug | |
| # display(df) | |
| # Create a figure and axes | |
| fig, ax = plt.subplots() | |
| # Plot the data on the axes objects | |
| for column in df.columns: | |
| x, y = zip(*sorted(zip(df.index, df[column].values))) # Must sort the data before plotting it; https://stackoverflow.com/questions/37414916/matplotlib-plotting-in-wrong-order | |
| ax.plot(x, y, marker='o', label=column) | |
| # ax.plot(df.index, df[column].values) | |
| # Set plot details | |
| ax.set_title("Journal Frequencies by Year") | |
| ax.set_xlabel("Year") | |
| ax.set_ylabel("Frequency") | |
| ax.grid(True) | |
| ax.legend() | |
| # ax.set_xticks([tick for tick in ax.get_xticks() if tick.is_integer()]) | |
| # ax.set_yticks([tick for tick in ax.get_yticks() if tick.is_integer()]) | |
| return fig | |
| def pl_frequency(data=df, year_start=1975, year_end=2000, n=15): | |
| ''' | |
| Returns the frequency of the top n most common places of publication in a given year range. | |
| ''' | |
| pl_dict = {} | |
| added = set() # List used to keep track of all of the items which have already been added to the ner_dict | |
| dates = data['DP'].tolist() | |
| pl = data['PL'].tolist() | |
| # display(pl) | |
| index = 0 | |
| for item in pl: | |
| if dates[index] >= year_start and dates[index] <= year_end: | |
| if item in added: # pl is already in the dictionary | |
| pl_dict[item] += 1 | |
| else: | |
| pl_dict[item] = 1 | |
| added.add(item) | |
| index += 1 | |
| #display(pl_dict) # Used for testing and debugging | |
| pl_df = pd.DataFrame.from_dict( | |
| pl_dict, orient='index', columns=['Frequency']) | |
| pl_df.sort_values(by=['Frequency'], inplace=True, ascending=False) | |
| pl_df = pl_df.head(n) | |
| # Create matplotlib figure from dataframe | |
| fig, ax = plt.subplots() | |
| ax.barh(y=pl_df.index, width=pl_df['Frequency'], align='center') | |
| ax.bar_label(ax.containers[0], fmt='%d', label_type='edge') | |
| ax.set_xlim(right=pl_df['Frequency'].max() + pl_df['Frequency'].max()*0.10) # Prevents labels from overlapping with border in most cases; increase the number if needed | |
| ax.set_title('Top {} Most Common Places of Publications'.format(n)) | |
| ax.set_xlabel('Number of publications') | |
| ax.set_ylabel('Journal NlmId/JID') | |
| return fig | |
| ###### Named Biological Entities Functions ###### | |
| def ner_frequency(data=df, n=15, entity_toggle='ALL', year_start=1975, year_end=2000): | |
| """ | |
| Displays the frequency of the top n most common named entities in a graph. | |
| Parameters | |
| data: the medline dataframe (or a subset of it) | |
| n: int. The number of entities to display. | |
| entity_toggle: str. The type of entity to display. This info is stored in the tuples in the 'ner' column. | |
| Options are 'ALL', 'ANATOMY', 'DISEASE'. If 'ALL', displays all entities. | |
| year_start: int. The start of the year range to display. | |
| year_end: int. The end of the year range to display. Can be the same as year_start. | |
| """ | |
| # Function can be slow. Could be improved by using group_entity_by_year() then counting the number of times each year appears. | |
| ner_dict = {} | |
| added = set() # List used to keep track of all of the items which have already been added to the ner_dict | |
| dates = data['DP'].tolist() | |
| ner = data['NER'].tolist() | |
| index = 0 | |
| for item in ner: | |
| if dates[index] >= year_start and dates[index] <= year_end: | |
| for tup in item: | |
| if entity_toggle == 'ALL' or entity_toggle == tup[1]: | |
| ner = tup[0] | |
| if ner in added: # ner is already in the dictionary | |
| ner_dict[ner] += 1 | |
| else: | |
| ner_dict[ner] = 1 | |
| added.add(ner) | |
| index += 1 | |
| # display(ner_dict) # Used for testing and debugging | |
| ner_df = pd.DataFrame.from_dict( | |
| ner_dict, orient='index', columns=['Frequency']) | |
| ner_df.sort_values(by=['Frequency'], inplace=True, ascending=False) | |
| ner_df = ner_df.head(n) | |
| fig, ax = plt.subplots() | |
| ax.barh(y=ner_df.index, width=ner_df['Frequency'], align='center') | |
| ax.bar_label(ax.containers[0], fmt='%d', label_type='edge') | |
| ax.set_xlim(right=ner_df['Frequency'].max() + ner_df['Frequency'].max()*0.08) # Prevents labels from overlapping with border in most cases; increase the number if needed | |
| if entity_toggle == 'ALL': | |
| ax.set_title('Top {} Most Common Named Biological Entities'.format(n)) | |
| else: | |
| ax.set_title('Top {} Most Common Named {} Entities'.format(n, entity_toggle.lower().capitalize())) | |
| ax.set_xlabel('Number of publications') | |
| ax.set_ylabel('Journal NlmId/JID') | |
| return fig | |
| def ner_frequency_by_year(data=df, n=15, entity_toggle='ALL', year_start=1975, year_end=2000) : | |
| """ | |
| Creates a line graph showing the frequency at which the most common n entities appear in the dataset for each year in given year range. | |
| In other words, X axis = year, Y axis = frequency, and each line represents a different entity. | |
| Parameters | |
| data: the medline dataframe | |
| n: int. The number of entities to display. | |
| entity_toggle: str. The type of entity to display. This info is stored in the tuples in the 'NER' column. | |
| Options are 'ALL', 'ANATOMY', 'DISEASE'. If 'ALL', displays all entities. | |
| year_start: int. The start of the year range to display. Years for each data point are taken from the 'DP' column. | |
| year_end: int. The end of the year range to display. Can be the same as year_start. | |
| Returns | |
| Matplotib figure object. | |
| """ | |
| filtered_by_year = filter_by_year(data, year_start, year_end) | |
| filtered_by_entity = filter_tuples_by_entity_type(filtered_by_year, entity_toggle) | |
| entitiesDict = group_entity_by_year(filtered_by_entity) | |
| sortedEntities = sorted(entitiesDict.items(), key = lambda x: x[1][1], reverse = True) | |
| topEntities = sortedEntities[0:n] | |
| df = pd.DataFrame() | |
| print(topEntities) | |
| for entity, data in topEntities: | |
| # Convert the dictionary to a Series and assign it to a new column in the DataFrame | |
| yearly_data = data[0] | |
| df[entity] = pd.Series(yearly_data) | |
| # Fill NaN values with 0, in case some entities don't have data for all years | |
| df = df.fillna(0) | |
| print(df) | |
| fig, ax = plt.subplots() | |
| # Plot the data on the axes object | |
| for column in df.columns: | |
| x, y = zip(*sorted(zip(df.index, df[column].values))) # Must sort the data before plotting it; https://stackoverflow.com/questions/37414916/matplotlib-plotting-in-wrong-order | |
| ax.plot(x, y, marker='o', label=column) | |
| # Set plot details | |
| ax.set_title("Entity Frequencies by Year") | |
| ax.set_xlabel("Year") | |
| ax.set_ylabel("Frequency") | |
| ax.grid(True) | |
| ax.legend(loc='upper left') | |
| return fig | |
| ###### MeSH Terms Functions ###### | |
| def mh_frequency(country, globally=False, data=df, year_start=1975, year_end=2000, n=15): | |
| ''' | |
| Returns the frequency of the top n most common MeSH terms in a given year range. | |
| ''' | |
| mh_dict = {} | |
| added = set() # List used to keep track of all of the items which have already been added to the mh_dict | |
| if globally: | |
| pass | |
| else: | |
| data = filter_by_country(data, country) | |
| dates = data['DP'].tolist() | |
| mh = data['MH'].tolist() | |
| index = 0 | |
| for item in mh: | |
| if dates[index] >= year_start and dates[index] <= year_end: | |
| for i in item: | |
| if i in added: # mh is already in the dictionary | |
| mh_dict[i] += 1 | |
| else: | |
| mh_dict[i] = 1 | |
| added.add(i) | |
| index += 1 | |
| #display(mh_dict) # Used for testing and debugging | |
| mh_df = pd.DataFrame.from_dict( | |
| mh_dict, orient='index', columns=['Frequency']) | |
| mh_df.sort_values(by=['Frequency'], inplace=True, ascending=False) | |
| mh_df = mh_df.head(n) | |
| fig, ax = plt.subplots() | |
| ax.barh(y=mh_df.index, width=mh_df['Frequency'], align='center') | |
| ax.bar_label(ax.containers[0], fmt='%d', label_type='edge') | |
| ax.set_xlim(right=mh_df['Frequency'].max() + mh_df['Frequency'].max()*0.08) # Prevents labels from overlapping with border in most cases; increase the number if needed | |
| if globally: | |
| ax.set_title('Top {} Most Common MeSH Terms Globally'.format(n)) | |
| else: | |
| ax.set_title('Top {} Most Common MeSH Terms in {}'.format(n, country)) | |
| ax.set_xlabel('Number of occurrences') | |
| ax.set_ylabel('Term') | |
| return fig | |
| # mh_df.plot(kind='barh', figsize=(10, 10), title='Top ' + | |
| # str(n) + ' Most Common Places of Publication', legend=False) | |
| # #display(mh_df) | |
| # return(mh_df) | |
| def mh_time_series(country, globally=False, data=df, year_start=1975, year_end=2000, n=15): | |
| """ | |
| Creates a matplotlib line plot showing the frequency of the top n most common MeSH topics in a given country (or globally) for each year in a given range. | |
| The MH column contains MeSH terms (controlled library of words used to categorize articles). | |
| The PL column contains publication locations. | |
| Parameters | |
| country: str. The country to display. | |
| n: int. The number of MESH topics to display. | |
| year_start: int. The start of the year range to display. Years for each data point are taken from the 'DP' column. | |
| year_end: int. The end of the year range to display. Can be the same as year_start. | |
| Returns | |
| Matplotib figure object. | |
| # Stores the data in a dictionary with this format: | |
| pl_mh_dict = {} # {year: {(pl, mh): (count)}} Storing the pl is redundant, but I'd like to keep it for now in case of future changes | |
| """ | |
| filtered_by_year = filter_by_year(data, year_start, year_end) | |
| ### TODO: Check if this is fixed | |
| if globally: | |
| filtered_by_country = filtered_by_year | |
| else: | |
| filtered_by_country = filter_by_country(filtered_by_year, country) | |
| meshesDict = group_mesh_by_year(filtered_by_country) | |
| sortedMeshes = sorted(meshesDict.items(), key = lambda x: x[1][1], reverse = True) | |
| topMeshes = sortedMeshes[0:n] | |
| df = pd.DataFrame() | |
| for mesh, data in topMeshes: | |
| # Convert the dictionary to a Series and assign it to a new column in the DataFrame | |
| yearly_data = data[0] | |
| df[mesh] = pd.Series(yearly_data) | |
| # Fill NaN values with 0, in case some entities don't have data for all years | |
| df = df.fillna(0) | |
| fig, ax = plt.subplots() | |
| # Plot the data on the axes object | |
| for column in df.columns: | |
| x, y = zip(*sorted(zip(df.index, df[column].values))) | |
| ax.plot(x, y, marker='o', label=column) | |
| # ax.plot(df.index, df[column], marker='o', label=column) | |
| # Set plot details | |
| if globally: | |
| ax.set_title("MeSH Topic Frequencies by Year Globally") | |
| else: | |
| ax.set_title("MeSH Topic Frequencies by Year in " + country) | |
| ax.set_xlabel("Year") | |
| ax.set_ylabel("Frequency") | |
| ax.grid(True) | |
| ax.legend() | |
| return fig | |
| ###### User Interface ###### | |
| with st.sidebar: | |
| with st.expander('Word Collocation'): | |
| with st.form(key='collocation_form'): | |
| word = st.text_input(label='Enter a word to see the most common surrounding words', | |
| placeholder='Enter a keyword') | |
| degree = st.selectbox('Select the number of adjacent words', [1, 2, 3], | |
| help='''If you select 1, the output will include words immediately before and after the keyword. | |
| If you select 2, the output will include words two words before and after the keyword, etc.''') | |
| # Could also provide an option to override the lemmatization here | |
| submit_collocation = st.form_submit_button('Submit') | |
| with st.expander('Journal Analytics'): | |
| with st.form(key='journal_form'): | |
| year_start_journal, year_end_journal = st.slider(label='Select a year range', min_value=1975, max_value=2000, value=(1975, 2000), step=1) | |
| n_journal = st.number_input(label='Enter the number of journals to display', min_value=5, max_value=25, value=10, step=1) | |
| submit_journal = st.form_submit_button('Submit') | |
| with st.expander('Named Biological Entities'): | |
| with st.form(key='nbe_form'): | |
| entity = st.selectbox('Select an entity type', ['ALL', 'ANATOMY', 'DISEASE'], | |
| help='''The natural language processing model that extracts these entities categorizes them as either a disease or an anatomical term. This is an artifact of the corpus that trained the model.''') | |
| n_nbe = st.number_input(label='Enter the number of entities to display', min_value=5, max_value=25, value=10, step=1) | |
| year_start_nbe, year_end_nbe = st.slider(label='Select a year range', min_value=1975, max_value=2000, value=(1975, 2000), step=1) | |
| submit_nbe = st.form_submit_button('Submit') | |
| with st.expander('MeSH Terms'): | |
| with st.form(key='mesh_form'): | |
| year_start_mesh, year_end_mesh = st.slider(label='Select a year range', min_value=1975, max_value=2000, value=(1975, 2000), step=1) | |
| n_mesh = st.number_input(label='Enter the number of MeSH terms to display', min_value=5, max_value=25, value=10, step=1) | |
| country_mesh = st.selectbox(label='Select a country to view the data related to that location', | |
| help='The names shown here are drawn directly from data attached to the articles.', | |
| options=sorted(df['PL'].unique().tolist())) | |
| submit_mesh = st.form_submit_button('Submit') | |
| if submit_collocation: | |
| st.subheader('Collocation') | |
| st.markdown('This program attempts to find the stem (or root form) of the word you enter and compare it against the database. This can lead to surprising results.') | |
| collocation_plot = run_collocation(word, degree=degree, number_of_items=10) | |
| st.pyplot(collocation_plot) | |
| elif submit_journal: | |
| ''' | |
| Eliminated JID metrics for now because JID is not actually a unique identifier for journals, despite claims to the contrary (https://www.nlm.nih.gov/bsd/mms/medlineelements.html#jid). | |
| Did something go wrong in the data collection/cleaning process? | |
| ''' | |
| # st.subheader('Journal Frequency') | |
| # st.markdown('The NLM lists the names of journals associated with these identifiers [here](https://ftp.ncbi.nih.gov/pubmed/J_Medline.txt).') | |
| # st.pyplot(jid_frequency(year_start=year_start_journal, year_end=year_end_journal, n=n_journal)) | |
| # st.pyplot(jid_frequency_by_year(year_start=year_start_journal, year_end=year_end_journal, n=n_journal)) | |
| # st.divider() | |
| st.subheader('Place of Publication Frequency') | |
| st.markdown('Note that the place of publication is derived from the location of the journal, not the location of the author(s).') | |
| st.pyplot(pl_frequency(year_start=year_start_journal, year_end=year_end_journal, n=n_journal)) | |
| elif submit_nbe: | |
| st.subheader('Named Biological Entities') | |
| st.markdown('This program uses a natural language processing model to extract named biological entities from the abstracts in the database. The model categorizes these entities as either a disease or an anatomical term. This is an artifact of the corpus that trained the model.') | |
| st.pyplot(ner_frequency(n=n_nbe, entity_toggle=entity, year_start=year_start_nbe, year_end=year_end_nbe)) | |
| st.pyplot(ner_frequency_by_year(n=n_nbe, entity_toggle=entity, year_start=year_start_nbe, year_end=year_end_nbe)) | |
| elif submit_mesh: | |
| st.header('MeSH Term Data') | |
| st.markdown('MeSH (Medical Subject Headings) is the NLM controlled vocabulary thesaurus used for indexing articles. For more information, visit the [MeSH homepage](https://www.nlm.nih.gov/mesh/meshhome.html).') | |
| st.subheader('MeSH Terms by Country') | |
| st.pyplot(mh_frequency(country=country_mesh, globally=False, data=df, year_start=year_start_mesh, year_end=year_end_mesh, n=n_mesh)) | |
| st.pyplot(mh_time_series(country=country_mesh, globally=False, data=df, year_start=year_start_mesh, year_end=year_end_mesh, n=n_mesh)) | |
| st.divider() | |
| st.subheader('Mesh Terms Globally') | |
| st.pyplot(mh_frequency(country=None, globally=True, data=df, year_start=year_start_mesh, year_end=year_end_mesh, n=n_mesh)) | |
| st.pyplot(mh_time_series(country=None, globally=True, data=df, year_start=year_start_mesh, year_end=year_end_mesh, n=n_mesh)) |