Update app.py

app.py

@@ -68,7 +68,7 @@ class Journal:
     def __repr__(self):
         return f"Journal(name='{self.name}', bytes='{self.bytes}')"
 
-llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo-1106")
+llm = ChatOpenAI(temperature=0, model="gpt-4-0125-preview")
 
 textex_chain = create_extraction_chain(textex_schema, llm)
 tablex_chain = create_extraction_chain(tablex_schema, llm)
@@ -104,7 +104,7 @@ if uploaded_files:
             if on_h:
                 chunk_size_h = st.selectbox(
                     'Tokens amounts per process :',
-                    (15000, 12000, 10000, 8000, 5000), key='table_h'
+                    (120000, 96000, 64000, 32000), key='table_h'
                 )
                 parseButtonH = st.button("Get Result", key='table_H')
         
@@ -116,7 +116,7 @@ if uploaded_files:
             if on_v:
                 chunk_size_v = st.selectbox(
                     'Tokens amounts per process :',
-                    (15000, 12000, 10000, 8000, 5000), key='table_v'
+                    (120000, 96000, 64000, 32000), key='table_v'
                 )
                 parseButtonV = st.button("Get Result", key='table_V')
     with col3:
@@ -127,7 +127,7 @@ if uploaded_files:
             if on_t:
                 chunk_size_t = st.selectbox(
                     'Tokens amounts per process :',
-                    (15000, 12000, 10000, 8000, 5000), key='no_table'
+                    (120000, 96000, 64000, 32000), key='no_table'
                 )
                 parseButtonT = st.button("Get Result", key="no_Table")
 
@@ -161,7 +161,10 @@ if uploaded_files:
                             try:
                                 df = pd.DataFrame(literal_eval(str(json.dumps(tablex_chain.run(inp)[0])).replace("\'", '\"')), index=[0]).fillna('')
                             except:
-                                df = pd.DataFrame(literal_eval(str(json.dumps(tablex_chain.run(inp)[0]) + ']').replace("\'", '\"')), index=[0]).fillna('')
+                                try:
+                                    df = pd.DataFrame(literal_eval(str(json.dumps(tablex_chain.run(inp)[0]) + ']').replace("\'", '\"')), index=[0]).fillna('')
+                                except SyntaxError:
+                                    df = pd.DataFrame(literal_eval('[' + str(json.dumps(tablex_chain.run(inp)[0]) + ']').replace("\'", '\"')), index=[0]).fillna('')
                             # df = pd.DataFrame(repair_json(tablex_chain.run(inp)[0]))
                             chunkdf.append(df)
 
@@ -203,7 +206,7 @@ if uploaded_files:
                                 embeddings = OpenAIEmbeddings()
 
                                 db = Chroma.from_documents(docs, embeddings)
-                                llm_table = ChatOpenAI(model_name="gpt-3.5-turbo-16k", temperature=0)
+                                llm_table = ChatOpenAI(model_name="gpt-4-0125-preview", temperature=0)
                                 qa_chain = RetrievalQA.from_chain_type(llm_table, retriever=db.as_retriever())
                                 
                                 # List of questions
@@ -232,6 +235,7 @@ if uploaded_files:
                             if output_list[0]['result'].split('\n')[i] != "":
                                 try:
                                     row = literal_eval(repair_json(output_list[0]['result'].split('\n')[i]))[0]
+                                    st.write(row)
                                     row = {**row, **{
                                                 'Title' : concat['title'][0],
                                                 'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
@@ -397,26 +401,33 @@ if uploaded_files:
                                                 'Recommendation' : summary,
                                             }
                                     }
-                                    if row['SNPs'] != "Not available":
+                                    if 'SNPs' in list(row.keys()):
+                                        if row['SNPs'] != "Not available":
+                                            row.update({
+                                                'SNPs' : "Not available"
+                                        })
+                                    else:
                                         row.update({
-                                            'SNPs' : "Not available"
-                                    })
-                                    if len(row['Genes'].strip().split(',')) > 1:
-                                        for g in row['Genes'].strip().split(','):
-                                            L.append({
-                                                'Title' : concat['title'][0],
-                                                'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
-                                                'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
-                                                'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
-                                                'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
-                                                'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
-                                                'Recommendation' : summary,
-                                                'Genes' : g.strip().upper().replace('Unknown', ''),
-                                                "SNPs" : "Not available",
-                                                "Diseases" : ''.join(list(row['Diseases'].title() if row['Diseases'] not in ['T2D', 'T2DM', 'NAFLD', 'CVD'] else row['Diseases'])).replace('Unknown', '').replace('Unknown', '')
-                                            })
+                                                'SNPs' : "Not available"
+                                        })
+                                        
+                                    if 'Genes' in list(row.keys()):
+                                        if len(row['Genes'].strip().split(',')) > 1:
+                                            for g in row['Genes'].strip().split(','):
+                                                L.append({
+                                                    'Title' : concat['title'][0],
+                                                    'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
+                                                    'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
+                                                    'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
+                                                    'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
+                                                    'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
+                                                    'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                    'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
+                                                    'Recommendation' : summary,
+                                                    'Genes' : g.strip().upper().replace('Unknown', ''),
+                                                    "SNPs" : "Not available",
+                                                    "Diseases" : ''.join(list(row['Diseases'].title() if row['Diseases'] not in ['T2D', 'T2DM', 'NAFLD', 'CVD'] else row['Diseases'])).replace('Unknown', '').replace('Unknown', '')
+                                                })
                                     else:
                                         L.append(row)
                                 except SyntaxError:
@@ -616,7 +627,17 @@ if uploaded_files:
                             chunkdf = []
                             for i, chunk in enumerate(text_chunk):
                                 inp = chunk
-                                df = pd.DataFrame(literal_eval(str(json.dumps(tablex_chain.run(inp)[0])).replace("\'", '\"')), index=[0]).fillna('')
+                                # Assuming tablex_chain.run(inp)[0] returns a dictionary
+                                original_dict = tablex_chain.run(inp)[0]
+                                # Convert the dictionary to a JSON string
+                                json_str = json.dumps(original_dict)
+                                # Replace single quotes with double quotes in the JSON string
+                                json_str_fixed = json_str.replace("'", '"')
+                                # Use literal_eval to safely evaluate the JSON string as a Python dictionary
+                                fixed_dict = literal_eval(json_str_fixed)
+                                # Create a DataFrame from the fixed dictionary
+                                df = pd.DataFrame(fixed_dict, index=[0]).fillna('')
+                                # df = pd.DataFrame(literal_eval(str(json.dumps(tablex_chain.run(inp)[0])).replace("\'", '\"')), index=[0]).fillna('')
                                 chunkdf.append(df)
 
                             concat = pd.concat(chunkdf, axis=0).reset_index().drop('index', axis=1).fillna('')
@@ -687,11 +708,10 @@ if uploaded_files:
                                                 'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                 'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                 'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                 'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
-                                                'Recommendation' : summary,
+                                                'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                         }}
                                         if len(row['Genes'].strip().split(',')) > 1:
                                             for g in row['Genes'].strip().split(','):
@@ -705,8 +725,8 @@ if uploaded_files:
                                                     'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
                                                     'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
                                                     'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                    'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                    'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                    'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                     'Recommendation' : summary,
                                                 })
                                         else:
@@ -718,10 +738,10 @@ if uploaded_files:
                                                 'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                 'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                 'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                 'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                 'Recommendation' : summary,
                                         }}
                                         if len(row['Genes'].strip().split(',')) > 1:
@@ -734,10 +754,10 @@ if uploaded_files:
                                                     'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                     'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                     'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                    'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                     'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                    'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                    'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                    'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                     'Recommendation' : summary,
                                                 })
                                         else:
@@ -750,10 +770,10 @@ if uploaded_files:
                                                     'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                     'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                     'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                    'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                     'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                    'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                    'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                    'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                     'Recommendation' : summary,
                                                 }
                                             }
@@ -770,10 +790,10 @@ if uploaded_files:
                                                     'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                     'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                     'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                    'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                     'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                    'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                    'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                    'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                     'Recommendation' : summary,
                                                 }
                                         }
@@ -793,10 +813,10 @@ if uploaded_files:
                                                 'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                 'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                 'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                 'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                 'Recommendation' : summary,
                                         }}
                                         if row['SNPs'] != "Not available":
@@ -813,10 +833,10 @@ if uploaded_files:
                                                     'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                     'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                     'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                    'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                     'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                    'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                    'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                    'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                     'Recommendation' : summary,
                                                 })
                                         else:
@@ -828,10 +848,10 @@ if uploaded_files:
                                                 'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                 'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                 'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                 'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                 'Recommendation' : summary,
                                         }}
                                         if row['SNPs'] != "Not available":
@@ -848,10 +868,10 @@ if uploaded_files:
                                                     'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                     'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                     'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                    'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                     'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                    'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                    'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                    'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                     'Recommendation' : summary,
                                                 })
                                         else:
@@ -864,10 +884,10 @@ if uploaded_files:
                                                     'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                     'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                     'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                    'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                     'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                    'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                    'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                    'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                     'Recommendation' : summary,
                                                 }
                                             }
@@ -884,10 +904,10 @@ if uploaded_files:
                                                     'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                     'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                     'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                    'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                     'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                    'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                    'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                    'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                     'Recommendation' : summary,
                                                 }
                                         }
@@ -907,10 +927,10 @@ if uploaded_files:
                                                 'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                 'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                 'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                 'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                 'Recommendation' : summary,
                                             }
                                     }
@@ -927,10 +947,10 @@ if uploaded_files:
                                                 'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                 'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                 'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                 'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                 'Recommendation' : summary,
                                             }
                                     }
@@ -948,10 +968,10 @@ if uploaded_files:
                                                     'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                     'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                     'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                    'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                     'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                    'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                    'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                    'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                     'Recommendation' : summary,
                                                 }
                                             }
@@ -968,10 +988,10 @@ if uploaded_files:
                                                     'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                     'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
                                                     'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                    'Population' : ' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
                                                     'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                    'Study Methodology' : ' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title(),
-                                                    'Study Level' : ' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title(),
+                                                    'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                    'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                     'Recommendation' : summary,
                                                 }
                                         }
@@ -1039,7 +1059,10 @@ if uploaded_files:
                         time.sleep(0.1)
                         st.write("☑ Generating Summary ...")
 
-                        concat['SNPs'] = concat['SNPs'].apply(lambda x: x if x.startswith('rs') else '')
+                        if 'SNPs' in list(concat.columns):
+                            concat['SNPs'] = concat['SNPs'].apply(lambda x: x if x.startswith('rs') else '')
+                        else:
+                            concat['SNPs'] = ''
                         for col in list(concat.columns):
                             concat[col] = concat[col].apply(lambda x: x if x not in ['N/A', 'not mentioned', 'Not mentioned', 'Unknown'] else '')
 
@@ -1096,7 +1119,7 @@ if uploaded_files:
                                             'Recommendation' : summary,
                                         })
                         
-                        csv = pd.concat([csv, pd.DataFrame(L)], ignore_index=True)
+                        csv = pd.concat([csv, pd.DataFrame(L)], ignore_index=True).drop_duplicates(subset='Genes')
                     status.update(label="Gene and SNPs succesfully collected.")
                     st.dataframe(csv)
                     with pd.ExcelWriter(buffer, engine='xlsxwriter') as writer: