update dblp code

knowledge_extraction.py

@@ -60,20 +60,24 @@ def knowledge_extraction():
 
         # Iterate over rows and update 'doi' column if necessary
         for index, row in data.iterrows():
-            doi = row['doi']
-            title = row['title']
+            doi = row['DOI']
+            title = row['Title']
             
             # If 'doi' is None, attempt to get DOI from title
-            if pd.isnull(doi):
+            if doi == 'None':
                 doi = scholarly.get_doi_from_title(title)
                 # Update the DataFrame with the retrieved DOI
                 if doi:
-                    data.at[index, 'doi'] = doi
+                    data.at[index, 'DOI'] = doi
 
         # Display the updated data table
         st.write("Data with DOI")
+        count_none = (data['DOI'] == 'None').sum()
+        total_values = len(data['DOI'])
+        st.write("Number of values in DOI column equal to None / Total values: " + str(count_none) + "/" + str(total_values))
         st.write(data)
 
+
     # Step 3: Loop over DOIs and retrieve citation counts
     step_3 = st.sidebar.checkbox("3 - Loop over DOIs and retrieve citation counts")
     if step_3:
@@ -82,7 +86,7 @@ def knowledge_extraction():
 
         # Loop over DOIs and retrieve citation counts
         for index, row in data.iterrows():
-            doi = row['doi']
+            doi = row['DOI']
             if doi:
                 citation_count = scholarly.get_citation_count(doi)
                 if citation_count is not None:
@@ -91,7 +95,7 @@ def knowledge_extraction():
                     count += 1
                 else:
                     # Handle cases where DOI is None (e.g., bytitle lookup)
-                    title = row['title']
+                    title = row['Title']
                     doi_bytitle = scholarly.get_doi_from_title(str(title))
                     citation_count_title = scholarly.get_citation_count(doi_bytitle)
                     if citation_count_title is not None:
@@ -102,12 +106,14 @@ def knowledge_extraction():
         # Add the citation count column to the DataFrame
         data['Citation Count'] = cit_array
         st.write(data)
+        output_path = "output/crawler_doi_citation.csv"
+        data.to_csv(output_path, sep=';', index=False)
 
     # Step 4: Download Filtered Data as CSV
     if not data.empty and step_3:
         st.download_button(
             label="Download Filtered Data as CSV",
             data=data.to_csv(index=False).encode(),
-            file_name="filtered_data.csv",
-            key="download_filtered_data",
+            file_name="crawler_doi_citation.csv",
+            key="crawler_doi_citation",
         )

merge.py

@@ -36,10 +36,12 @@ def merge_and_enrich_datasets(scholar_df, semantics_df, dblp_df):
                         # Aggiungi solo le colonne 'doi' e 'citazioni' da scholar_matches
                         merged_record['doi_scholar'] = scholar_matches.iloc[0]['DOI']
                         merged_record['cites_scholar'] = scholar_matches.iloc[0]['Cites']
+                        merged_record['abstract_scholar'] = scholar_matches.iloc[0]['Abstract']
                     if not semantics_matches.empty:
                         # Aggiungi solo le colonne 'doi' e 'citazioni' da semantics_matches
                         merged_record['doi_semantic'] = semantics_matches.iloc[0]['DOI']
                         merged_record['cites_semantic'] = semantics_matches.iloc[0]['Cites']
+                        merged_record['abstract_scholar'] = 'None'
                         
                     merged_records.append(merged_record)
                     # Scrivi il record nel file CSV
@@ -60,39 +62,12 @@ def merge_and_enrich_datasets(scholar_df, semantics_df, dblp_df):
 def main():
     # Interfaccia utente Streamlit
     st.title("Knowledge enrichment")
-    
-    if st.button("generate dblp csv"):
-        # Carica il file XML
-        tree = ET.parse('output/dblp.xml')
-        root = tree.getroot()
-        # Crea un file CSV per scrivere i dati
-        with open('output/dblp.csv', 'w', newline='', encoding='utf-8') as csvfile:
-            csvwriter = csv.writer(csvfile, delimiter=';')
-
-            # Scrivi l'intestazione del CSV
-            csvwriter.writerow(['DBLP', 'Authors', 'Title', 'Year', 'DOI', 'URL'])
-
-            # Estrai i dati XML e scrivili nel CSV
-            for hit in root.findall('.//hit'):
-                authors = ', '.join(author.text for author in hit.findall('.//authors/author'))
-                title = hit.find('.//title').text
-                year = hit.find('.//year').text
-                doi_element = hit.find('.//doi')
-                doi = doi_element.text if doi_element is not None else 'None'
-                url = hit.find('.//url').text
-                id = hit.attrib['id']
-                
-
-                # Scrivi i dati nel file CSV
-                csvwriter.writerow([id, authors, title, year, doi, url])
-
-        print("Conversione XML in CSV completata.")
 
     if st.button("create enriched dataset"):
         # Carica i file CSV in DataFrame
         scholar_df = pd.read_csv("output/googleScholarcsv.csv", sep=";", encoding='utf-8')
         semantics_df = pd.read_csv("output/semanticscholarcsv.csv", sep=";", encoding='utf-8')
-        dblp_df = pd.read_csv("output/dblp.csv", sep=";", encoding='utf-8')
+        dblp_df = pd.read_csv("output/crawler_doi_citation.csv", sep=";", encoding='utf-8')
 
         # Call the merge_and_enrich_datasets function
         enriched_dataset = merge_and_enrich_datasets(scholar_df, semantics_df, dblp_df)