##packages code import streamlit as st from shapely.geometry import Point import pandas as pd from tqdm import tqdm import numpy as np import json, requests import pandas as pd #from pandas.io.json import json_normalize import matplotlib.pyplot as plt import seaborn as sns from math import radians, cos, sin, asin, sqrt from sentence_transformers import SentenceTransformer, util @st.cache_resource def model_nlp(): model = SentenceTransformer('all-MiniLM-L6-v2') return model path = 'Climate_site/python_scripts/' @st.cache_data # 👈 Add the caching decorator def load_data(): url = path + "institutions.tsv" dic = pd.read_csv(url, delimiter = "\t" , index_col = 1).to_dict('index') return dic dic_institutions = load_data() import unicodedata from metaphone import doublemetaphone from fuzzywuzzy import fuzz from difflib import SequenceMatcher import re import geopandas as gpd from geopandas import GeoDataFrame #################### General Functions ############################# def URL(base_URL , entity_type , filters): url = base_URL + entity_type + filters return url def get_data(url): url = requests.get(url) text = url.text import json data = json.loads(text) return data ## encoding the abstract def reconstruction_abstract(abstract_inverted_index): # return the abstract is the abstract exists in the database, else, return None if abstract_inverted_index != None: list_values = list(abstract_inverted_index.values()) list_keys = list(abstract_inverted_index.keys()) #from the words in the abstract (keys of abstract_inverted_index) and their position in the text (values of abstract_inverted_index), reconstruct the abstract size_abstract = max([ max(elem) for elem in abstract_inverted_index.values() ] ) abstract = [""]*(size_abstract +1) for i in range(len(list_values)): for pos in list_values[i]: abstract[pos] = list_keys[i] return " ".join(list(abstract)) else: return None ## calculate efficiently the dot product between two vectors def norm(vector): return np.sqrt(sum(x * x for x in vector)) def cosine_similarity2(vec_a, vec_b): norm_a = norm(vec_a) norm_b = norm(vec_b) dot = sum(a * b for a, b in zip(vec_a, vec_b)) return dot / (norm_a * norm_b) ## Extracted texts def print_extracted_text(name_file): file = open(path + 'iea.txt', "r", encoding='utf8') lines = file.readlines() count = 0 for index, line in enumerate(lines): read_line = line.strip() print(read_line) file.close() iea.txt def details(name_file , display): file = open(path + "iea.txt", "r") lines = file.readlines() mark = 0 dic_details = {} count = -1 for index, line in enumerate(lines): line = line.strip() if line == "Close explanation": break if line != "" and (line[0].isnumeric() and ">" in line and " " in line) : count += 1 if mark == 1 and line != "" and line[0] == "*": if display == True: print(count) print(text) print(" ") dic_details[count] = text mark = 0 if mark == 1: text = text + line + " " if line.split(" ")[-1] == "Details" or line.split(" ")[-1] == "Hide": mark = 1 text = "" return dic_details def key_initiatives(name_file , display ): file = open(path + 'iea.txt', "r", encoding='utf8') lines = file.readlines() mark = 0 dic_key_initiatives = {} count = -1 for index, line in enumerate(lines): line = line.strip() if line == "Close explanation": break if line != "" and (line[0].isnumeric() and ">" in line and " " in line) : count += 1 if mark == 1 and line != "" and ( (line[0].isnumeric() and ">" in line and " " in line) or line == "*Deployment targets:*" or line == "*Announced development targets:*"): if display == True: print(count) print(text) print(" ") dic_key_initiatives[count] = text mark = 0 if mark == 1: text = text + line + " " if line == "*Key initiatives:*": mark = 1 text = "" return dic_key_initiatives def deployment_target(name_file , display): file = open(path + 'iea.txt', "r", encoding='utf8') lines = file.readlines() mark = 0 dic_target = {} count = -1 for index, line in enumerate(lines): line = line.strip() if line == "Close explanation": break if line != "" and (line[0].isnumeric() and ">" in line and " " in line) : count += 1 if mark == 1 and line != "" and ((line[0].isnumeric() and ">" in line and " " in line) or line == "*Announced cost reduction targets:*" or line == "*Announced development targets:*"): if display == True: print(count) print(text) print(" ") dic_target[count] = text mark = 0 if mark == 1: text = text + line + " " if line == "*Deployment targets:*" or line == "*Announced development targets:*": mark = 1 text = "" return dic_target def cost_reduction_target(name_file , display): file = open(path + 'iea.txt', "r", encoding='utf8') lines = file.readlines() mark = 0 dic_cost = {} count = -1 for index, line in enumerate(lines): line = line.strip() if line == "Close explanation": break if line != "" and (line[0].isnumeric() and ">" in line and " " in line) : count += 1 if mark == 1 and line != "" and (line[0].isnumeric() and ">" in line and " " in line) : if display == True: print(count) print(text) print(" ") dic_cost[count] = text mark = 0 if mark == 1: text = text + line + " " if line == "*Announced cost reduction targets:*": mark = 1 text = "" return dic_cost def key_words(name_file, display ): file = open(path + 'iea.txt', "r", encoding='utf8') lines = file.readlines() list_categories = [] count = -1 for index, line in enumerate(lines): line = line.strip() if line == "Close explanation": break if line != "" and (line[0].isnumeric() and ">" in line and " " in line) : count += 1 if display == True: print("Technologies" , count+1 , ":") if line != "": if line[0].isnumeric() and ">" in line and " " in line: i = 0 try: line = line.split(" ")[2] except: print(line) break if "Details" not in lines[index] and "Moderate" not in lines[index]: while " " not in line: i += 1 if "Details"==lines[index + i][:7] or "End-use"==lines[index + i][:7]: break else: line = line + " " + lines[index + i] #if " Production" in line: #line = line.replace(" Production" , "") line = line.replace("\n" , " ") line = line.replace("/" , " ") line = line.replace("-" , " ") line = line.split(" ")[0] if " " in line: line = line.replace(" ", " ") line = line.split(">") if "(" in line[-1]: line[-1] = line[-1].split("(")[0] for i in range(len(line)): # remove multiple spaces line[i] = re.sub(' +', ' ', line[i]) # remove trailing spaces line[i] = line[i].strip() if display == True: print(line) print(" ") if '' in line: line.remove('') list_categories.append([count , line]) return list_categories def technology(name_file, display ): # Filepath too specific, need to change to relative path file = open(path + 'iea.txt', "r", encoding='utf8') lines = file.readlines() list_categories = [] count = -1 for index, line in enumerate(lines): line = line.strip() if line == "Close explanation": break if line != "" and (line[0].isnumeric() and ">" in line and " " in line) : count += 1 if display == True: print("Technologies" , count+1 , ":") if line != "": if line[0].isnumeric() and ">" in line and " " in line: i = 0 try: line = line.split(" ")[1] except: print(line) break line = line.replace("\n" , " ") line = line.replace("/" , " ") line = line.replace("-" , " ") line = line.strip() line = re.sub(' +', ' ', line) line = line.split(" ")[0] line = line.split(">") if "(" in line[-1]: line[-1] = line[-1].split("(")[0] for i in range(len(line)): # remove multiple spaces line[i] = re.sub(' +', ' ', line[i]) # remove trailing spaces line[i] = line[i].strip() if display == True: print(line) print(" ") list_categories.append([count , line]) return list_categories #################### Paper Functions ############################# def related_papers(number_technology , carbon_related , display , key_words ): """Returns a dictionary of data related to the papers the user chooses This is done through generating URLs to connect with OpenAlex API and parsing through the json file Parameters ---------- number_technology : int, the index of the technology with respect to the 23 IEA technologies list_categories : str, an array of all the technologies related to the 23 IEA technologies carbon_related: bool, a setting to select papers related to carbon capture display: key_words: str, an array of keywords selected based off technology/ user input """ dic = {} max_count = 0 base_URL_OA = f'https://api.openalex.org/' filter_works = f'works?' filter_openalex = f"search=" + key_words if carbon_related == True: filter_openalex += "&filter=concepts.id:https://openalex.org/C132651083|https://openalex.org/C115343472|https://openalex.org/C530467964&per_page=100&mailto=emma_scharfmann@berkeley.edu" else: filter_openalex += "&per_page=100&mailto=emma_scharfmann@berkeley.edu" filter_openalex = filter_openalex.replace(" " , "%20") url = URL(base_URL_OA , filter_works, filter_openalex) data = get_data(url) count = data["meta"]["count"] if display == True: print( data["meta"]["count"] , [elem[i] for i in range(1,len(elem))] ) print(url) for i in range(len(data["results"])): dic[ data["results"][i]["id"]] = {} dic[ data["results"][i]["id"]]["title"] = data["results"][i]["title"] dic[ data["results"][i]["id"]]["abstract"] = reconstruction_abstract(data["results"][i]["abstract_inverted_index"]) dic[ data["results"][i]["id"]]["concepts"] = data["results"][i]["concepts"] dic[ data["results"][i]["id"]]["date"] = data["results"][i]["publication_date"] dic[ data["results"][i]["id"]]["authorships"] = data["results"][i]["authorships"] dic[ data["results"][i]["id"]]["cited_by_count"] = data["results"][i]["cited_by_count"] if len(data["results"][i]["authorships"]) > 0: if data["results"][i]["authorships"][0]["institutions"] != []: dic[ data["results"][i]["id"]]["countries"] = data["results"][i]["authorships"][0]["institutions"][0]["country_code"] dic[ data["results"][i]["id"]]["institutions"] = data["results"][i]["authorships"][0]["institutions"][0]["display_name"] else: dic[ data["results"][i]["id"]]["countries"] = "" dic[ data["results"][i]["id"]]["institutions"] = "" dic[ data["results"][i]["id"]]["authors"] = data["results"][i]["authorships"][0]["author"]["display_name"] for j in range(1 , len(data["results"][i]["authorships"])): if data["results"][i]["authorships"][j]["institutions"] != []: dic[ data["results"][i]["id"]]["institutions"] += ", " + data["results"][i]["authorships"][j]["institutions"][0]["display_name"] dic[ data["results"][i]["id"]]["countries"] = data["results"][i]["authorships"][j]["institutions"][0]["country_code"] dic[ data["results"][i]["id"]]["authors"] += ", " + data["results"][i]["authorships"][j]["author"]["display_name"] if display == True: print(" ") return dic ## Ranking Papers def ranking_papers(number_technology, dic_details , display , size , carbon_related , search_words ): """Returns a dictionary of data related to the papers the user chooses This is done through generating URLs to connect with OpenAlex API and parsing through the json file Parameters ---------- number_technology : int, the index of the technology with respect to the 500+ IEA technologies dic_details : str, dictionary where key is index of technology with respect to 500+ IEA technolgies and values are the details related to the technologies dic: bool, a setting to select papers related to carbon capture list_categories: str, an array of keywords selected based off technology/ user input display: bool, chooses to display information or not size: str, an array of keywords selected based off technology/ user input """ model = model_nlp() dic = related_papers(number_technology , carbon_related , display , search_words ) dic_scores_papers = {} if display == True: print("Technology " + str(number_technology)) print("Key words: " , list_categories[number_technology][1]) if number_technology in dic_details: reference_text = dic_details[number_technology] if display== True: print("Technology details: " , reference_text) print(" ") encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist() if len(dic) == 0: return "No paper found, select other key words" for ids in list(dic.keys()): dic_scores_papers[ids] = {} if dic[ids]["title"] != None: encoded_title = model.encode(dic[ids]["title"], convert_to_tensor=False).tolist() score_title = cosine_similarity2(encoded_title, encoded_text) else: score_title = None if dic[ids]["abstract"] != None: encoded_abstract = model.encode(dic[ids]["abstract"], convert_to_tensor=False).tolist() score_abstract = cosine_similarity2(encoded_abstract, encoded_text) else: score_abstract = None #concepts = "" #for elem in dic[ids]["concepts"]: # concepts += elem["display_name"] + " " #encoded_concepts = model.encode(concepts, convert_to_tensor=False).tolist() #score_concepts = cosine_similarity2(encoded_concepts, encoded_text) dic_scores_papers[ids]["title comparison"] = score_title dic_scores_papers[ids]["abstract comparison"] = score_abstract #dic_scores_papers[ids]["concepts comparison"] = score_concepts dic_scores_papers[ids]["title"] = dic[ids]["title"] dic_scores_papers[ids]["citations"] = dic[ids]["cited_by_count"] dic_scores_papers[ids]["date"] = dic[ids]["date"][:4] if "institutions" in dic[ids]: dic_scores_papers[ids]["institutions"] = dic[ids]["institutions"] # dic_scores_papers[ids]["countries"] = dic[ids]["countries"] else: dic_scores_papers[ids]["institutions"] = None # dic_scores_papers[ids]["countries"] = None if "authors" in dic[ids] and dic[ids]["authors"] != None: dic_scores_papers[ids]["number of co-authors"] = len(dic[ids]["authors"].split(",")) dic_scores_papers[ids]["authors"] = dic[ids]["authors"] else: dic_scores_papers[ids]["number of co-authors"] = None dic_scores_papers[ids]["authors"] = None return dic , dic_scores_papers def get_ranking_related_papers( technologies , number_technology , carbon_related , size , research_words): name_file = "iea" dic_details = details(name_file , False) dic , dic_scores_papers = ranking_papers(number_technology, dic_details , False , size , carbon_related , research_words ) if dic_scores_papers == {}: return "No paper found" elif type(dic_scores_papers) == str: return dic_scores_papers else: return pd.DataFrame(dic_scores_papers).T.sort_values(by="abstract comparison" , ascending = False).head(size) ## Extract quantitative data def extract_sentences_with_numbers(text , text_name): if text != None: text = text.replace("CO 2" , "CO2") text = text.replace("CO 3" , "CO3") text = text.replace("CO(2)" , "CO2") text = text.replace("CO(3)" , "CO3") print(text_name + ": " , text) print(" ") list_text = list(text) for i in range(1,len(list_text)-1): if list_text[i] == " " and list_text[i-1] == "." and list_text[i+1].isupper(): list_text[i] = "~" text = "".join(list_text) text = text.split("~") for sentence in text: if any(char.isdigit() for char in sentence): if "CO2" in sentence: print("\x1b[31mCARBON RELATED:\x1b[0m", sentence) print(" ") if "GJ" in sentence or "MJ" in sentence: print("\x1b[31mENERGY:\x1b[0m" , sentence) print(" ") ##price if "€" in sentence or "$" in sentence or "EUR" in sentence or "dollars" in sentence.lower(): print("\x1b[31mPRICE:\x1b[0m" , sentence) print(" ") ##dates digits = [] for word in sentence.replace("," , "").replace("%" , "").replace("." , " ").split(): if word.isdigit() and 1850 < int(word) < 2200 : digits.append(word) if digits != []: print("\x1b[31mDATE:\x1b[0m" , sentence) print(" ") ##CO quantity if "Mt" in sentence or "tC" in sentence or "t-C" in sentence: print("\x1b[31mCARBON QUANTITY:\x1b[0m" , sentence) print(" ") print(" ") def extract_quantitative_data_technology(technologies, number_technology): model = model_nlp() count = 0 name_file = "iea" dic_target = deployment_target(name_file , False) dic_cost = cost_reduction_target(name_file , False) dic_details = details(name_file , False) cost_target_text = 'No information' cost_text = 'No information' if number_technology in dic_details: reference_text = dic_details[number_technology] #print("\033[96mFROM IEA website: ") #print("\033[92mTechnology details: \x1b[0m" , reference_text) #print(" ") #encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist() if number_technology in dic_target: cost_target_text = dic_target[number_technology] #print("\033[96mFROM IEA website: ") #sentences = extract_sentences_with_numbers(cost_target_text , "\033[92mDeployment target and Announced development target\x1b[0m") if number_technology in dic_cost: cost_text = dic_cost[number_technology] #print("\033[96mFROM IEA website: ") #sentences = extract_sentences_with_numbers(cost_text , "\033[92mAnnounced cost reduction targets\x1b[0m") return reference_text, cost_target_text, cost_text def extract_quantitative_data_paper(work_id): url = "https://api.openalex.org/works/" + str(work_id) url_google = "https://explore.openalex.org/works/" + str(work_id) data = get_data(url) date = data["publication_date"] title = data["title"] abstract = reconstruction_abstract(data["abstract_inverted_index"]) concepts = ", ".join( [elem["display_name"] for elem in data["concepts"]] ) authors = ", ".join( [elem["author"]["display_name"] for elem in data["authorships"]] ) institutions = ", ".join( set([elem["institutions"][0]["display_name"] for elem in data["authorships"] if len(elem["institutions"]) > 0]) ) return url_google , title , abstract , date , authors , institutions ## Get papers related to cited projects def related_projects(technologies, number_technology , carbon_related , size): spacy_nlp = spacy_func() model = model_nlp() name_file = "iea" dic_details = details(name_file , False) dic_key_initiatives = key_initiatives(name_file , False ) dic2 = {} reference_text = dic_details[number_technology] print("\033[92mTechnology details: \x1b[0m" , reference_text) print(" ") encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist() # reference_text key_initiative location_entities if number_technology in dic_key_initiatives: key_initiative = dic_key_initiatives[number_technology] print("\033[92mTechnology key initiatives: \x1b[0m" , key_initiative) location_entities = set() doc = spacy_nlp(key_initiative.strip()) for j in doc.ents: entry = str(j.lemma_).lower() key_initiative = key_initiative.replace(str(j).lower(), "") if j.label_ in ["ORG"]: location_entities.add(j) print(" ") print("\033[92mExtracted projects and organizations names: \x1b[0m", location_entities) base_URL_OA = f'https://api.openalex.org/' filter_works = f'works?' for element in location_entities: filter_openalex = f"search=" filter_openalex += str(element) if carbon_related == True: filter_openalex += "&filter=concepts.id:https://openalex.org/C2780021121&per_page=50&mailto=emma_scharfmann@berkeley.edu" else: filter_openalex += "&per_page=50&mailto=emma_scharfmann@berkeley.edu" filter_openalex = filter_openalex.replace(" " , "%20") url = URL(base_URL_OA , filter_works, filter_openalex) data = get_data(url) count = len(data["results"]) if count < 40: for k in range(count): if data["results"][k]["abstract_inverted_index"] != None: encoded_abstract = model.encode(reconstruction_abstract(data["results"][k]["abstract_inverted_index"]) , convert_to_tensor=False).tolist() abstract_similarity = cosine_similarity2(encoded_abstract, encoded_text) else: abstract_similarity = None if data["results"][k]["title"] != None: encoded_title = model.encode(data["results"][k]["title"] , convert_to_tensor=False).tolist() title_similarity = cosine_similarity2(encoded_title, encoded_text) else: title_similarity = None work_id = data["results"][k]["id"] dic2[work_id] = {} dic2[work_id]["project/institution name"] = str(element) dic2[work_id]["title comparison"] = title_similarity dic2[work_id]["abstract comparison"] = abstract_similarity dic2[work_id]["title"] = data["results"][k]["title"] #dic2[work_id]["abstract"] = reconstruction_abstract(data["results"][k]["abstract_inverted_index"]) dic2[work_id]["date"] = data["results"][k]["publication_date"] else: list_categories = key_words( name_file, False) key_word = list_categories[number_technology] elem = key_word[1] string = "" for j in range(1,len(elem)): if elem[j] == "CCUS": elem[j] = "Carbon capture and storage" string += elem[j] + " " string += str(element) filter_openalex = f"search=" + string if carbon_related == True: filter_openalex += "&filter=concepts.id:https://openalex.org/C2780021121&per_page=50&mailto=emma_scharfmann@berkeley.edu" else: filter_openalex += "&per_page=50&mailto=emma_scharfmann@berkeley.edu" filter_openalex = filter_openalex.replace(" " , "%20") url = URL(base_URL_OA , filter_works, filter_openalex) data = get_data(url) count = len(data["results"]) for k in range(count): if data["results"][k]["abstract_inverted_index"] != None: encoded_abstract = model.encode(reconstruction_abstract(data["results"][k]["abstract_inverted_index"]) , convert_to_tensor=False).tolist() abstract_similarity = cosine_similarity2(encoded_abstract, encoded_text) else: abstract_similarity = None if data["results"][k]["title"] != None: encoded_title = model.encode(data["results"][k]["title"] , convert_to_tensor=False).tolist() title_similarity = cosine_similarity2(encoded_title, encoded_text) else: title_similarity = None work_id = data["results"][k]["id"] dic2[work_id] = {} dic2[work_id]["project/institution name"] = str(element) dic2[work_id]["title comparison"] = title_similarity dic2[work_id]["abstract comparison"] = abstract_similarity dic2[work_id]["title"] = data["results"][k]["title"] #dic2[work_id]["abstract"] = reconstruction_abstract(data["results"][k]["abstract_inverted_index"]) dic2[work_id]["date"] = data["results"][k]["publication_date"] if dic2 != {}: return reference_text,key_initiative,location_entities,pd.DataFrame(dic2).T.sort_values(by="abstract comparison" , ascending = False).head(size) else: return reference_text,key_initiative,location_entities, "No paper found" ## Mapping the authors #merge the nobiliary particles with the last name #ln_suff file can be modified if more or less nobiliary particles want to be suppressed ln_suff = ['oster', 'nordre', 'vaster', 'aust', 'vesle', 'da', 'van t', 'af', 'al', 'setya', 'zu', 'la', 'na', 'mic', 'ofver', 'el', 'vetle', 'van het', 'dos', 'ui', 'vest', 'ab', 'vste', 'nord', 'van der', 'bin', 'ibn', 'war', 'fitz', 'alam', 'di', 'erch', 'fetch', 'nga', 'ka', 'soder', 'lille', 'upp', 'ua', 'te', 'ni', 'bint', 'von und zu', 'vast', 'vestre', 'over', 'syd', 'mac', 'nin', 'nic', 'putri', 'bet', 'verch', 'norr', 'bath', 'della', 'van', 'ben', 'du', 'stor', 'das', 'neder', 'abu', 'degli', 'vre', 'ait', 'ny', 'opp', 'pour', 'kil', 'der', 'oz', 'von', 'at', 'nedre', 'van den', 'setia', 'ap', 'gil', 'myljom', 'van de', 'stre', 'dele', 'mck', 'de', 'mellom', 'mhic', 'binti', 'ath', 'binte', 'snder', 'sre', 'ned', 'ter', 'bar', 'le', 'mala', 'ost', 'syndre', 'sr', 'bat', 'sndre', 'austre', 'putra', 'putera', 'av', 'lu', 'vetch', 'ver', 'puteri', 'mc', 'tre', 'st'] #suppress all the unwanted suffixes from a string. #name_del file can be modified if more or less suffixes want to be suppressed name_del = ['2nd', '3rd', 'Jr', 'Jr.', 'Junior', 'Sr', 'Sr.', 'Senior'] def name_delete(string): for elmt in name_del: if f" {elmt}" in string: return string.replace(f" {elmt}","") return string def ln_suff_merge(string): for suff in ln_suff: if f" {suff} " in string or string.startswith(f"{suff} "): return string.replace(f"{suff} ",suff.replace(" ","")) return string #normalize a string dat that represents often a name. def normalize(data): normal = unicodedata.normalize('NFKD', data).encode('ASCII', 'ignore') val = normal.decode("utf-8") # delete unwanted elmt val = name_delete(val) # lower full name in upper val = re.sub(r"[A-Z]{3,}", lambda x: x.group().lower(), val) # add space in front of upper case val = re.sub(r"(\w)([A-Z])", r"\1 \2", val) # Lower case val = val.lower() # remove special characters val = re.sub('[^A-Za-z0-9 ]+', ' ', val) # remove multiple spaces val = re.sub(' +', ' ', val) # remove trailing spaces val = val.strip() # suffix merge val = ln_suff_merge(val) return val def main_authors(technologies,number_technology , carbon_related , size , research_words): name_file = "iea" list_categories = key_words( name_file, False) dic_details = details(name_file , False) dic_papers , dic_papers_ranked = ranking_papers(number_technology, dic_details , False , 200 , carbon_related , research_words ) dic_papers_co_authors = {} for paper in list(dic_papers_ranked.keys())[:size]: for k in range(len(dic_papers[paper]["authorships"])): coauthor_id = dic_papers[paper]["authorships"][k]["author"]["id"] author_name = dic_papers[paper]["authorships"][k]["author"]["display_name"] author_name_norm = normalize(dic_papers[paper]["authorships"][k]["author"]["display_name"]).split() if len(author_name_norm ) > 0: author_name_norm = author_name_norm[0] + " " + author_name_norm[-1] if author_name_norm not in dic_papers_co_authors: dic_papers_co_authors[author_name_norm] = {} dic_papers_co_authors[author_name_norm]["Author's name(s)"] = author_name if coauthor_id != None : dic_papers_co_authors[author_name_norm]["Author's id(s)"] = coauthor_id[21:] else: dic_papers_co_authors[author_name_norm]["Author's id(s)"] = "" dic_papers_co_authors[author_name_norm]["Number of occurence within the " + str(size) + " most related papers"] = 1 dic_papers_co_authors[author_name_norm]["Number of related citations"] = dic_papers[paper]["cited_by_count"] else: dic_papers_co_authors[author_name_norm]["Number of occurence within the " + str(size) + " most related papers"] += 1 if author_name not in dic_papers_co_authors[author_name_norm]["Author's name(s)"]: dic_papers_co_authors[author_name_norm]["Author's name(s)"] += ", " + author_name[0] if coauthor_id != None and coauthor_id[21:] not in dic_papers_co_authors[author_name_norm]["Author's id(s)"]: dic_papers_co_authors[author_name_norm]["Author's id(s)"] += ", " + coauthor_id[21:] dic_papers_co_authors[author_name_norm]["Number of related citations"] += dic_papers[paper]["cited_by_count"] if dic_papers_co_authors != {}: dic_papers_co_authors = {k: v for k, v in sorted(dic_papers_co_authors.items(), key=lambda item: item[1]["Number of occurence within the " + str(size) + " most related papers"] , reverse = True)} for author_name_norm in list(dic_papers_co_authors.keys())[:size]: list_ids = dic_papers_co_authors[author_name_norm]["Author's id(s)"].split(", ") work_count = 0 cited_by_count = 0 institutions = '' for elem in list_ids: if len(elem) > 3: url = 'https://api.openalex.org/authors?filter=openalex_id:' + elem + "&page=1&per_page=200&mailto=emma_scharfmann@berkeley.edu" try: data = get_data(url)["results"][0] except: continue work_count += data["works_count"] cited_by_count += data["cited_by_count"] if data["last_known_institution"] != None and len(data["last_known_institution"]) > 0 and data["last_known_institution"]["display_name"] != None: institutions += data["last_known_institution"]["display_name"] dic_papers_co_authors[author_name_norm]["Last Known Institution"] = institutions dic_papers_co_authors[author_name_norm]["Number of works"] = work_count dic_papers_co_authors[author_name_norm]["Number of citations"] = cited_by_count return pd.DataFrame(dic_papers_co_authors, index = [ "Author's name(s)" , "Number of occurence within the " + str(200) + " most related papers" , "Last Known Institution" , "Number of works" , "Number of citations" , "Number of related citations", "Author's id(s)"]).T.sort_values("Number of occurence within the " + str(200) + " most related papers" , ascending = False).head(size) else: return ("Select another category") def map_authors(technologies, number_technology , carbon_related , size , research_words): name_file = "iea" dic_details = details(name_file , False) dic_papers , dic_papers_ranked = ranking_papers(number_technology, dic_details , False , size , carbon_related , research_words ) dic_papers_co_authors = {} count = 0 for paper in list(dic_papers_ranked.keys())[:size]: for k in range(len(dic_papers[paper]["authorships"])): if dic_papers[paper]["authorships"][k]["institutions"] != [] and "id" in dic_papers[paper]["authorships"][k]["institutions"][0] and dic_papers[paper]["authorships"][k]["institutions"][0]["id"] != None: institution_id = dic_papers[paper]["authorships"][k]["institutions"][0]["id"][21:] if institution_id in dic_institutions: data = dic_institutions[institution_id] dic_papers_co_authors[count] = {} dic_papers_co_authors[count]["longitude"] = data["longitude"] dic_papers_co_authors[count]["latitude"] = data["latitude"] dic_papers_co_authors[count]["abstract comparison"] = dic_papers_ranked[paper]["abstract comparison"] dic_papers_co_authors[count]["author"] = dic_papers[paper]["authorships"][k]["author"]["display_name"] dic_papers_co_authors[count]["institution"] = dic_papers[paper]["authorships"][k]["institutions"][0]["display_name"] dic_papers_co_authors[count]["date"] = dic_papers[paper]["date"] count += 1 if dic_papers_co_authors == {}: return "No papers" map_df = pd.DataFrame(dic_papers_co_authors).T map_df["longitude"]=map_df['longitude'].astype(float) map_df['latitude']=map_df['latitude'].astype(float) map_df = map_df[map_df["latitude"].notnull()] return map_df ################################### Extracted texts ############################################################### #@title Which patents are related to the technology? def finder(): name_file = 'iea' res = technology("iea", False ) list_categories_tech = [] list_categories = key_words("iea" , False) list_technologies = [ ( ", ".join(list_categories[i][1]) , i ) for i in range(len(list_categories)) ] dic_technologies = {} for i in range(len(res)): names = res[i][1] if ", ".join(names) not in list_categories_tech: list_categories_tech.append(", ".join(names)) dic_technologies[", ".join(names)] = [] dic_technologies[", ".join(names)].append( (", ".join(list_categories[i][1]) , i )) list_climate = [ ("Any related papers" , False ) , ("Climate related papers" , True)] dic_categories = {} for elem in list_technologies: list_words = elem[0].split(",")[-3:] for i in range(len(list_words)): if "CCUS" in list_words[i]: list_words[i] = list_words[i].replace("CCUS" , "carbon capture storage") dic_categories[elem[1]] = [ ", ".join([ " ".join(words.split()[:3]) for words in list_words ] ) , ", ".join([ " ".join(words.split()[:3]) for words in list_words[:-1] ]) , ", ".join([ " ".join(words.split()[:3]) for words in list_words[1:] ] ) ] return dic_technologies, dic_categories, list_categories_tech, list_technologies # technologies_widget = widgets.Dropdown(options=list_categories_tech, # description="Choose the category:" , # style = {'description_width':'initial' } , # layout=Layout(width='500px')); # category_widget = widgets.RadioButtons( values = 1, # description='Patents where the abstract contains:' , # style = {'description_width':'initial' } , # layout=Layout(width='1000px')); # number_technology_widget = widgets.Dropdown(options=list_technologies, # description="Choose the technology:" , # style = {'description_width':'initial' } , # layout=Layout(width='500px')); # def update_category(*args): # category_widget.options = dic_categories[number_technology_widget.value] # def update_technology(*args): # number_technology_widget.options = dic_technologies[technologies_widget.value] # number_technology_widget.observe(update_category) # technologies_widget.observe(update_technology) # print("Which patents are related to the technology?") # interact_manual(get_ranking_patents, # technologies=technologies_widget, # number_technology= number_technology_widget , # carbon_related=widgets.Dropdown(options=list_climate, # description="Select the type of patents:" , # style = {'description_width':'initial' } , # layout=Layout(width='500px') ) , # size = widgets.IntSlider(min=3, # max=100, # value=10, # step=1, # description="Select the number of patents:" , # style = {'description_width':'initial' } , # layout=Layout(width='500px')), # category = category_widget);