add additional onet scraping functionality

main.py

@@ -18,7 +18,7 @@ from mangum import Mangum
 from localStoragePy import localStoragePy
 localStorage = localStoragePy('pathfinder', 'text')
 
-from scrape_onet import get_onet_code, get_onet_description, get_onet_tasks, get_job_postings
+from scrape_onet import get_onet_code, get_onet_description, get_onet_tasks, get_onet_activities, get_onet_context, get_onet_skills, get_onet_knowledge, get_onet_abilities, get_onet_interests, get_onet_styles, get_onet_values, get_job_postings
 from match_utils import neighborhoods, get_resume, skillNER, sim_result_loop, get_links, coSkillEmbed, sim_result_loop_jobFinder, sim_result_loop_candFinder
 from user_utils import Hash
 
@@ -122,13 +122,30 @@ def post_job(request: Request, bt: BackgroundTasks, jobtitle: str = Form(enum=[x
         onetCode = get_onet_code(jobtitle)
         jobdescription = get_onet_description(onetCode)
         tasks = get_onet_tasks(onetCode)
+        activities = get_onet_activities(onetCode)
+        context = get_onet_context(onetCode)
+        skills = get_onet_skills(onetCode)
+        knowledge = get_onet_knowledge(onetCode)
+        abilities = get_onet_abilities(onetCode)
+        interests = get_onet_interests(onetCode)
+        values = get_onet_values(onetCode)
+        styles = get_onet_styles(onetCode)
+
         bt.add_task(neighborhoods, jobtitle)
         return templates.TemplateResponse('job_list.html', context={
             'request': request, 
             'joblist': joblist, 
             'jobtitle': jobtitle, 
             'jobdescription': jobdescription, 
-            'tasks': tasks})
+            'tasks': tasks, 
+            'activities': activities, 
+            'context': context,
+            'knowledge': knowledge,
+            'abilities': abilities,
+            'skills': skills,
+            'interests': interests,
+            'values': values, 
+            'styles': styles})
 
 ### JOB NEIGHBORHOODS ###
 @app.get("/explore-job-neighborhoods/", response_class=HTMLResponse)

requirements.txt

@@ -21,4 +21,5 @@ bcrypt==4.0.1
 passlib==1.7.4
 localStoragePy==0.2.3
 sentence-transformers==2.2.2
-mangum==0.17.0
+mangum==0.17.0
+certifi==2023.7.22

scrape_onet.py

@@ -36,12 +36,11 @@ def get_onet_tasks(onetCode):
     tasks = [''.join(map(lambda c: '' if c in '0123456789-' else c, task)) for task in tasks]
     return tasks
 
-def get_onet_ratings(onetCode):
+def get_onet_activities(onetCode):
     headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}
     
     activities_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=wa&n_wa=0&s_wa=IM&c_wa=0"
-    context_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=cx&n_cx=0&c_cx=0&s_cx=n"
-    
+
     response = requests.get(activities_url, headers=headers)
     soup = BeautifulSoup(response.text, 'html.parser')
     tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")
@@ -57,7 +56,14 @@ def get_onet_ratings(onetCode):
         num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(' ) ', '')])
     df = pd.DataFrame(num_desc, columns = ['Importance', 'Work Characteristic'])
     df = df[df['Importance'] != '']
+    activities = df
+    return activities
     
+def get_onet_context(onetCode):
+    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}
+
+    context_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=cx&n_cx=0&c_cx=0&s_cx=n"
+
     response = requests.get(context_url, headers=headers)
     soup = BeautifulSoup(response.text, 'html.parser')
     tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")
@@ -73,16 +79,14 @@ def get_onet_ratings(onetCode):
         num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])
     df2 = pd.DataFrame(num_desc, columns = ['Importance', 'Work Characteristic'])
     df2 = df2[df2['Importance'] != '']
-    
-    job_df = pd.concat([df, df2], axis = 0)
+    context = df2
+    return context
 
-    skills_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=sk&n_sk=0&s_sk=IM&c_sk=0"
-    knowledge_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=kn&n_kn=0&s_kn=IM&c_kn=0"
-    abilities_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=ab&n_ab=0&s_ab=IM&c_ab=0"
-    interests_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=in&c_in=0"
-    values_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=wv&c_wv=0"
-    style_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=ws&n_ws=0&c_ws=0"
+def get_onet_skills(onetCode):
+    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}
 
+    skills_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=sk&n_sk=0&s_sk=IM&c_sk=0"
+    
     response = requests.get(skills_url, headers=headers)
     soup = BeautifulSoup(response.text, 'html.parser')
     tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")
@@ -98,6 +102,13 @@ def get_onet_ratings(onetCode):
         num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])
     df3 = pd.DataFrame(num_desc, columns = ['Importance', 'Candidate Characteristic'])
     df3 = df3[df3['Importance'] != '']
+    skills = df3
+    return skills
+
+def get_onet_knowledge(onetCode):
+    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}
+
+    knowledge_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=kn&n_kn=0&s_kn=IM&c_kn=0"
 
     response = requests.get(knowledge_url, headers=headers)
     soup = BeautifulSoup(response.text, 'html.parser')
@@ -114,7 +125,14 @@ def get_onet_ratings(onetCode):
         num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])
     df4 = pd.DataFrame(num_desc, columns = ['Importance', 'Candidate Characteristic'])
     df4 = df4[df4['Importance'] != '']
-    
+    knowledge = df4
+    return knowledge
+
+def get_onet_abilities(onetCode):
+    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}
+
+    abilities_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=ab&n_ab=0&s_ab=IM&c_ab=0"
+
     response = requests.get(abilities_url, headers=headers)
     soup = BeautifulSoup(response.text, 'html.parser')
     tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")
@@ -130,7 +148,14 @@ def get_onet_ratings(onetCode):
         num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])
     df5 = pd.DataFrame(num_desc, columns = ['Importance', 'Candidate Characteristic'])
     df5 = df5[df5['Importance'] != '']
-    
+    abilities = df5
+    return abilities
+
+def get_onet_interests(onetCode):
+    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}
+
+    interests_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=in&c_in=0"
+
     response = requests.get(interests_url, headers=headers)
     soup = BeautifulSoup(response.text, 'html.parser')
     tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")
@@ -146,7 +171,14 @@ def get_onet_ratings(onetCode):
         num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])
     df6 = pd.DataFrame(num_desc, columns = ['Importance', 'Candidate Characteristic'])
     df6 = df6[df6['Importance'] != '']
+    interests = df6
+    return interests
 
+def get_onet_values(onetCode):
+    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}
+
+    values_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=wv&c_wv=0"
+    
     response = requests.get(values_url, headers=headers)
     soup = BeautifulSoup(response.text, 'html.parser')
     tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")
@@ -162,6 +194,13 @@ def get_onet_ratings(onetCode):
         num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])
     df7 = pd.DataFrame(num_desc, columns = ['Importance', 'Candidate Characteristic'])
     df7 = df7[df7['Importance'] != '']
+    values = df7
+    return values
+
+def get_onet_styles(onetCode):
+    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}
+
+    style_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=ws&n_ws=0&c_ws=0"
 
     response = requests.get(style_url, headers=headers)
     soup = BeautifulSoup(response.text, 'html.parser')
@@ -178,10 +217,8 @@ def get_onet_ratings(onetCode):
         num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])
     df8 = pd.DataFrame(num_desc, columns = ['Importance', 'Candidate Characteristic'])
     df8 = df8[df8['Importance'] != '']
-
-    cand_df = pd.concat([df3, df4, df5, df6, df7, df8], axis = 0)
-
-    return [job_df, cand_df]
+    styles = df8
+    return styles
 
 def get_job_postings(onetCode, state):
     headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}