performance optimization enhancements

main.py

@@ -7,13 +7,13 @@
 # License: MIT License
 
 # IMPORTS
-from fastapi import FastAPI, Request, Form, File, UploadFile
+from fastapi import FastAPI, Request, Form, File, UploadFile, BackgroundTasks
 from fastapi.templating import Jinja2Templates
 from fastapi.staticfiles import StaticFiles
 from fastapi.responses import HTMLResponse
 import pandas as pd
 from scrape_onet import get_onet_code, get_onet_description, get_onet_tasks
-from match_utils import get_resume, get_simresults, skillNER
+from match_utils import neighborhoods, get_resume, coSkillEmbed, sim_result_loop, skillNER
 
 # APP SETUP
 app = FastAPI()
@@ -26,18 +26,19 @@ onet = pd.read_csv('static/ONET_JobTitles.csv')
 ### JOB INFORMATION CENTER ###
 # GET
 @app.get("/")
-def render_job_list(request: Request):
+def get_job(request: Request):
     joblist = onet['JobTitle']
     return templates.TemplateResponse('job_list.html', context={'request': request, 'joblist': joblist})
 
 # POST
 @app.post("/")
-def render_job_info(request: Request, jobtitle: str = Form(enum=[x for x in onet['JobTitle']])):
+def post_job(request: Request, bt: BackgroundTasks, jobtitle: str = Form(enum=[x for x in onet['JobTitle']])):
     joblist = onet['JobTitle']
     if jobtitle: 
         onetCode = get_onet_code(jobtitle)
         jobdescription = get_onet_description(onetCode)
         tasks = get_onet_tasks(onetCode)
+        bt.add_task(neighborhoods, jobtitle)
         return templates.TemplateResponse('job_list.html', context={
             'request': request, 
             'joblist': joblist, 
@@ -47,19 +48,20 @@ def render_job_info(request: Request, jobtitle: str = Form(enum=[x for x in onet
 
 ### JOB NEIGHBORHOODS ###
 @app.get("/explore-job-neighborhoods/", response_class=HTMLResponse)
-def render_job_neighborhoods(request: Request):
+async def get_job_neighborhoods(request: Request):
     return templates.TemplateResponse('job_neighborhoods.html', context={'request': request})
 
 ### FIND-MY-MATCH ###
 # GET
 @app.get("/find-my-match/", response_class=HTMLResponse)
-def match_page(request: Request):
+def get_matches(request: Request):
     return templates.TemplateResponse('find_my_match.html', context={'request': request})
 
 # POST
 @app.post('/find-my-match/', response_class=HTMLResponse)
-async def match_page(request: Request, resume: UploadFile = File(...)):
+async def post_matches(request: Request, resume: UploadFile = File(...)):
     resume = get_resume(resume)
-    simResults = await get_simresults(resume)
+    embeds = await coSkillEmbed(resume)
+    simResults = await sim_result_loop(embeds)
     skills = await skillNER(resume)
     return templates.TemplateResponse('find_my_match.html', context={'request': request, 'resume': resume, 'skills': skills, 'simResults': simResults})

match_utils.py

@@ -12,7 +12,7 @@ import numpy as np
 from numpy.linalg import norm
 import ssl
 from dotenv import load_dotenv
-import concurrent.futures
+import plotly_express as px
 
 # SSL CERTIFICATE FIX
 try:
@@ -31,6 +31,7 @@ load_dotenv()
 
 # LOAD COHERE EMBEDDINGS:
 simdat = pd.read_csv('static/cohere_embeddings.csv')
+coheredat = pd.read_csv('static/cohere_tSNE_dat.csv')
 
 # LOAD FINE-TUNED MODEL 
 # (see https://huggingface.co/celise88/distilbert-base-uncased-finetuned-binary-classifier)
@@ -39,6 +40,18 @@ tokenizer = AutoTokenizer.from_pretrained('static/tokenizer_shards', low_cpu_mem
 classifier = pipeline('text-classification', model = model, tokenizer = tokenizer)
 
 # UTILITY FUNCTIONS
+async def neighborhoods(jobtitle=None):
+    def format_title(logo, title, subtitle, title_font_size = 28, subtitle_font_size=14):
+        logo = f'<a href="/" target="_self">{logo}</a>'
+        subtitle = f'<span style="font-size: {subtitle_font_size}px;">{subtitle}</span>'
+        title = f'<span style="font-size: {title_font_size}px;">{title}</span>'
+        return f'{logo}{title}<br>{subtitle}'
+    
+    fig = px.scatter(coheredat, x = 'longitude', y = 'latitude', color = 'Category', hover_data = ['Category', 'Title'], 
+        title=format_title("Pathfinder", "     Job Neighborhoods: Explore the Map!", "(Generated using Co-here AI's LLM & ONET's Task Statements)"))
+    fig['layout'].update(height=1000, width=1500, font=dict(family='Courier New, monospace', color='black'))
+    fig.write_html('templates/job_neighborhoods.html')
+
 def get_resume(resume):
     path = f"static/{resume.filename}"
     with open(path, 'wb') as buffer:
@@ -50,7 +63,7 @@ def get_resume(resume):
     resume = "\n".join(text)
     return resume
 
-def coSkillEmbed(text):
+async def coSkillEmbed(text):
     try:
         co = cohere.Client(os.getenv("COHERE_TOKEN"))
         response = co.embed(
@@ -60,10 +73,9 @@ def coSkillEmbed(text):
     except CohereError as e:
         return e
 
-async def get_simresults(resume):
+async def sim_result_loop(embeds):
     def cosine(A, B):
         return np.dot(A,B)/(norm(A)*norm(B))
-    embeds = coSkillEmbed(resume)
     simResults = []
     for i in range(len(simdat)):
         simResults.append(cosine(np.array(embeds), np.array(simdat.iloc[i,1:])))
@@ -103,5 +115,5 @@ async def skillNER(resume):
             labels.append("Skill")
         else:
             labels.append("Not Skill")
-        labels_dict = dict(zip(resume, labels))
-    return labels_dict
+        skills = dict(zip(resume, labels))
+    return skills

static/cohere_tSNE_dat.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac3dbbea21867638654b3c399b988ca95c5573cc602383d8835cffe36952a7cb
+size 1858107