revise flow logic

main.py

@@ -7,10 +7,16 @@ import requests
 from bs4 import BeautifulSoup
 from cleantext import clean
 from docx import Document
+import os
+import cohere
+import string
 import numpy as np
+from numpy.linalg import norm
+from nltk.tokenize import SpaceTokenizer
+import nltk
 from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
-import utils
-from utils import coSkillEmbed, cosine, clean_my_text
+from dotenv import load_dotenv
+load_dotenv()
 
 app = FastAPI()
 app.mount("/static", StaticFiles(directory='static'), name="static")
@@ -18,9 +24,11 @@ templates = Jinja2Templates(directory="templates/")
 
 onet = pd.read_csv('static/ONET_JobTitles.csv')
 simdat = pd.read_csv('static/cohere_embeddings.csv')
+coheredat = pd.read_csv("static/cohere_tSNE_dat.csv")
 
 model = AutoModelForSequenceClassification.from_pretrained('static/model_shards', low_cpu_mem_usage=True)
 tokenizer = AutoTokenizer.from_pretrained('static/tokenizer_shards', low_cpu_mem_usage=True)
+classifier = pipeline('text-classification', model = model, tokenizer = tokenizer)
 
 ### job information center ###
 # get
@@ -67,20 +75,28 @@ def render_job_info(request: Request, jobtitle: str = Form(enum=[x for x in onet
 ### job neighborhoods ###
 @app.get("/explore-job-neighborhoods/", response_class=HTMLResponse)
 async def render_job_neighborhoods(request: Request):
+    def format_title(logo, title, subtitle, title_font_size = 28, subtitle_font_size=14):
+        logo = f'<a href="/" target="_self">{logo}</a>'
+        subtitle = f'<span style="font-size: {subtitle_font_size}px;">{subtitle}</span>'
+        title = f'<span style="font-size: {title_font_size}px;">{title}</span>'
+        return f'{logo}{title}<br>{subtitle}'
+    
+    fig = px.scatter(coheredat, x = 'longitude', y = 'latitude', color = 'Category', hover_data = ['Category', 'Title'], 
+        title=format_title("Pathfinder", "     Job Neighborhoods: Explore the Map!", "(Generated using Co-here AI's LLM & ONET's Task Statements)"))
+    fig['layout'].update(height=1000, width=1500, font=dict(family='Courier New, monospace', color='black'))
+    fig.write_html('templates/job_neighborhoods.html')
+
     return templates.TemplateResponse('job_neighborhoods.html', context={'request': request})
 
 ### find my match ###
 # get
-@app.get("/find-my-match/", response_class=HTMLResponse)
+@app.get("/find-my-match", response_class=HTMLResponse)
 async def match_page(request: Request):
     return templates.TemplateResponse('find_my_match.html', context={'request': request})
 
 # post
-@app.post('/find-my-match/', response_class=HTMLResponse)
+@app.post('/find-my-match', response_class=HTMLResponse)
 def get_resume(request: Request, resume: UploadFile = File(...)):
-
-    classifier = pipeline('text-classification', model = model, tokenizer = tokenizer)
-
     path = f"static/{resume.filename}"
     with open(path, 'wb') as buffer:
         buffer.write(resume.file.read())
@@ -90,6 +106,22 @@ def get_resume(request: Request, resume: UploadFile = File(...)):
         text.append(para.text)
     resume = "\n".join(text)
 
+    def clean_my_text(text):
+        clean_text = ' '.join(text.splitlines())
+        clean_text = clean_text.replace('-', " ").replace("/"," ")
+        clean_text = clean(clean_text.translate(str.maketrans('', '', string.punctuation)))
+        return clean_text
+
+    def coSkillEmbed(text):
+        co = cohere.Client(os.getenv("COHERE_TOKEN"))
+        response = co.embed(
+            model='large',
+            texts=[text])
+        return response.embeddings
+    
+    def cosine(A, B):
+        return np.dot(A,B)/(norm(A)*norm(B))
+
     embeds = coSkillEmbed(resume)
     simResults = []
 
@@ -107,14 +139,28 @@ def get_resume(request: Request, resume: UploadFile = File(...)):
     for x in range(len(simResults)):
         simResults.iloc[x,1] = "{:0.2f}".format(simResults.iloc[x,1])
     
-    cleantext = clean_my_text(resume)
-    labels = []
-    for i in range(len(cleantext)):
-        classification = classifier(cleantext[i])[0]['label']
+    # EXTRACT SKILLS FROM RESUME 
+    def skillNER(resume):
+        resume = clean_my_text(resume)
+        stops = set(nltk.corpus.stopwords.words('english'))
+        stops = stops.union({'eg', 'ie', 'etc', 'experience', 'experiences', 'experienced', 'experiencing', 'knowledge', 
+        'ability', 'abilities', 'skill', 'skills', 'skilled', 'including', 'includes', 'included', 'include'
+        'education', 'follow', 'following', 'follows', 'followed', 'make', 'made', 'makes', 'making', 'maker',
+        'available', 'large', 'larger', 'largescale', 'client', 'clients', 'responsible', 'x', 'many', 'team', 'teams'})
+        resume = [word for word in SpaceTokenizer().tokenize(resume) if word not in stops]
+        resume = [word for word in resume if ")" not in word]
+        resume = [word for word in resume if "(" not in word]
+        
+        labels = []
+        for i in range(len(resume)):
+            classification = classifier(resume[i])[0]['label']
         if classification == 'LABEL_1':
             labels.append("Skill")
         else:
             labels.append("Not Skill")
-        skills = dict(zip(cleantext, labels))
+            labels_dict = dict(zip(resume, labels))
+        return labels_dict
+    
+    skills=skillNER(resume)
 
-    return templates.TemplateResponse('find_my_match.html', context={'request': request, 'resume': resume, 'skills': skills, 'simResults': simResults})
+    return templates.TemplateResponse('find_my_match.html', context={'request': request, 'resume': resume, 'skills': skills, 'simResults': simResults})

utils.py

@@ -1,34 +0,0 @@
-from cleantext import clean
-import cohere
-import string
-import numpy as np
-from numpy.linalg import norm
-from nltk.tokenize import SpaceTokenizer
-import nltk
-import os
-from dotenv import load_dotenv
-load_dotenv()
-
-def coSkillEmbed(text):
-    co = cohere.Client(os.getenv("COHERE_TOKEN"))
-    response = co.embed(
-        model='large',
-        texts=[text])
-    return response.embeddings
-    
-def cosine(A, B):
-    return np.dot(A,B)/(norm(A)*norm(B))
-
-def clean_my_text(resume):
-    clean_text = ' '.join(resume.splitlines())
-    clean_text = clean_text.replace('-', " ").replace("/"," ")
-    clean_text = clean(clean_text.translate(str.maketrans('', '', string.punctuation)))
-    stops = set(nltk.corpus.stopwords.words('english'))
-    stops = stops.union({'eg', 'ie', 'etc', 'experience', 'experiences', 'experienced', 'experiencing', 'knowledge', 
-    'ability', 'abilities', 'skill', 'skills', 'skilled', 'including', 'includes', 'included', 'include'
-    'education', 'follow', 'following', 'follows', 'followed', 'make', 'made', 'makes', 'making', 'maker',
-    'available', 'large', 'larger', 'largescale', 'client', 'clients', 'responsible', 'x', 'many', 'team', 'teams'})
-    resume = [word for word in SpaceTokenizer().tokenize(resume) if word not in stops]
-    resume = [word for word in resume if ")" not in word]
-    resume = [word for word in resume if "(" not in word]
-    return resume