Spaces:
Sleeping
Sleeping
Update skill_extraction.py
Browse files- skill_extraction.py +13 -8
skill_extraction.py
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
|
|
|
|
|
| 1 |
import json
|
| 2 |
import spacy
|
| 3 |
import numpy as np
|
|
@@ -18,9 +20,12 @@ model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
|
|
| 18 |
with open("skills_vocab.json", "r", encoding="utf-8") as f:
|
| 19 |
SKILL_VOCAB = json.load(f)["skills"]
|
| 20 |
|
| 21 |
-
# Precompute
|
| 22 |
skill_embeddings = model.encode(SKILL_VOCAB, convert_to_numpy=True, normalize_embeddings=True)
|
| 23 |
|
|
|
|
|
|
|
|
|
|
| 24 |
def extract_skills(text, threshold=0.50):
|
| 25 |
doc = nlp(text)
|
| 26 |
sentences = [sent.text.strip() for sent in doc.sents if sent.text.strip()]
|
|
@@ -29,17 +34,17 @@ def extract_skills(text, threshold=0.50):
|
|
| 29 |
return []
|
| 30 |
|
| 31 |
sentence_embeddings = model.encode(sentences, convert_to_numpy=True, normalize_embeddings=True)
|
|
|
|
| 32 |
|
| 33 |
-
extracted = []
|
| 34 |
-
|
| 35 |
-
# Iterate skill-by-skill
|
| 36 |
for j, skill_emb in enumerate(skill_embeddings):
|
| 37 |
sims = np.dot(sentence_embeddings, skill_emb)
|
| 38 |
max_sim = float(np.max(sims))
|
| 39 |
if max_sim >= threshold:
|
| 40 |
-
|
| 41 |
-
extracted.append((SKILL_VOCAB[j], round(max_sim, 2), best_sentence))
|
| 42 |
|
| 43 |
-
|
| 44 |
-
return sorted(extracted, key=lambda x: x[1], reverse=True)
|
| 45 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# skill_extraction.py
|
| 2 |
+
|
| 3 |
import json
|
| 4 |
import spacy
|
| 5 |
import numpy as np
|
|
|
|
| 20 |
with open("skills_vocab.json", "r", encoding="utf-8") as f:
|
| 21 |
SKILL_VOCAB = json.load(f)["skills"]
|
| 22 |
|
| 23 |
+
# Precompute embeddings
|
| 24 |
skill_embeddings = model.encode(SKILL_VOCAB, convert_to_numpy=True, normalize_embeddings=True)
|
| 25 |
|
| 26 |
+
# Map skill -> embedding for quick lookup
|
| 27 |
+
SKILL_TO_EMB = {skill: emb for skill, emb in zip(SKILL_VOCAB, skill_embeddings)}
|
| 28 |
+
|
| 29 |
def extract_skills(text, threshold=0.50):
|
| 30 |
doc = nlp(text)
|
| 31 |
sentences = [sent.text.strip() for sent in doc.sents if sent.text.strip()]
|
|
|
|
| 34 |
return []
|
| 35 |
|
| 36 |
sentence_embeddings = model.encode(sentences, convert_to_numpy=True, normalize_embeddings=True)
|
| 37 |
+
skill_confidences = {}
|
| 38 |
|
|
|
|
|
|
|
|
|
|
| 39 |
for j, skill_emb in enumerate(skill_embeddings):
|
| 40 |
sims = np.dot(sentence_embeddings, skill_emb)
|
| 41 |
max_sim = float(np.max(sims))
|
| 42 |
if max_sim >= threshold:
|
| 43 |
+
skill_confidences[SKILL_VOCAB[j]] = round(max_sim, 2)
|
|
|
|
| 44 |
|
| 45 |
+
return sorted(skill_confidences.items(), key=lambda x: x[1], reverse=True)
|
|
|
|
| 46 |
|
| 47 |
+
# --- NEW FUNCTION ---
|
| 48 |
+
def get_skill_embedding(skill_name):
|
| 49 |
+
"""Return the embedding vector for a skill, or None if not in vocab"""
|
| 50 |
+
return SKILL_TO_EMB.get(skill_name)
|