Spaces:
Running
Running
File size: 7,628 Bytes
1177cc0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | """
task3_internship_profiles_module.py
────────────────────────────────────
Extracted from task3_internship_profiles.ipynb for import in Task 4.
Contains: boost engine, InternshipProfile, generate_internship_profile.
Usage:
from task3_internship_profiles_module import (
boost_tfidf_vector,
generate_internship_profile,
InternshipProfile,
detect_category,
extract_skills_from_text,
load_skill_clusters,
)
"""
import json
import numpy as np
from dataclasses import dataclass, field
from typing import List
from collections import defaultdict
from task1_module import flatten, tfidf_transform
from task2_module import bert_transform
# ─────────────────────────────────────────────────────────────────────────────
# Loaders
# ─────────────────────────────────────────────────────────────────────────────
def load_skill_clusters(path: str = "skill_clusters.json") -> dict:
with open(path) as f:
return json.load(f)
def load_skill_to_category(path: str = "skill_to_category.json") -> dict:
with open(path) as f:
return json.load(f)
def build_skill_to_category_map(skill_clusters: dict) -> dict:
"""Invert cluster dict: { skill: category }. First occurrence wins."""
skill_to_category = {}
for category, skills in skill_clusters.items():
for skill in skills:
if skill not in skill_to_category:
skill_to_category[skill] = category
return skill_to_category
# ─────────────────────────────────────────────────────────────────────────────
# Helpers
# ─────────────────────────────────────────────────────────────────────────────
def detect_category(flat_text: str,
skill_to_category: dict,
skill_clusters: dict) -> str:
"""Vote-based: category with most skill-token matches wins."""
votes = defaultdict(int)
tokens = set(flat_text.lower().split())
for token in tokens:
if token in skill_to_category:
votes[skill_to_category[token]] += 1
return max(votes, key=votes.get) if votes else "Unknown"
def extract_skills_from_text(flat_text: str,
skill_to_category: dict) -> List[str]:
"""Return all cluster-skill tokens found in the text."""
tokens = set(flat_text.lower().split())
return sorted([t for t in tokens if t in skill_to_category])
# ─────────────────────────────────────────────────────────────────────────────
# Boost engine
# ─────────────────────────────────────────────────────────────────────────────
def boost_tfidf_vector(base_vector: np.ndarray,
text: str,
vectorizer,
skill_clusters: dict,
skill_to_category: dict,
boost_weight: float = 0.15) -> np.ndarray:
"""
Add cluster-sibling skill weights to a TF-IDF vector, then re-normalise.
If the text mentions 'react', all Frontend siblings (vue, angular, js …)
get +boost_weight in the vector — even if they weren't in the text.
"""
vocab = vectorizer.vocabulary_
boosted = base_vector.copy()
tokens_in_text = set(text.lower().split())
boosted_categories = set()
for token in tokens_in_text:
if token not in skill_to_category:
continue
category = skill_to_category[token]
if category in boosted_categories:
continue
boosted_categories.add(category)
for sibling in skill_clusters.get(category, []):
if sibling in vocab and sibling not in tokens_in_text:
boosted[vocab[sibling]] += boost_weight
norm = np.linalg.norm(boosted)
return boosted / norm if norm > 0 else boosted
# ─────────────────────────────────────────────────────────────────────────────
# InternshipProfile dataclass
# ─────────────────────────────────────────────────────────────────────────────
@dataclass
class InternshipProfile:
title: str
raw_description: str
flat_text: str = field(repr=False)
detected_category: str = "Unknown"
required_skills: List[str] = field(default_factory=list)
tfidf_vector: np.ndarray = field(default=None, repr=False)
bert_vector: np.ndarray = field(default=None, repr=False)
hybrid_vector: np.ndarray = field(default=None, repr=False)
# ─────────────────────────────────────────────────────────────────────────────
# Main generator
# ─────────────────────────────────────────────────────────────────────────────
def generate_internship_profile(
title: str,
description: str,
tfidf_vectorizer,
bert_model,
skill_clusters: dict,
skill_to_category: dict,
boost_weight: float = 0.15,
) -> InternshipProfile:
"""
Raw internship description → InternshipProfile with hybrid vectors.
"""
flat_text = flatten(description, mode="text")
detected_category = detect_category(flat_text, skill_to_category, skill_clusters)
required_skills = extract_skills_from_text(flat_text, skill_to_category)
base_tfidf = tfidf_transform(description, tfidf_vectorizer, mode="text")
boosted_tfidf = boost_tfidf_vector(
base_vector = base_tfidf,
text = flat_text,
vectorizer = tfidf_vectorizer,
skill_clusters = skill_clusters,
skill_to_category = skill_to_category,
boost_weight = boost_weight,
)
bert_vec = bert_transform(description, bert_model, mode="text")
return InternshipProfile(
title = title,
raw_description = description,
flat_text = flat_text,
detected_category = detected_category,
required_skills = required_skills,
tfidf_vector = boosted_tfidf,
bert_vector = bert_vec,
hybrid_vector = np.concatenate([boosted_tfidf, bert_vec]),
)
|