Eng-Musa commited on
Commit
dfafe5a
·
1 Parent(s): a3b3079

Delete cv utils and fetch them from the scrapper

Browse files
main.py CHANGED
@@ -20,6 +20,7 @@ from services.cv_chunker import chunk_cv
20
  from services.cv_converter import CVConverter
21
  from services.job_matcher import JobInput, JobMatcher
22
  from services.workers import CvWorker, QueueManager, CvTask
 
23
 
24
  # ---------------------------------------------------------------------------
25
  # Configuration
@@ -61,6 +62,7 @@ queue_manager = QueueManager(cv_worker, concurrency=1)
61
 
62
  @asynccontextmanager
63
  async def lifespan(app: FastAPI):
 
64
  await queue_manager.start()
65
  yield
66
  await queue_manager.stop()
 
20
  from services.cv_converter import CVConverter
21
  from services.job_matcher import JobInput, JobMatcher
22
  from services.workers import CvWorker, QueueManager, CvTask
23
+ from services.taxonomy_client import load_taxonomy
24
 
25
  # ---------------------------------------------------------------------------
26
  # Configuration
 
62
 
63
  @asynccontextmanager
64
  async def lifespan(app: FastAPI):
65
+ load_taxonomy(max_retries=15, initial_delay=3.0)
66
  await queue_manager.start()
67
  yield
68
  await queue_manager.stop()
services/cv_chunker.py CHANGED
@@ -3,7 +3,7 @@ from __future__ import annotations
3
  import re
4
  from typing import Any, Dict, List, Optional, Set, Tuple
5
 
6
- from services.cv_utils import (
7
  DEFAULT_SKILL_ALIASES,
8
  DEFAULT_CATEGORY_SKILLS,
9
  SOFT_SKILL_KEYS,
@@ -774,6 +774,8 @@ def chunk_cv(cv_markdown: str, embedder=None) -> Dict[str, Any]:
774
  contact_chunk = _parse_contact(sections.get("contact", ""), cv_markdown)
775
  links_chunk = _parse_links(sections.get("links", ""), cv_markdown)
776
  skills_chunk = _parse_skills(sections.get("skills", ""), cv_markdown)
 
 
777
  experience_entries = _parse_experience(sections.get("experience", ""))
778
  education_entries = _parse_education(sections.get("education", ""))
779
  projects_entries = _parse_projects(sections.get("projects", ""))
 
3
  import re
4
  from typing import Any, Dict, List, Optional, Set, Tuple
5
 
6
+ from services.taxonomy_client import (
7
  DEFAULT_SKILL_ALIASES,
8
  DEFAULT_CATEGORY_SKILLS,
9
  SOFT_SKILL_KEYS,
 
774
  contact_chunk = _parse_contact(sections.get("contact", ""), cv_markdown)
775
  links_chunk = _parse_links(sections.get("links", ""), cv_markdown)
776
  skills_chunk = _parse_skills(sections.get("skills", ""), cv_markdown)
777
+ if not skills_chunk["items"]:
778
+ skills_chunk["items"] = sorted(list(cv_skills))
779
  experience_entries = _parse_experience(sections.get("experience", ""))
780
  education_entries = _parse_education(sections.get("education", ""))
781
  projects_entries = _parse_projects(sections.get("projects", ""))
services/cv_utils.py DELETED
@@ -1,613 +0,0 @@
1
- """
2
- cv_utils.py
3
- ===========
4
- Shared taxonomy data and pure helper functions used by both
5
- cv_chunker.py and job_matcher.py.
6
-
7
- All skill aliases, category mappings, seniority patterns, and
8
- normalisation utilities live here so both consumers stay in sync.
9
- """
10
- from __future__ import annotations
11
-
12
- import re
13
- from datetime import datetime as _dt
14
- from typing import Any, Dict, List, Optional, Set, Tuple
15
-
16
-
17
- # ---------------------------------------------------------------------------
18
- # Universal Skill Taxonomy
19
- # ---------------------------------------------------------------------------
20
-
21
- DEFAULT_SKILL_ALIASES: Dict[str, List[str]] = {
22
- # ── Office / Admin ──────────────────────────────────────────────────────
23
- "microsoft word": ["microsoft word", "ms word", "word"],
24
- "microsoft excel": ["microsoft excel", "ms excel", "excel", "spreadsheets"],
25
- "microsoft powerpoint": ["microsoft powerpoint", "ms powerpoint", "powerpoint", "power point"],
26
- "microsoft outlook": ["microsoft outlook", "ms outlook", "outlook"],
27
- "microsoft office": ["microsoft office", "ms office", "office suite", "office 365", "microsoft 365"],
28
- "google workspace": ["google workspace", "google docs", "google sheets", "google slides", "gsuite", "g suite"],
29
- "data entry": ["data entry", "typing", "data input", "data capture"],
30
- "filing": ["filing", "document management", "records management", "file management", "document control"],
31
- "scheduling": ["scheduling", "calendar management", "appointment setting", "diary management"],
32
- "communication": ["communication", "communicate", "interpersonal", "verbal communication", "written communication"],
33
- "teamwork": ["teamwork", "team work", "team player", "collaboration", "team member"],
34
- "customer service": ["customer service", "client service", "customer support", "client relations"],
35
- "problem solving": ["problem solving", "problem-solving", "critical thinking", "analytical thinking"],
36
- "time management": ["time management", "prioritization", "prioritisation", "multitasking"],
37
- "reporting": ["reporting", "report writing", "report generation", "daily report", "weekly report"],
38
- "inventory management": ["inventory management", "inventory", "stock control", "stock management", "stock checking"],
39
- "warehouse": ["warehouse", "stockroom", "store room", "stores"],
40
- "leadership": ["leadership", "team leadership", "people management"],
41
- "attention to detail": ["attention to detail", "accuracy", "meticulous"],
42
- "adaptability": ["adaptability", "adaptable", "flexible"],
43
-
44
- # ── Finance & Accounting ────────────────────────────────────────────────
45
- "accounting": ["accounting", "accountancy", "bookkeeping", "accounts"],
46
- "financial analysis": ["financial analysis", "financial modelling", "financial modeling", "financial reporting", "financial planning"],
47
- "auditing": ["auditing", "internal audit", "external audit", "audit", "statutory audit"],
48
- "taxation": ["taxation", "tax", "tax compliance", "tax preparation", "tax returns", "vat"],
49
- "payroll": ["payroll", "payroll processing", "payroll management", "payroll administration"],
50
- "budgeting": ["budgeting", "budget management", "budget planning", "forecasting", "variance analysis"],
51
- "accounts payable": ["accounts payable", "vendor payments", "creditors"],
52
- "accounts receivable": ["accounts receivable", "invoicing", "debtors", "collections"],
53
- "quickbooks": ["quickbooks", "quick books"],
54
- "sage": ["sage", "sage accounting", "sage 50", "sage 200"],
55
- "sap": ["sap", "sap finance", "sap erp", "sap s/4hana", "sap fico"],
56
- "xero": ["xero"],
57
- "ifrs": ["ifrs", "international financial reporting standards"],
58
- "gaap": ["gaap", "generally accepted accounting principles"],
59
- "reconciliation": ["reconciliation", "bank reconciliation", "account reconciliation"],
60
- "cash flow": ["cash flow", "cash management", "treasury"],
61
-
62
- # ── Human Resources ─────────────────────────────────────────────────────
63
- "recruitment": ["recruitment", "recruiting", "talent acquisition", "hiring", "staffing", "talent sourcing"],
64
- "onboarding": ["onboarding", "new hire orientation", "induction"],
65
- "performance management":["performance management", "performance review", "appraisal", "kpi management"],
66
- "employee relations": ["employee relations", "labour relations", "labor relations", "grievance handling"],
67
- "training and development": ["training", "learning and development", "l&d", "employee training", "talent development"],
68
- "hris": ["hris", "hr information system", "workday", "bamboohr", "zoho hr", "oracle hcm"],
69
- "compensation": ["compensation", "benefits", "compensation and benefits", "remuneration", "total rewards"],
70
- "hr compliance": ["hr compliance", "compliance", "regulatory compliance", "employment law"],
71
- "job analysis": ["job analysis", "job description", "job evaluation"],
72
-
73
- # ── Sales & Business Development ────────────────────────────────────────
74
- "sales": ["sales", "selling", "business development", "b2b sales", "b2c sales", "direct sales"],
75
- "crm": ["crm", "salesforce", "hubspot", "zoho crm", "customer relationship management", "pipedrive"],
76
- "negotiation": ["negotiation", "contract negotiation", "deal closing"],
77
- "lead generation": ["lead generation", "prospecting", "cold calling", "cold outreach"],
78
- "account management": ["account management", "key account management", "client management"],
79
- "retail": ["retail", "retail sales", "merchandising", "point of sale", "pos"],
80
- "business analysis": ["business analysis", "business analyst", "requirements gathering", "process improvement"],
81
-
82
- # ── Customer Support ────────────────────────────────────────────────────
83
- "help desk": ["help desk", "helpdesk", "it support", "technical support", "service desk"],
84
- "ticketing systems": ["ticketing", "zendesk", "freshdesk", "jira service desk", "servicenow"],
85
- "call centre": ["call centre", "call center", "contact centre", "contact center", "inbound calls", "outbound calls"],
86
-
87
- # ── Healthcare ──────────────────────────────────────────────────────────
88
- "patient care": ["patient care", "patient management", "bedside manner", "patient assessment"],
89
- "nursing": ["nursing", "registered nurse", "rn", "clinical nursing", "enrolled nurse"],
90
- "clinical skills": ["clinical", "clinical skills", "clinical assessment", "clinical procedures"],
91
- "pharmacy": ["pharmacy", "dispensing", "pharmaceutical", "pharmacology"],
92
- "first aid": ["first aid", "cpr", "bls", "basic life support", "acls", "emergency response"],
93
- "medical records": ["medical records", "ehr", "electronic health records", "emr", "medical documentation"],
94
- "infection control": ["infection control", "infection prevention", "ips", "sterilization"],
95
- "midwifery": ["midwifery", "obstetrics", "maternity", "antenatal", "postnatal"],
96
- "physiotherapy": ["physiotherapy", "physical therapy", "rehabilitation"],
97
- "laboratory": ["laboratory", "lab technician", "medical laboratory", "specimen processing"],
98
-
99
- # ── Legal ───────────────────────────────────────────────────────────────
100
- "legal research": ["legal research", "case research", "statute interpretation"],
101
- "contract drafting": ["contract drafting", "contract review", "drafting agreements", "legal drafting"],
102
- "litigation": ["litigation", "court proceedings", "legal proceedings", "advocacy"],
103
- "legal compliance": ["legal compliance", "regulatory affairs", "regulatory compliance"],
104
- "conveyancing": ["conveyancing", "property law", "land transactions"],
105
- "corporate law": ["corporate law", "company law", "corporate governance"],
106
- "legal writing": ["legal writing", "pleadings", "legal opinions", "memoranda"],
107
-
108
- # ── Education & Training ────────────────────────────────────────────────
109
- "teaching": ["teaching", "instruction", "lecturing", "tutoring", "classroom teaching"],
110
- "curriculum development":["curriculum development", "curriculum design", "lesson planning", "scheme of work"],
111
- "classroom management": ["classroom management", "student management", "behaviour management"],
112
- "e-learning": ["e-learning", "elearning", "lms", "moodle", "online learning", "instructional design"],
113
- "assessment": ["assessment", "grading", "marking", "examination", "evaluation"],
114
- "mentoring": ["mentoring", "mentorship", "coaching", "academic guidance"],
115
-
116
- # ── Operations & Logistics ──────────────────────────────────────────────
117
- "supply chain": ["supply chain", "supply chain management", "scm"],
118
- "procurement": ["procurement", "purchasing", "vendor management", "sourcing", "tendering"],
119
- "logistics": ["logistics", "fleet management", "transportation management", "freight", "delivery"],
120
- "quality control": ["quality control", "qc", "quality assurance", "qa", "iso", "quality management"],
121
- "project management": ["project management", "pmp", "prince2", "agile project management", "project planning"],
122
- "lean": ["lean", "lean manufacturing", "lean six sigma", "continuous improvement", "kaizen"],
123
- "six sigma": ["six sigma", "6 sigma", "dmaic"],
124
-
125
- # ── Software Engineering ────────────────────────────────────────────────
126
- "html": ["html", "html5"],
127
- "css": ["css", "css3"],
128
- "javascript": ["javascript", "js"],
129
- "typescript": ["typescript", "ts"],
130
- "react": ["react", "react.js", "reactjs"],
131
- "next.js": ["next.js", "nextjs"],
132
- "vue": ["vue", "vue.js", "vuejs"],
133
- "angular": ["angular"],
134
- "tailwind css": ["tailwind", "tailwind css"],
135
- "node.js": ["node", "node.js", "nodejs"],
136
- "express.js": ["express", "express.js"],
137
- "python": ["python"],
138
- "java": ["java"],
139
- "kotlin": ["kotlin"],
140
- "android": ["android", "android studio"],
141
- "php": ["php"],
142
- "laravel": ["laravel"],
143
- "mysql": ["mysql"],
144
- "postgresql": ["postgresql", "postgres", "supabase"],
145
- "mongodb": ["mongodb", "mongo"],
146
- "rest api": ["rest api", "restful api", "api development"],
147
- "git": ["git", "github", "gitlab"],
148
- "docker": ["docker"],
149
- "spring boot": ["spring boot", "spring", "spring framework"],
150
- "ci/cd": ["ci/cd", "ci cd", "github actions", "jenkins", "gitlab ci", "pipelines", "devops"],
151
- "kubernetes": ["kubernetes", "k8s"],
152
- "aws": ["aws", "amazon web services"],
153
- "azure": ["azure", "microsoft azure"],
154
- "gcp": ["gcp", "google cloud"],
155
- "flutter": ["flutter"],
156
- "dart": ["dart"],
157
- "swift": ["swift"],
158
- "ios": ["ios", "xcode"],
159
- "c#": ["c#", "csharp", "c sharp"],
160
- "c++": ["c++", "cpp"],
161
- "rust": ["rust"],
162
- "go": ["golang", "go lang"],
163
- "redis": ["redis"],
164
- "elasticsearch": ["elasticsearch", "elastic search"],
165
- "graphql": ["graphql"],
166
- "machine learning": ["machine learning", "ml", "deep learning", "neural networks"],
167
- "sql": ["sql", "pl/sql", "t-sql"],
168
- "data analysis": ["data analysis", "analytics", "data analytics"],
169
-
170
- # ── Design & Creative ───────────────────────────────────────────────────
171
- "ui design": ["ui design", "user interface design", "ui"],
172
- "ux design": ["ux design", "user experience design", "ux", "usability"],
173
- "figma": ["figma"],
174
- "wireframing": ["wireframe", "wireframing"],
175
- "prototyping": ["prototype", "prototyping"],
176
- "adobe photoshop": ["photoshop", "adobe photoshop"],
177
- "adobe illustrator": ["illustrator", "adobe illustrator"],
178
- "adobe xd": ["adobe xd", "xd"],
179
- "graphic design": ["graphic design", "graphics", "visual design"],
180
- "video editing": ["video editing", "video production", "premiere pro", "final cut pro", "davinci resolve"],
181
- "photography": ["photography", "photo editing", "lightroom", "photo retouching"],
182
-
183
- # ── Marketing ───────────────────────────────────────────────────────────
184
- "seo": ["seo", "search engine optimization", "search engine optimisation"],
185
- "content writing": ["content writing", "copywriting", "content creation", "blog writing"],
186
- "digital marketing": ["digital marketing", "online marketing", "marketing campaigns", "marketing strategy"],
187
- "social media": ["social media", "social media management", "instagram", "facebook marketing", "tiktok", "twitter"],
188
- "email marketing": ["email marketing", "mailchimp", "email campaigns", "newsletter"],
189
- "google analytics": ["google analytics", "ga4", "web analytics"],
190
- "google ads": ["google ads", "google adwords", "ppc", "pay per click", "sem"],
191
- "facebook ads": ["facebook ads", "meta ads", "social media advertising"],
192
- "brand management": ["brand management", "branding", "brand strategy"],
193
-
194
- # ── Data & Analytics ────────────────────────────────────────────────────
195
- "power bi": ["power bi", "powerbi", "power bi desktop"],
196
- "tableau": ["tableau"],
197
- "r": ["r programming", "r language", "rstudio"],
198
- "data science": ["data science", "data scientist"],
199
- "statistics": ["statistics", "statistical analysis", "spss", "stata"],
200
- "data visualization": ["data visualization", "data visualisation", "data viz"],
201
- }
202
-
203
-
204
- DEFAULT_CATEGORY_SKILLS: Dict[str, Set[str]] = {
205
- "Admin & Office": {
206
- "microsoft word", "microsoft excel", "microsoft powerpoint", "microsoft office",
207
- "google workspace", "data entry", "filing", "scheduling", "communication",
208
- "teamwork", "customer service", "reporting", "inventory management",
209
- "warehouse", "time management", "attention to detail",
210
- },
211
- "Software Engineering": {
212
- "html", "css", "javascript", "typescript", "react", "next.js", "vue",
213
- "angular", "tailwind css", "node.js", "express.js", "python", "java",
214
- "kotlin", "android", "php", "laravel", "mysql", "postgresql", "mongodb",
215
- "rest api", "git", "docker", "spring boot", "ci/cd", "kubernetes",
216
- "aws", "azure", "gcp", "flutter", "dart", "swift", "ios", "c#", "c++",
217
- "rust", "go", "redis", "elasticsearch", "graphql", "sql",
218
- },
219
- "Design & Creative": {
220
- "ui design", "ux design", "figma", "wireframing", "prototyping",
221
- "adobe photoshop", "adobe illustrator", "adobe xd", "graphic design",
222
- "video editing", "photography",
223
- },
224
- "Marketing & Growth": {
225
- "seo", "content writing", "digital marketing", "social media",
226
- "email marketing", "google analytics", "google ads", "facebook ads",
227
- "brand management",
228
- },
229
- "Data & Analytics": {
230
- "sql", "python", "machine learning", "data analysis", "microsoft excel",
231
- "power bi", "tableau", "r", "data science", "statistics", "data visualization",
232
- "elasticsearch",
233
- },
234
- "Finance & Accounting": {
235
- "accounting", "financial analysis", "auditing", "taxation", "payroll",
236
- "budgeting", "accounts payable", "accounts receivable", "quickbooks",
237
- "sage", "sap", "xero", "ifrs", "gaap", "reconciliation", "cash flow",
238
- "microsoft excel",
239
- },
240
- "Human Resources": {
241
- "recruitment", "onboarding", "performance management", "employee relations",
242
- "training and development", "hris", "compensation", "hr compliance",
243
- "job analysis", "communication", "microsoft excel",
244
- },
245
- "Sales & Business Dev": {
246
- "sales", "crm", "negotiation", "lead generation", "account management",
247
- "retail", "communication", "customer service",
248
- },
249
- "Healthcare": {
250
- "patient care", "nursing", "clinical skills", "pharmacy", "first aid",
251
- "medical records", "infection control", "midwifery", "physiotherapy",
252
- "laboratory",
253
- },
254
- "Legal": {
255
- "legal research", "contract drafting", "litigation", "legal compliance",
256
- "conveyancing", "corporate law", "legal writing",
257
- },
258
- "Education & Training": {
259
- "teaching", "curriculum development", "classroom management", "e-learning",
260
- "assessment", "mentoring", "communication",
261
- },
262
- "Operations & Logistics": {
263
- "supply chain", "procurement", "logistics", "quality control",
264
- "project management", "lean", "six sigma", "inventory management",
265
- "reporting",
266
- },
267
- "Customer Support": {
268
- "customer service", "help desk", "ticketing systems", "call centre",
269
- "communication", "crm", "reporting",
270
- },
271
- }
272
-
273
-
274
- SENIORITY_PATTERNS: Dict[str, List[str]] = {
275
- "intern": [
276
- "intern", "internship", "trainee", "industrial attachment",
277
- "attachment student", "graduate trainee", "pupil",
278
- ],
279
- "junior": [
280
- "junior", "entry level", "entry-level", "assistant", "fresh graduate",
281
- "graduate", "associate", "junior officer", "junior analyst",
282
- ],
283
- "mid": [
284
- "mid", "middle", "officer", "specialist", "developer", "designer",
285
- "engineer", "analyst", "coordinator", "technician", "executive",
286
- "consultant", "representative",
287
- ],
288
- "senior": [
289
- "senior", "lead", "manager", "head", "principal", "architect",
290
- "director", "vp", "vice president", "chief", "cto", "ceo", "coo",
291
- "cfo", "superintendent", "supervisor", "managing",
292
- ],
293
- }
294
-
295
- SENIORITY_ORDER: Dict[str, int] = {"intern": 0, "junior": 1, "mid": 2, "senior": 3}
296
-
297
-
298
- # Words that look like "City, Country" but are actually tech terms
299
- _TECH_LOC_BLACKLIST: Set[str] = {
300
- "java", "spring", "boot", "sql", "python", "node", "react", "angular",
301
- "vue", "docker", "linux", "windows", "android", "swift", "kotlin",
302
- "scala", "ruby", "rails", "django", "flask", "express", "mongo",
303
- "postgres", "redis", "kafka", "hadoop", "spark", "airflow", "jenkins",
304
- "gitlab", "github", "bitbucket", "jira", "confluence", "slack", "figma",
305
- "oracle", "mysql", "azure", "aws", "gcp",
306
- }
307
-
308
-
309
- # ---------------------------------------------------------------------------
310
- # Soft-skill keys (used by chunker to split canonical list)
311
- # ---------------------------------------------------------------------------
312
-
313
- SOFT_SKILL_KEYS: Set[str] = {
314
- "communication", "teamwork", "problem solving", "time management",
315
- "customer service", "mentoring", "negotiation", "leadership",
316
- "reporting", "scheduling", "filing", "adaptability", "attention to detail",
317
- }
318
-
319
-
320
- # ---------------------------------------------------------------------------
321
- # Title role words & phrase regex (universal)
322
- # ---------------------------------------------------------------------------
323
-
324
- TITLE_ROLE_WORDS: List[str] = [
325
- # Tech
326
- "developer", "engineer", "programmer", "architect", "devops",
327
- # General
328
- "officer", "designer", "assistant", "accountant", "manager", "analyst",
329
- "specialist", "coordinator", "director", "executive", "lead", "scientist",
330
- "strategist", "advisor", "trainer", "consultant", "researcher", "technician",
331
- "supervisor", "associate", "administrator",
332
- # Healthcare
333
- "nurse", "doctor", "physician", "pharmacist", "therapist", "clinician",
334
- "midwife", "radiographer", "physiotherapist",
335
- # Finance
336
- "auditor", "bookkeeper", "controller", "treasurer",
337
- # HR
338
- "recruiter",
339
- # Sales
340
- "salesperson", "representative", "agent", "broker",
341
- # Education
342
- "teacher", "instructor", "lecturer", "professor", "tutor",
343
- # Other
344
- "admin", "intern", "student", "cashier", "marketing",
345
- "procurement", "buyer", "dispatcher",
346
- ]
347
-
348
- TITLE_PHRASE_RE = re.compile(
349
- r"\b((?:(?:full[\s\-]?stack|front[\s\-]?end|back[\s\-]?end|senior|junior|lead|"
350
- r"mid[\s\-]?level|entry[\s\-]?level|chief|head\s+of)\s+)?(?:[\w\-]+\s){0,3}"
351
- r"(?:developer|engineer|designer|analyst|manager|specialist|consultant|architect|"
352
- r"programmer|researcher|officer|director|executive|scientist|trainer|accountant|"
353
- r"nurse|doctor|pharmacist|therapist|teacher|instructor|lecturer|recruiter|"
354
- r"coordinator|supervisor|administrator|technician|advisor|auditor|bookkeeper|"
355
- r"physiotherapist|midwife|clinician|representative|salesperson|buyer|dispatcher))\b",
356
- re.IGNORECASE,
357
- )
358
-
359
-
360
- # ---------------------------------------------------------------------------
361
- # Normalisation helpers
362
- # ---------------------------------------------------------------------------
363
-
364
- def _normalize(text: str) -> str:
365
- text = (text or "").lower()
366
- text = text.replace("/", " ").replace("_", " ")
367
- text = re.sub(r"[^a-z0-9+.#\s-]", " ", text)
368
- return re.sub(r"\s+", " ", text).strip()
369
-
370
-
371
- def _has_alias(text_norm: str, alias: str) -> bool:
372
- alias_norm = _normalize(alias)
373
- if not alias_norm:
374
- return False
375
- return re.search(rf"(?<![a-z0-9]){re.escape(alias_norm)}(?![a-z0-9])", text_norm) is not None
376
-
377
-
378
- # ---------------------------------------------------------------------------
379
- # Skill extraction
380
- # ---------------------------------------------------------------------------
381
-
382
- def _extract_skills(
383
- text: str,
384
- aliases: Optional[Dict[str, List[str]]] = None,
385
- ) -> Set[str]:
386
- if aliases is None:
387
- aliases = DEFAULT_SKILL_ALIASES
388
- text_norm = _normalize(text)
389
- return {
390
- canon
391
- for canon, ali_list in aliases.items()
392
- if any(_has_alias(text_norm, a) for a in ali_list)
393
- }
394
-
395
-
396
- # ---------------------------------------------------------------------------
397
- # Category inference (universal)
398
- # ---------------------------------------------------------------------------
399
-
400
- _CATEGORY_TITLE_KEYWORDS: Dict[str, List[str]] = {
401
- "Admin & Office": ["admin", "office", "stock", "inventory", "warehouse", "cashier", "receptionist", "clerk"],
402
- "Software Engineering": ["developer", "engineer", "programmer", "frontend", "backend", "full stack", "mobile", "fullstack", "software", "devops"],
403
- "Design & Creative": ["designer", "ui", "ux", "creative", "graphic", "visual", "illustrator", "photographer"],
404
- "Marketing & Growth": ["marketing", "seo", "content", "social media", "growth", "digital", "brand"],
405
- "Data & Analytics": ["data", "analyst", "analytics", "machine learning", "ai", "scientist", "bi", "intelligence"],
406
- "Finance & Accounting": ["accountant", "finance", "accounting", "auditor", "bookkeeper", "treasurer", "payroll", "tax", "financial"],
407
- "Human Resources": ["hr", "human resource", "recruitment", "recruiter", "talent", "people", "hris"],
408
- "Sales & Business Dev": ["sales", "business development", "account manager", "sales executive", "representative"],
409
- "Healthcare": ["nurse", "doctor", "clinical", "medical", "pharmacy", "patient", "health", "therapist", "midwife", "radiographer"],
410
- "Legal": ["legal", "lawyer", "attorney", "advocate", "counsel", "paralegal", "solicitor"],
411
- "Education & Training": ["teacher", "lecturer", "tutor", "instructor", "educator", "academic", "trainer", "professor"],
412
- "Operations & Logistics":["logistics", "supply chain", "procurement", "operations", "fleet", "warehouse manager", "quality"],
413
- "Customer Support": ["customer support", "help desk", "call centre", "contact centre", "customer care"],
414
- }
415
-
416
-
417
- def _infer_category(
418
- text: str,
419
- skills: Set[str],
420
- category_skills: Optional[Dict[str, Set[str]]] = None,
421
- ) -> str:
422
- if category_skills is None:
423
- category_skills = DEFAULT_CATEGORY_SKILLS
424
- text_norm = _normalize(text)
425
- best, best_score = "Other", 0.0
426
- for cat, cat_skills in category_skills.items():
427
- skill_overlap = len(skills & cat_skills)
428
- kw_hits = sum(1 for kw in _CATEGORY_TITLE_KEYWORDS.get(cat, []) if kw in text_norm)
429
- score = skill_overlap * 2.0 + kw_hits * 1.25
430
- if score > best_score:
431
- best_score, best = score, cat
432
- return best
433
-
434
-
435
- # ---------------------------------------------------------------------------
436
- # Years-of-experience extraction (text mention)
437
- # ---------------------------------------------------------------------------
438
-
439
- def _extract_years(text: str) -> int:
440
- text_norm = _normalize(text)
441
- patterns = [
442
- r"(\d+)\+?\s*(?:years|year|yrs|yr)\s*(?:of)?\s*(?:experience|exp)",
443
- r"experience\s*(?:of)?\s*(\d+)\+?\s*(?:years|year|yrs|yr)",
444
- r"(\d+)\+?\s*(?:years|year|yrs|yr)\s+(?:in\s+)?(?:the\s+)?(?:industry|field|software|development|practice|profession)",
445
- ]
446
- years = [int(m.group(1)) for p in patterns for m in re.finditer(p, text_norm)]
447
- return max(years) if years else 0
448
-
449
-
450
- # ---------------------------------------------------------------------------
451
- # Years computed from parsed experience date ranges (more accurate)
452
- # ---------------------------------------------------------------------------
453
-
454
- _MONTH_SHORT: Dict[str, int] = {
455
- "jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
456
- "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12,
457
- }
458
-
459
-
460
- def _parse_date_token(token: str) -> Optional[_dt]:
461
- token = token.strip().lower()
462
- if re.match(r"present|current|now|ongoing|till\s*date|to\s*date", token):
463
- return _dt.now()
464
- m = re.match(r"([a-z]+)\s+(\d{4})", token)
465
- if m:
466
- month = _MONTH_SHORT.get(m.group(1)[:3], 1)
467
- try:
468
- return _dt(int(m.group(2)), month, 1)
469
- except ValueError:
470
- return None
471
- m = re.match(r"(\d{4})", token)
472
- if m:
473
- return _dt(int(m.group(1)), 6, 1) # mid-year when month unknown
474
- return None
475
-
476
-
477
- def compute_years_from_experience(entries: List[Dict[str, Any]]) -> int:
478
- """Sum date-range durations across all experience entries → total years."""
479
- total_months = 0
480
- for entry in entries:
481
- period = (entry.get("period") or "").strip()
482
- if not period:
483
- continue
484
- parts = re.split(r"\s*[-–—]\s*", period, maxsplit=1)
485
- if len(parts) != 2:
486
- continue
487
- start = _parse_date_token(parts[0])
488
- end = _parse_date_token(parts[1])
489
- if start and end and end > start:
490
- months = (end.year - start.year) * 12 + (end.month - start.month)
491
- total_months += max(0, months)
492
- return total_months // 12
493
-
494
-
495
- # ---------------------------------------------------------------------------
496
- # Seniority detection
497
- # ---------------------------------------------------------------------------
498
-
499
- def _detect_seniority(text: str, *, is_cv: bool = False) -> str:
500
- text_norm = _normalize(text)
501
- order = ["senior", "junior", "intern", "mid"] if is_cv else ["senior", "intern", "junior", "mid"]
502
- for level in order:
503
- if any(_has_alias(text_norm, term) for term in SENIORITY_PATTERNS[level]):
504
- return level
505
- years = _extract_years(text)
506
- if years >= 7:
507
- return "senior"
508
- if years >= 3:
509
- return "mid"
510
- return "junior" if is_cv else "mid"
511
-
512
-
513
- def refine_seniority_with_years(detected: str, years: int) -> str:
514
- """
515
- Override a weakly-detected seniority using computed years.
516
-
517
- 'mid' is the weakest keyword signal (matches many generic job titles).
518
- 'intern' can appear from a past role and should not dominate once the
519
- candidate has accumulated meaningful professional experience.
520
- """
521
- if detected == "mid":
522
- if years >= 7:
523
- return "senior"
524
- if years >= 3:
525
- return "mid"
526
- if 0 < years < 2:
527
- return "junior"
528
- return detected
529
-
530
- if detected == "intern" and years >= 2:
531
- # A past intern title should not override the current career level.
532
- if years >= 7:
533
- return "senior"
534
- if years >= 3:
535
- return "mid"
536
- return "junior"
537
-
538
- return detected
539
-
540
-
541
- # ---------------------------------------------------------------------------
542
- # CV title extractor (universal, scans more lines)
543
- # ---------------------------------------------------------------------------
544
-
545
- def _extract_cv_title(cv_text: str, clean_line_fn=None) -> str:
546
- """
547
- Extract the candidate's professional title from the top of the CV.
548
-
549
- Parameters
550
- ----------
551
- cv_text : str
552
- Raw markdown string.
553
- clean_line_fn : callable, optional
554
- A function(raw_line) -> str that strips markdown / HTML.
555
- Falls back to a simple regex strip if not provided.
556
- """
557
- def _simple_clean(raw: str) -> str:
558
- line = re.sub(r"^#+\s*", "", raw)
559
- line = re.sub(r"\*{1,3}|\_{1,2}|`", "", line)
560
- return re.sub(r"\s+", " ", line).strip()
561
-
562
- clean = clean_line_fn or _simple_clean
563
- clean_lines = [clean(x) for x in cv_text.splitlines() if x.strip()]
564
-
565
- # Pass 1: scan lines 1–30, find a short standalone title line.
566
- # Use word-boundary regex so "lead" does NOT match inside "leadership",
567
- # "developer" does NOT match inside "redevelopment", etc.
568
- _role_wb_re = re.compile(
569
- r"\b(" + "|".join(re.escape(w) for w in TITLE_ROLE_WORDS) + r")\b",
570
- re.IGNORECASE,
571
- )
572
- for line in clean_lines[1:30]:
573
- stripped = line.strip("| \t")
574
- if (
575
- len(stripped) <= 80
576
- and bool(_role_wb_re.search(stripped))
577
- and "|" not in stripped # skip leftover table cells
578
- and not re.search(r"[@http]", stripped)
579
- and not re.search(r"\d{4}\s*[-–]\s*(?:\d{4}|present)", stripped, re.IGNORECASE)
580
- ):
581
- return stripped.title()
582
-
583
-
584
- # Pass 2: phrase regex over filtered ~1000 chars.
585
- # Strip known section-header lines first so that patterns like
586
- # "CAREER SUMMARY" never bleed into the matched title phrase.
587
- _SECTION_HDR_RE = re.compile(
588
- r"^(career\s*summary|professional\s*summary|executive\s*summary|"
589
- r"personal\s*statement|career\s*objective|professional\s*profile|"
590
- r"personal\s*profile|about\s*me|work\s*experience|"
591
- r"professional\s*experience|technical\s*skills?|core\s*skills?|"
592
- r"key\s*skills?|educational?\s*background|academic\s*background|"
593
- r"employment\s*history|career\s*history|skills?\s*&?\s*tools?|"
594
- r"skills?|summary|profile|experience|education|projects?|awards?|"
595
- r"achievements?|contact|links?|certifications?)$",
596
- re.IGNORECASE,
597
- )
598
- non_header = [
599
- ln for ln in clean_lines[:40]
600
- if not _SECTION_HDR_RE.match(
601
- re.sub(r"[^a-zA-Z\s&]", " ", ln).strip()
602
- )
603
- ]
604
- # Join with '. ' so the phrase regex CANNOT span across line boundaries.
605
- # Without this, "... LinkedIn Portolio Full-stack developer ..." gets joined
606
- # into one string and the greedy {0,3}-word prefix matches "Portolio" as a
607
- # title prefix word on any CV whose contact/links line precedes the summary.
608
- head = ". ".join(non_header)[:1000]
609
- m = TITLE_PHRASE_RE.search(head)
610
- if m:
611
- return m.group(1).strip().title()
612
-
613
- return "Unknown"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
services/job_matcher.py CHANGED
@@ -7,7 +7,7 @@ import numpy as np
7
  from pydantic import BaseModel, Field
8
  from sentence_transformers import SentenceTransformer
9
 
10
- from services.cv_utils import (
11
  DEFAULT_SKILL_ALIASES,
12
  DEFAULT_CATEGORY_SKILLS,
13
  SENIORITY_ORDER,
 
7
  from pydantic import BaseModel, Field
8
  from sentence_transformers import SentenceTransformer
9
 
10
+ from services.taxonomy_client import (
11
  DEFAULT_SKILL_ALIASES,
12
  DEFAULT_CATEGORY_SKILLS,
13
  SENIORITY_ORDER,
services/taxonomy_client.py ADDED
@@ -0,0 +1,406 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ taxonomy_client.py
3
+ ==================
4
+ HTTP client that fetches the skill taxonomy from the scrapper service at
5
+ startup and re-exports the same names that cv_chunker.py and job_matcher.py
6
+ previously imported from the deleted cv_utils.py.
7
+
8
+ Usage (processor main.py lifespan):
9
+ from services.taxonomy_client import load_taxonomy
10
+ load_taxonomy() # blocks until scrapper is reachable; retries with backoff
11
+
12
+ All other modules just import from here as they did from cv_utils:
13
+ from services.taxonomy_client import (
14
+ DEFAULT_SKILL_ALIASES, DEFAULT_CATEGORY_SKILLS, SOFT_SKILL_KEYS,
15
+ SENIORITY_ORDER, _TECH_LOC_BLACKLIST,
16
+ _normalize, _has_alias, _extract_skills, _infer_category,
17
+ _detect_seniority, _extract_years, compute_years_from_experience,
18
+ refine_seniority_with_years, _extract_cv_title,
19
+ )
20
+ """
21
+ from __future__ import annotations
22
+
23
+ import logging
24
+ import os
25
+ import re
26
+ import time
27
+ from datetime import datetime as _dt
28
+ from typing import Any, Dict, List, Optional, Set, Tuple
29
+
30
+ import httpx
31
+
32
+ logger = logging.getLogger(__name__)
33
+
34
+ # ---------------------------------------------------------------------------
35
+ # Configuration
36
+ # ---------------------------------------------------------------------------
37
+
38
+ _SCRAPPER_URL = os.environ.get(
39
+ "TAXONOMY_SERVICE_URL", "https://eng-musa-job-scrapper.hf.space"
40
+ ).rstrip("/")
41
+ _TAXONOMY_ENDPOINT = f"{_SCRAPPER_URL}/taxonomy"
42
+
43
+ # ---------------------------------------------------------------------------
44
+ # Module-level taxonomy state (populated by load_taxonomy() at startup)
45
+ # ---------------------------------------------------------------------------
46
+
47
+ DEFAULT_SKILL_ALIASES: Dict[str, List[str]] = {}
48
+ DEFAULT_CATEGORY_SKILLS: Dict[str, Set[str]] = {}
49
+ SOFT_SKILL_KEYS: Set[str] = set()
50
+ SENIORITY_PATTERNS: Dict[str, List[str]] = {
51
+ "intern": ["intern", "internship", "trainee"],
52
+ "junior": ["junior", "entry level", "entry-level", "associate"],
53
+ "mid": ["mid", "developer", "engineer", "analyst", "specialist"],
54
+ "senior": ["senior", "lead", "manager", "head", "principal", "architect"],
55
+ }
56
+ SENIORITY_ORDER: Dict[str, int] = {"intern": 0, "junior": 1, "mid": 2, "senior": 3}
57
+ _TECH_LOC_BLACKLIST: Set[str] = set()
58
+
59
+ # These are structural/algorithmic — hardcoded as a safe fallback even when
60
+ # the API response doesn't include them or the client hasn't loaded yet.
61
+ TITLE_ROLE_WORDS: List[str] = [
62
+ "developer", "engineer", "programmer", "architect", "devops",
63
+ "officer", "designer", "assistant", "accountant", "manager", "analyst",
64
+ "specialist", "coordinator", "director", "executive", "lead", "scientist",
65
+ "strategist", "advisor", "trainer", "consultant", "researcher", "technician",
66
+ "supervisor", "associate", "administrator",
67
+ "nurse", "doctor", "physician", "pharmacist", "therapist", "clinician",
68
+ "midwife", "radiographer", "physiotherapist",
69
+ "auditor", "bookkeeper", "controller", "treasurer", "actuary",
70
+ "recruiter",
71
+ "salesperson", "representative", "agent", "broker",
72
+ "teacher", "instructor", "lecturer", "professor", "tutor",
73
+ "admin", "intern", "student", "cashier", "marketing",
74
+ "procurement", "buyer", "dispatcher", "operator", "superintendent",
75
+ ]
76
+
77
+ TITLE_PHRASE_RE = re.compile(
78
+ r"\b((?:(?:full[\s\-]?stack|front[\s\-]?end|back[\s\-]?end|senior|junior|lead|"
79
+ r"mid[\s\-]?level|entry[\s\-]?level|chief|head\s+of|staff|principal)?\s+)?(?:[\w\-]+\s){0,3}"
80
+ r"(?:developer|engineer|designer|analyst|manager|specialist|consultant|architect|"
81
+ r"programmer|researcher|officer|director|executive|scientist|trainer|accountant|"
82
+ r"nurse|doctor|pharmacist|therapist|teacher|instructor|lecturer|recruiter|"
83
+ r"coordinator|supervisor|administrator|technician|advisor|auditor|bookkeeper|"
84
+ r"physiotherapist|midwife|clinician|representative|salesperson|buyer|dispatcher|"
85
+ r"actuary|superintendent|operator))\b",
86
+ re.IGNORECASE,
87
+ )
88
+
89
+ # ---------------------------------------------------------------------------
90
+ # Taxonomy loader (call once at processor startup)
91
+ # ---------------------------------------------------------------------------
92
+
93
+ def load_taxonomy(
94
+ *,
95
+ max_retries: int = 15,
96
+ initial_delay: float = 3.0,
97
+ max_delay: float = 60.0,
98
+ ) -> None:
99
+ """
100
+ Fetch the taxonomy from the scrapper and populate module-level globals.
101
+
102
+ Retries with exponential back-off until the scrapper responds or
103
+ max_retries is exhausted (in which case RuntimeError is raised).
104
+
105
+ Logs progress at INFO level so it's visible in the processor console.
106
+ """
107
+ global DEFAULT_SKILL_ALIASES, DEFAULT_CATEGORY_SKILLS, SOFT_SKILL_KEYS
108
+ global SENIORITY_PATTERNS, SENIORITY_ORDER, _TECH_LOC_BLACKLIST
109
+
110
+ delay = initial_delay
111
+ last_error: Optional[Exception] = None
112
+
113
+ for attempt in range(1, max_retries + 1):
114
+ try:
115
+ logger.info(
116
+ "[Taxonomy] Attempt %d/%d — fetching %s",
117
+ attempt, max_retries, _TAXONOMY_ENDPOINT,
118
+ )
119
+ with httpx.Client(timeout=30.0) as client:
120
+ resp = client.get(_TAXONOMY_ENDPOINT)
121
+ resp.raise_for_status()
122
+ data: Dict[str, Any] = resp.json()
123
+
124
+ # Populate globals
125
+ DEFAULT_SKILL_ALIASES = data.get("aliases", {})
126
+ DEFAULT_CATEGORY_SKILLS = {
127
+ cat: set(skills)
128
+ for cat, skills in data.get("category_skills", {}).items()
129
+ }
130
+ SOFT_SKILL_KEYS = set(data.get("soft_skill_keys", []))
131
+ SENIORITY_PATTERNS = data.get("seniority_patterns", SENIORITY_PATTERNS)
132
+ SENIORITY_ORDER = data.get("seniority_order", SENIORITY_ORDER)
133
+ _TECH_LOC_BLACKLIST = set(data.get("tech_loc_blacklist", []))
134
+
135
+ # Update TITLE_ROLE_WORDS from API if provided
136
+ global TITLE_ROLE_WORDS
137
+ api_roles = data.get("title_role_words")
138
+ if api_roles:
139
+ TITLE_ROLE_WORDS = api_roles
140
+
141
+ logger.info(
142
+ "[Taxonomy] ✅ Loaded: %d skills, %d categories",
143
+ len(DEFAULT_SKILL_ALIASES),
144
+ len(DEFAULT_CATEGORY_SKILLS),
145
+ )
146
+ return # success
147
+
148
+ except Exception as exc:
149
+ last_error = exc
150
+ logger.warning(
151
+ "[Taxonomy] ⚠️ Attempt %d failed: %s", attempt, exc
152
+ )
153
+ if attempt < max_retries:
154
+ logger.info(
155
+ "[Taxonomy] Retrying in %.1f seconds…", delay
156
+ )
157
+ time.sleep(delay)
158
+ delay = min(delay * 1.5, max_delay)
159
+
160
+ raise RuntimeError(
161
+ f"[Taxonomy] ❌ Failed to load taxonomy after {max_retries} attempts. "
162
+ f"Last error: {last_error}"
163
+ )
164
+
165
+
166
+ # ---------------------------------------------------------------------------
167
+ # Pure normalisation helpers (logic only — no taxonomy data needed)
168
+ # ---------------------------------------------------------------------------
169
+
170
+ def _normalize(text: str) -> str:
171
+ text = (text or "").lower()
172
+ text = text.replace("/", " ")
173
+ text = text.replace("_", " ")
174
+ text = re.sub(r"[^a-z0-9+.#\s-]", " ", text)
175
+ return re.sub(r"\s+", " ", text).strip()
176
+
177
+
178
+ def _has_alias(text_norm: str, alias: str) -> bool:
179
+ alias_norm = _normalize(alias)
180
+ if not alias_norm:
181
+ return False
182
+ return re.search(rf"(?<![a-z0-9]){re.escape(alias_norm)}(?![a-z0-9])", text_norm) is not None
183
+
184
+
185
+ # ---------------------------------------------------------------------------
186
+ # Skill extraction
187
+ # ---------------------------------------------------------------------------
188
+
189
+ def _extract_skills(
190
+ text: str,
191
+ aliases: Optional[Dict[str, List[str]]] = None,
192
+ ) -> Set[str]:
193
+ if aliases is None:
194
+ aliases = DEFAULT_SKILL_ALIASES
195
+ text_norm = _normalize(text)
196
+ return {
197
+ canon
198
+ for canon, ali_list in aliases.items()
199
+ if any(_has_alias(text_norm, a) for a in ali_list)
200
+ }
201
+
202
+
203
+ # ---------------------------------------------------------------------------
204
+ # Category inference
205
+ # ---------------------------------------------------------------------------
206
+
207
+ _CATEGORY_TITLE_KEYWORDS: Dict[str, List[str]] = {
208
+ "Admin & Office": ["admin", "office", "stock", "inventory", "warehouse", "cashier", "receptionist", "clerk"],
209
+ "Software Engineering": ["developer", "engineer", "programmer", "frontend", "backend", "full stack", "mobile", "fullstack", "software", "devops"],
210
+ "Infrastructure & Cloud": ["infrastructure", "cloud", "sre", "network", "system administrator", "sysadmin", "it administrator", "cloud engineer"],
211
+ "Cybersecurity": ["security", "cybersecurity", "cyber", "infosec", "soc analyst", "penetration tester", "ethical hacker"],
212
+ "Design & Creative": ["designer", "ui", "ux", "creative", "graphic", "visual", "illustrator", "photographer"],
213
+ "Marketing & Growth": ["marketing", "seo", "content", "social media", "growth", "digital", "brand"],
214
+ "Data & Analytics": ["data analyst", "analytics", "machine learning", "ai", "scientist", "bi analyst", "business intelligence"],
215
+ "Data Engineering": ["data engineer", "etl", "data pipeline", "data architect", "analytics engineer"],
216
+ "Finance & Accounting": ["accountant", "finance", "accounting", "auditor", "bookkeeper", "treasurer", "payroll", "tax", "financial"],
217
+ "Human Resources": ["hr", "human resource", "recruitment", "recruiter", "talent", "people", "hris"],
218
+ "Sales & Business Dev": ["sales", "business development", "account manager", "sales executive", "representative"],
219
+ "Healthcare": ["nurse", "doctor", "clinical", "medical", "pharmacy", "patient", "health", "therapist", "midwife"],
220
+ "Legal": ["legal", "lawyer", "attorney", "advocate", "counsel", "paralegal", "solicitor"],
221
+ "Education & Training": ["teacher", "lecturer", "tutor", "instructor", "educator", "academic", "trainer", "professor"],
222
+ "Operations & Logistics": ["logistics", "supply chain", "procurement", "operations", "fleet", "warehouse manager", "quality"],
223
+ "Customer Support": ["customer support", "help desk", "call centre", "contact centre", "customer care"],
224
+ "Construction & Engineering": ["civil", "structural", "mechanical", "electrical", "quantity surveyor", "site engineer", "construction"],
225
+ "Hospitality & Property": ["hotel", "hospitality", "property", "real estate", "housekeeping", "f&b", "restaurant"],
226
+ }
227
+
228
+
229
+ def _infer_category(
230
+ text: str,
231
+ skills: Set[str],
232
+ category_skills: Optional[Dict[str, Set[str]]] = None,
233
+ ) -> str:
234
+ if category_skills is None:
235
+ category_skills = DEFAULT_CATEGORY_SKILLS
236
+ text_norm = _normalize(text)
237
+ best, best_score = "Other", 0.0
238
+ for cat, cat_skills in category_skills.items():
239
+ skill_overlap = len(skills & cat_skills)
240
+ kw_hits = sum(
241
+ 1 for kw in _CATEGORY_TITLE_KEYWORDS.get(cat, []) if kw in text_norm
242
+ )
243
+ score = skill_overlap * 2.0 + kw_hits * 1.25
244
+ if score > best_score:
245
+ best_score, best = score, cat
246
+ return best
247
+
248
+
249
+ # ---------------------------------------------------------------------------
250
+ # Years-of-experience extraction
251
+ # ---------------------------------------------------------------------------
252
+
253
+ def _extract_years(text: str) -> int:
254
+ text_norm = _normalize(text)
255
+ patterns = [
256
+ r"(\d+)\+?\s*(?:years|year|yrs|yr)\s*(?:of)?\s*(?:experience|exp)",
257
+ r"experience\s*(?:of)?\s*(\d+)\+?\s*(?:years|year|yrs|yr)",
258
+ r"(\d+)\+?\s*(?:years|year|yrs|yr)\s+(?:in\s+)?(?:the\s+)?(?:industry|field|software|development|practice|profession)",
259
+ ]
260
+ years = [int(m.group(1)) for p in patterns for m in re.finditer(p, text_norm)]
261
+ return max(years) if years else 0
262
+
263
+
264
+ # ---------------------------------------------------------------------------
265
+ # Years from parsed experience date ranges
266
+ # ---------------------------------------------------------------------------
267
+
268
+ _MONTH_SHORT: Dict[str, int] = {
269
+ "jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
270
+ "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12,
271
+ }
272
+
273
+
274
+ def _parse_date_token(token: str) -> Optional[_dt]:
275
+ token = token.strip().lower()
276
+ if re.match(r"present|current|now|ongoing|till\s*date|to\s*date", token):
277
+ return _dt.now()
278
+ m = re.match(r"([a-z]+)\s+(\d{4})", token)
279
+ if m:
280
+ month = _MONTH_SHORT.get(m.group(1)[:3], 1)
281
+ try:
282
+ return _dt(int(m.group(2)), month, 1)
283
+ except ValueError:
284
+ return None
285
+ m = re.match(r"(\d{4})", token)
286
+ if m:
287
+ return _dt(int(m.group(1)), 6, 1)
288
+ return None
289
+
290
+
291
+ def compute_years_from_experience(entries: List[Dict[str, Any]]) -> int:
292
+ """Sum date-range durations across all experience entries → total years."""
293
+ total_months = 0
294
+ for entry in entries:
295
+ period = (entry.get("period") or "").strip()
296
+ if not period:
297
+ continue
298
+ parts = re.split(r"\s*[-–—]\s*", period, maxsplit=1)
299
+ if len(parts) != 2:
300
+ continue
301
+ start = _parse_date_token(parts[0])
302
+ end = _parse_date_token(parts[1])
303
+ if start and end and end > start:
304
+ months = (end.year - start.year) * 12 + (end.month - start.month)
305
+ total_months += max(0, months)
306
+ return total_months // 12
307
+
308
+
309
+ # ---------------------------------------------------------------------------
310
+ # Seniority detection
311
+ # ---------------------------------------------------------------------------
312
+
313
+ def _detect_seniority(text: str, *, is_cv: bool = False) -> str:
314
+ """Detect seniority: title zone first, then body (guards against
315
+ false positives from advice/boilerplate text in job descriptions)."""
316
+ title_zone = text[:300]
317
+ title_norm = _normalize(title_zone)
318
+ text_norm = _normalize(text)
319
+
320
+ order = ["senior", "junior", "intern", "mid"] if is_cv else ["senior", "intern", "junior", "mid"]
321
+
322
+ for level in order:
323
+ if any(_has_alias(title_norm, term) for term in SENIORITY_PATTERNS[level]):
324
+ return level
325
+
326
+ for level in order:
327
+ if level == "senior" and not is_cv:
328
+ continue # already checked title zone; skip body re-check to avoid boilerplate inflation
329
+ if any(_has_alias(text_norm, term) for term in SENIORITY_PATTERNS[level]):
330
+ return level
331
+
332
+ years = _extract_years(text)
333
+ if years >= 7:
334
+ return "senior"
335
+ if years >= 3:
336
+ return "mid"
337
+ return "junior" if is_cv else "mid"
338
+
339
+
340
+ def refine_seniority_with_years(detected: str, years: int) -> str:
341
+ if detected == "mid":
342
+ if years >= 7:
343
+ return "senior"
344
+ if years >= 3:
345
+ return "mid"
346
+ if 0 < years < 2:
347
+ return "junior"
348
+ return detected
349
+ if detected == "intern" and years >= 2:
350
+ if years >= 7:
351
+ return "senior"
352
+ if years >= 3:
353
+ return "mid"
354
+ return "junior"
355
+ return detected
356
+
357
+
358
+ # ---------------------------------------------------------------------------
359
+ # CV title extractor
360
+ # ---------------------------------------------------------------------------
361
+
362
+ def _extract_cv_title(cv_text: str, clean_line_fn=None) -> str:
363
+ def _simple_clean(raw: str) -> str:
364
+ line = re.sub(r"^#+\s*", "", raw)
365
+ line = re.sub(r"\*{1,3}|\_{1,2}|`", "", line)
366
+ return re.sub(r"\s+", " ", line).strip()
367
+
368
+ clean = clean_line_fn or _simple_clean
369
+ clean_lines = [clean(x) for x in cv_text.splitlines() if x.strip()]
370
+
371
+ _role_wb_re = re.compile(
372
+ r"\b(" + "|".join(re.escape(w) for w in TITLE_ROLE_WORDS) + r")\b",
373
+ re.IGNORECASE,
374
+ )
375
+ for line in clean_lines[1:30]:
376
+ stripped = line.strip("| \t")
377
+ if (
378
+ len(stripped) <= 80
379
+ and bool(_role_wb_re.search(stripped))
380
+ and "|" not in stripped
381
+ and not re.search(r"[@http]", stripped)
382
+ and not re.search(r"\d{4}\s*[-–]\s*(?:\d{4}|present)", stripped, re.IGNORECASE)
383
+ ):
384
+ return stripped.title()
385
+
386
+ _SECTION_HDR_RE = re.compile(
387
+ r"^(career\s*summary|professional\s*summary|executive\s*summary|"
388
+ r"personal\s*statement|career\s*objective|professional\s*profile|"
389
+ r"personal\s*profile|about\s*me|work\s*experience|"
390
+ r"professional\s*experience|technical\s*skills?|core\s*skills?|"
391
+ r"key\s*skills?|educational?\s*background|academic\s*background|"
392
+ r"employment\s*history|career\s*history|skills?\s*&?\s*tools?|"
393
+ r"skills?|summary|profile|experience|education|projects?|awards?|"
394
+ r"achievements?|contact|links?|certifications?)$",
395
+ re.IGNORECASE,
396
+ )
397
+ non_header = [
398
+ ln for ln in clean_lines[:40]
399
+ if not _SECTION_HDR_RE.match(re.sub(r"[^a-zA-Z\s&]", " ", ln).strip())
400
+ ]
401
+ head = ". ".join(non_header)[:1000]
402
+ m = TITLE_PHRASE_RE.search(head)
403
+ if m:
404
+ return m.group(1).strip().title()
405
+
406
+ return "Unknown"