Eng-Musa commited on
Commit
dde994a
·
1 Parent(s): dfafe5a

remove the fallback

Browse files
Files changed (1) hide show
  1. services/taxonomy_client.py +45 -40
services/taxonomy_client.py CHANGED
@@ -28,6 +28,7 @@ from datetime import datetime as _dt
28
  from typing import Any, Dict, List, Optional, Set, Tuple
29
 
30
  import httpx
 
31
 
32
  logger = logging.getLogger(__name__)
33
 
@@ -47,32 +48,10 @@ _TAXONOMY_ENDPOINT = f"{_SCRAPPER_URL}/taxonomy"
47
  DEFAULT_SKILL_ALIASES: Dict[str, List[str]] = {}
48
  DEFAULT_CATEGORY_SKILLS: Dict[str, Set[str]] = {}
49
  SOFT_SKILL_KEYS: Set[str] = set()
50
- SENIORITY_PATTERNS: Dict[str, List[str]] = {
51
- "intern": ["intern", "internship", "trainee"],
52
- "junior": ["junior", "entry level", "entry-level", "associate"],
53
- "mid": ["mid", "developer", "engineer", "analyst", "specialist"],
54
- "senior": ["senior", "lead", "manager", "head", "principal", "architect"],
55
- }
56
- SENIORITY_ORDER: Dict[str, int] = {"intern": 0, "junior": 1, "mid": 2, "senior": 3}
57
  _TECH_LOC_BLACKLIST: Set[str] = set()
58
-
59
- # These are structural/algorithmic — hardcoded as a safe fallback even when
60
- # the API response doesn't include them or the client hasn't loaded yet.
61
- TITLE_ROLE_WORDS: List[str] = [
62
- "developer", "engineer", "programmer", "architect", "devops",
63
- "officer", "designer", "assistant", "accountant", "manager", "analyst",
64
- "specialist", "coordinator", "director", "executive", "lead", "scientist",
65
- "strategist", "advisor", "trainer", "consultant", "researcher", "technician",
66
- "supervisor", "associate", "administrator",
67
- "nurse", "doctor", "physician", "pharmacist", "therapist", "clinician",
68
- "midwife", "radiographer", "physiotherapist",
69
- "auditor", "bookkeeper", "controller", "treasurer", "actuary",
70
- "recruiter",
71
- "salesperson", "representative", "agent", "broker",
72
- "teacher", "instructor", "lecturer", "professor", "tutor",
73
- "admin", "intern", "student", "cashier", "marketing",
74
- "procurement", "buyer", "dispatcher", "operator", "superintendent",
75
- ]
76
 
77
  TITLE_PHRASE_RE = re.compile(
78
  r"\b((?:(?:full[\s\-]?stack|front[\s\-]?end|back[\s\-]?end|senior|junior|lead|"
@@ -121,22 +100,47 @@ def load_taxonomy(
121
  resp.raise_for_status()
122
  data: Dict[str, Any] = resp.json()
123
 
124
- # Populate globals
125
- DEFAULT_SKILL_ALIASES = data.get("aliases", {})
126
- DEFAULT_CATEGORY_SKILLS = {
127
- cat: set(skills)
128
- for cat, skills in data.get("category_skills", {}).items()
129
- }
130
- SOFT_SKILL_KEYS = set(data.get("soft_skill_keys", []))
131
- SENIORITY_PATTERNS = data.get("seniority_patterns", SENIORITY_PATTERNS)
132
- SENIORITY_ORDER = data.get("seniority_order", SENIORITY_ORDER)
133
- _TECH_LOC_BLACKLIST = set(data.get("tech_loc_blacklist", []))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
134
 
135
  # Update TITLE_ROLE_WORDS from API if provided
136
- global TITLE_ROLE_WORDS
137
  api_roles = data.get("title_role_words")
138
  if api_roles:
139
- TITLE_ROLE_WORDS = api_roles
 
140
 
141
  logger.info(
142
  "[Taxonomy] ✅ Loaded: %d skills, %d categories",
@@ -157,10 +161,11 @@ def load_taxonomy(
157
  time.sleep(delay)
158
  delay = min(delay * 1.5, max_delay)
159
 
160
- raise RuntimeError(
161
- f"[Taxonomy] ❌ Failed to load taxonomy after {max_retries} attempts. "
162
- f"Last error: {last_error}"
163
  )
 
164
 
165
 
166
  # ---------------------------------------------------------------------------
 
28
  from typing import Any, Dict, List, Optional, Set, Tuple
29
 
30
  import httpx
31
+ import sys
32
 
33
  logger = logging.getLogger(__name__)
34
 
 
48
  DEFAULT_SKILL_ALIASES: Dict[str, List[str]] = {}
49
  DEFAULT_CATEGORY_SKILLS: Dict[str, Set[str]] = {}
50
  SOFT_SKILL_KEYS: Set[str] = set()
51
+ SENIORITY_PATTERNS: Dict[str, List[str]] = {}
52
+ SENIORITY_ORDER: Dict[str, int] = {}
 
 
 
 
 
53
  _TECH_LOC_BLACKLIST: Set[str] = set()
54
+ TITLE_ROLE_WORDS: List[str] = []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
 
56
  TITLE_PHRASE_RE = re.compile(
57
  r"\b((?:(?:full[\s\-]?stack|front[\s\-]?end|back[\s\-]?end|senior|junior|lead|"
 
100
  resp.raise_for_status()
101
  data: Dict[str, Any] = resp.json()
102
 
103
+ if not data.get("aliases"):
104
+ raise ValueError("Taxonomy endpoint returned empty aliases")
105
+
106
+ # Populate globals in-place so modules that already imported them see the updates
107
+ aliases = data.get("aliases")
108
+ if aliases:
109
+ DEFAULT_SKILL_ALIASES.clear()
110
+ DEFAULT_SKILL_ALIASES.update(aliases)
111
+
112
+ cat_skills = data.get("category_skills")
113
+ if cat_skills:
114
+ DEFAULT_CATEGORY_SKILLS.clear()
115
+ DEFAULT_CATEGORY_SKILLS.update({
116
+ cat: set(skills) for cat, skills in cat_skills.items()
117
+ })
118
+
119
+ soft_keys = data.get("soft_skill_keys")
120
+ if soft_keys:
121
+ SOFT_SKILL_KEYS.clear()
122
+ SOFT_SKILL_KEYS.update(soft_keys)
123
+
124
+ sen_patterns = data.get("seniority_patterns")
125
+ if sen_patterns:
126
+ SENIORITY_PATTERNS.clear()
127
+ SENIORITY_PATTERNS.update(sen_patterns)
128
+
129
+ sen_order = data.get("seniority_order")
130
+ if sen_order:
131
+ SENIORITY_ORDER.clear()
132
+ SENIORITY_ORDER.update(sen_order)
133
+
134
+ tech_loc = data.get("tech_loc_blacklist")
135
+ if tech_loc:
136
+ _TECH_LOC_BLACKLIST.clear()
137
+ _TECH_LOC_BLACKLIST.update(tech_loc)
138
 
139
  # Update TITLE_ROLE_WORDS from API if provided
 
140
  api_roles = data.get("title_role_words")
141
  if api_roles:
142
+ TITLE_ROLE_WORDS.clear()
143
+ TITLE_ROLE_WORDS.extend(api_roles)
144
 
145
  logger.info(
146
  "[Taxonomy] ✅ Loaded: %d skills, %d categories",
 
161
  time.sleep(delay)
162
  delay = min(delay * 1.5, max_delay)
163
 
164
+ logger.error(
165
+ "[Taxonomy] ❌ CRITICAL: Failed to load taxonomy after %d attempts. "
166
+ "Last error: %s. Stopping the processor.", max_retries, last_error
167
  )
168
+ sys.exit(1)
169
 
170
 
171
  # ---------------------------------------------------------------------------