Eng-Musa commited on
Commit
65a2e47
·
unverified ·
2 Parent(s): 190fe1d56fdbbe

Merge pull request #1 from Eng-Musa/musa

Browse files
main.py CHANGED
@@ -1,7 +1,9 @@
1
  from __future__ import annotations
2
 
3
  import logging
 
4
  import tempfile
 
5
  import uuid
6
  import httpx
7
  from datetime import datetime, timezone
@@ -10,7 +12,13 @@ from typing import Any, List, Optional
10
 
11
  from contextlib import asynccontextmanager
12
 
13
- logger = logging.getLogger(__name__)
 
 
 
 
 
 
14
 
15
  from fastapi import FastAPI, File, Form, UploadFile
16
  from fastapi.exceptions import RequestValidationError
@@ -21,7 +29,7 @@ from starlette.exceptions import HTTPException as StarletteHTTPException
21
  from services import job_store
22
  from services.cv_chunker import chunk_cv
23
  from services.cv_converter import CVConverter
24
- from services.job_matcher import JobInput, JobMatcher
25
  from services.workers import CvWorker, QueueManager, CvTask, CvTaskV2
26
  from services.new.cv_processor_v2 import CvProcessorV2
27
  from services.new.cv_markdown_processor import CvMarkdownProcessor
@@ -380,14 +388,13 @@ class BatchMatchCVRequest(BaseModel):
380
  max_length=100,
381
  description="Array of job postings (1–100).",
382
  )
383
- cv_markdown: str
384
  cv_title: str
385
  cv_years: int
386
  cv_seniority: str
387
- cv_skills_canonical: List[str]
388
- cv_skills_technical: List[str]
389
- cv_skills_soft: List[str]
390
  cv_category: str
 
 
 
391
 
392
 
393
  @app.post("/batch-match-cv", response_model=APIResponse)
@@ -395,19 +402,28 @@ def batch_match_cv(payload: BatchMatchCVRequest):
395
  """
396
  Match one CV against an array of job postings in a single request.
397
  """
 
 
 
 
 
 
 
 
398
  try:
399
  results = matcher.predict_batch(
400
  jobs=[j.model_dump() for j in payload.jobs],
401
- cv_markdown=payload.cv_markdown,
402
  cv_title=payload.cv_title,
403
  cv_years=payload.cv_years,
404
  cv_seniority=payload.cv_seniority,
405
- cv_skills_canonical=payload.cv_skills_canonical,
406
- cv_skills_technical=payload.cv_skills_technical,
407
- cv_skills_soft=payload.cv_skills_soft,
408
  cv_category=payload.cv_category,
 
 
 
409
  )
410
 
 
 
411
  return APIResponse(
412
  message=f"Batch match completed: {len(results)} result(s)",
413
  statusCode=200,
@@ -417,6 +433,8 @@ def batch_match_cv(payload: BatchMatchCVRequest):
417
  },
418
  )
419
  except Exception as exc:
 
 
420
  return APIResponse(
421
  message=f"Batch match error: {exc}",
422
  statusCode=500,
@@ -505,7 +523,13 @@ async def process_cv_v2(file: UploadFile = File(...)):
505
  t_start = datetime.now(timezone.utc)
506
 
507
  try:
508
- payload = cv_processor_v2.process(file_bytes, file.filename, embedder=matcher._embed)
 
 
 
 
 
 
509
  except ValueError as exc:
510
  # Covers scanned PDF rejection and conversion failures
511
  return APIResponse(message=str(exc), statusCode=422)
@@ -622,6 +646,8 @@ def process_cv_markdown_v2(payload: ProcessCvMarkdownRequest):
622
  result = cv_markdown_processor.process(
623
  markdown=payload.markdown,
624
  embedder=matcher._embed,
 
 
625
  )
626
  except Exception as exc:
627
  return APIResponse(message=f"Processing error: {exc}", statusCode=500)
 
1
  from __future__ import annotations
2
 
3
  import logging
4
+ import sys
5
  import tempfile
6
+ import time
7
  import uuid
8
  import httpx
9
  from datetime import datetime, timezone
 
12
 
13
  from contextlib import asynccontextmanager
14
 
15
+ logging.basicConfig(
16
+ level=logging.INFO,
17
+ format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
18
+ handlers=[logging.StreamHandler(sys.stdout)],
19
+ force=True,
20
+ )
21
+ logger = logging.getLogger("processor.main")
22
 
23
  from fastapi import FastAPI, File, Form, UploadFile
24
  from fastapi.exceptions import RequestValidationError
 
29
  from services import job_store
30
  from services.cv_chunker import chunk_cv
31
  from services.cv_converter import CVConverter
32
+ from services.job_matcher import JobInput, JobMatcher, RawSectionInput
33
  from services.workers import CvWorker, QueueManager, CvTask, CvTaskV2
34
  from services.new.cv_processor_v2 import CvProcessorV2
35
  from services.new.cv_markdown_processor import CvMarkdownProcessor
 
388
  max_length=100,
389
  description="Array of job postings (1–100).",
390
  )
 
391
  cv_title: str
392
  cv_years: int
393
  cv_seniority: str
 
 
 
394
  cv_category: str
395
+ cv_skills_raw: List[str] = Field(default_factory=list)
396
+ cv_raw_sections: List[RawSectionInput] = Field(default_factory=list)
397
+ cv_section_embeddings: dict[str, Any] = Field(default_factory=dict)
398
 
399
 
400
  @app.post("/batch-match-cv", response_model=APIResponse)
 
402
  """
403
  Match one CV against an array of job postings in a single request.
404
  """
405
+ t_start = time.perf_counter()
406
+ emb = payload.cv_section_embeddings or {}
407
+ has_profile = bool(emb.get("profile_vec"))
408
+ num_chunks = len(emb.get("raw_sections_chunks") or [])
409
+ num_skills = len(emb.get("skill_embeddings") or {})
410
+ has_title = bool(emb.get("title_vec"))
411
+ num_roles = len(emb.get("role_title_vecs") or [])
412
+
413
  try:
414
  results = matcher.predict_batch(
415
  jobs=[j.model_dump() for j in payload.jobs],
 
416
  cv_title=payload.cv_title,
417
  cv_years=payload.cv_years,
418
  cv_seniority=payload.cv_seniority,
 
 
 
419
  cv_category=payload.cv_category,
420
+ cv_skills_raw=payload.cv_skills_raw,
421
+ cv_raw_sections=[s.model_dump() for s in payload.cv_raw_sections],
422
+ cv_section_embeddings=payload.cv_section_embeddings,
423
  )
424
 
425
+ duration_ms = (time.perf_counter() - t_start) * 1000
426
+
427
  return APIResponse(
428
  message=f"Batch match completed: {len(results)} result(s)",
429
  statusCode=200,
 
433
  },
434
  )
435
  except Exception as exc:
436
+ duration_ms = (time.perf_counter() - t_start) * 1000
437
+ logger.exception("❌ [BATCH-MATCH-CV] Error after %.2fms: %s", duration_ms, exc)
438
  return APIResponse(
439
  message=f"Batch match error: {exc}",
440
  statusCode=500,
 
523
  t_start = datetime.now(timezone.utc)
524
 
525
  try:
526
+ payload = cv_processor_v2.process(
527
+ file_bytes,
528
+ file.filename,
529
+ embedder=matcher._embed,
530
+ batch_embedder=matcher._embed_batch_texts,
531
+ title_batch_embedder=matcher._embed_title_batch,
532
+ )
533
  except ValueError as exc:
534
  # Covers scanned PDF rejection and conversion failures
535
  return APIResponse(message=str(exc), statusCode=422)
 
646
  result = cv_markdown_processor.process(
647
  markdown=payload.markdown,
648
  embedder=matcher._embed,
649
+ batch_embedder=matcher._embed_batch_texts,
650
+ title_batch_embedder=matcher._embed_title_batch,
651
  )
652
  except Exception as exc:
653
  return APIResponse(message=f"Processing error: {exc}", statusCode=500)
services/job_matcher.py CHANGED
The diff for this file is too large to render. See raw diff
 
services/new/cv_markdown_processor.py CHANGED
@@ -6,18 +6,11 @@ Processes raw markdown directly via LLM (Google Gemini) with NO fallback.
6
  from __future__ import annotations
7
 
8
  import logging
9
- from datetime import datetime, timezone
10
  from typing import Any, Callable, Dict, Optional
11
 
12
  from config import GEMINI_MODEL
13
- from services.cv_chunker import chunk_cv
14
  from services.new.google.gemini_extractor import GeminiCvExtractor
15
- from services.processor_utils import (
16
- DEFAULT_CATEGORY_SKILLS,
17
- DEFAULT_SKILL_ALIASES,
18
- _extract_skills,
19
- _infer_category,
20
- )
21
 
22
  logger = logging.getLogger(__name__)
23
 
@@ -32,96 +25,27 @@ class CvMarkdownProcessor:
32
  self,
33
  markdown: str,
34
  embedder: Optional[Callable] = None,
 
 
35
  ) -> Dict[str, Any]:
36
  """
37
  Extract structured CV fields directly from markdown using Gemini LLM only.
38
- Generates section embeddings using chunk_cv and returns the exact V2 flat payload.
39
  No fallback is performed if the LLM fails.
40
  """
41
  if not markdown or not markdown.strip():
42
  raise ValueError("Markdown content is empty.")
43
 
44
- # Step 1: LLM extraction (direct call, will raise if error occurs)
45
  extracted = self.gemini_extractor.extract(markdown)
46
-
47
- # Step 2: Infer category (consistent with V2)
48
- skills_found = set(
49
- s.lower() for s in extracted.get("skills", {}).get("items", [])
50
- )
51
- if not skills_found:
52
- skills_found = _extract_skills(markdown, DEFAULT_SKILL_ALIASES)
53
- inferred_category = _infer_category(
54
- text=markdown,
55
- skills=skills_found,
56
- category_skills=DEFAULT_CATEGORY_SKILLS,
57
  )
58
-
59
- # Step 3: Canonical skills mapping
60
- llm_skills = extracted.get("skills", {})
61
- canonical_skills = sorted(_extract_skills(markdown, DEFAULT_SKILL_ALIASES))
62
- skills_payload = {
63
- "items": llm_skills.get("items", []),
64
- "canonical": canonical_skills,
65
- "technical": llm_skills.get("technical", []),
66
- "soft": llm_skills.get("soft", []),
67
- }
68
-
69
- # Step 4: Generate section embeddings via chunk_cv
70
- t_chunk_start = datetime.now(timezone.utc)
71
- section_embeddings = None
72
- chunking_ms = 0
73
- if embedder is not None:
74
- chunk_result = chunk_cv(markdown, embedder=embedder)
75
- section_embeddings = chunk_result.get("section_embeddings")
76
- t_chunk_end = datetime.now(timezone.utc)
77
- chunking_ms = int((t_chunk_end - t_chunk_start).total_seconds() * 1000)
78
-
79
- # Step 5: Normalize usage metadata
80
- raw_usage = extracted.get("usage_metadata", {})
81
- usage_metadata = _normalise_usage(raw_usage)
82
-
83
- return {
84
- "method_used": "google",
85
- "model_used": GEMINI_MODEL,
86
- "markdown": markdown,
87
- "cv_title": extracted.get("cv_title", ""),
88
- "seniority": extracted.get("seniority", "entry"),
89
- "years_experience": extracted.get("years_experience", 0),
90
- "category": inferred_category,
91
- "completeness_score": extracted.get("completeness_score", 0),
92
- "summary": extracted.get("summary", ""),
93
- "contact": extracted.get("contact", {}),
94
- "links": extracted.get("links", []),
95
- "skills": skills_payload,
96
- "experience": extracted.get("experience", []),
97
- "education": extracted.get("education", []),
98
- "projects": extracted.get("projects", []),
99
- "awards": extracted.get("awards", []),
100
- "raw_sections": extracted.get("raw_sections", []),
101
- "usage_metadata": usage_metadata,
102
- "section_embeddings": section_embeddings,
103
- "file_type": "markdown",
104
- "is_scanned": False,
105
- "page_count": 1,
106
- "warnings": [],
107
- "chunking_ms": chunking_ms,
108
- }
109
-
110
-
111
- def _normalise_usage(raw: Any) -> Optional[Dict[str, Any]]:
112
- """Return usage metadata dict preserving all raw usage keys plus normalized token counts."""
113
- if not raw or not isinstance(raw, dict):
114
- return None
115
- res = dict(raw)
116
- input_tokens = raw.get("inputTokenCount") or raw.get("input_tokens") or raw.get("promptTokenCount")
117
- output_tokens = raw.get("outputTokenCount") or raw.get("output_tokens") or raw.get("candidatesTokenCount")
118
- total_tokens = raw.get("totalTokenCount") or raw.get("total_tokens")
119
- if total_tokens is None and input_tokens is not None and output_tokens is not None:
120
- total_tokens = input_tokens + output_tokens
121
- if input_tokens is not None:
122
- res["input_tokens"] = input_tokens
123
- if output_tokens is not None:
124
- res["output_tokens"] = output_tokens
125
- if total_tokens is not None:
126
- res["total_tokens"] = total_tokens
127
- return res
 
6
  from __future__ import annotations
7
 
8
  import logging
 
9
  from typing import Any, Callable, Dict, Optional
10
 
11
  from config import GEMINI_MODEL
12
+ from services.new.cv_postprocessor import finalize_v2_cv_payload
13
  from services.new.google.gemini_extractor import GeminiCvExtractor
 
 
 
 
 
 
14
 
15
  logger = logging.getLogger(__name__)
16
 
 
25
  self,
26
  markdown: str,
27
  embedder: Optional[Callable] = None,
28
+ batch_embedder: Optional[Callable] = None,
29
+ title_batch_embedder: Optional[Callable] = None,
30
  ) -> Dict[str, Any]:
31
  """
32
  Extract structured CV fields directly from markdown using Gemini LLM only.
33
+ Generates section embeddings using cleaned raw_sections and returns the V2 flat payload.
34
  No fallback is performed if the LLM fails.
35
  """
36
  if not markdown or not markdown.strip():
37
  raise ValueError("Markdown content is empty.")
38
 
 
39
  extracted = self.gemini_extractor.extract(markdown)
40
+ return finalize_v2_cv_payload(
41
+ markdown=markdown,
42
+ extracted=extracted,
43
+ model_used=GEMINI_MODEL,
44
+ file_type="markdown",
45
+ is_scanned=False,
46
+ page_count=1,
47
+ warnings=[],
48
+ embedder=embedder,
49
+ batch_embedder=batch_embedder,
50
+ title_batch_embedder=title_batch_embedder,
51
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
services/new/cv_postprocessor.py ADDED
@@ -0,0 +1,353 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Shared post-processing for V2 LLM CV extraction: skills, seniority, category, and ONNX embeddings."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from datetime import datetime, timezone
7
+ from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
8
+
9
+ from services.processor_utils import (
10
+ DEFAULT_CATEGORY_SKILLS,
11
+ DEFAULT_SKILL_ALIASES,
12
+ _detect_seniority,
13
+ _extract_skills,
14
+ _infer_category,
15
+ refine_seniority_with_years,
16
+ )
17
+
18
+ _SKIP_SECTION_TYPES = {"education", "awards", "contact"}
19
+ _SKIP_SECTION_TITLE_RE = re.compile(
20
+ r"\b(?:education|academic\s+background|awards?|honours?|honors?|referees?|references?|contact)\b",
21
+ re.IGNORECASE,
22
+ )
23
+
24
+ _CHUNK_MAX_CHARS = 1600
25
+ _CHUNK_STEP_CHARS = 1350
26
+ _MAX_SECTION_CHUNKS = 8
27
+ _MAX_SKILL_EMBEDDINGS = 60
28
+
29
+
30
+ def dedupe_preserving_case(values: Iterable[Any]) -> List[str]:
31
+ """Deduplicate skill strings case-insensitively while preserving display casing."""
32
+ seen = set()
33
+ out: List[str] = []
34
+ for raw in values:
35
+ cleaned = re.sub(r"\s+", " ", str(raw or "")).strip()
36
+ key = cleaned.lower()
37
+ if cleaned and key not in seen:
38
+ seen.add(key)
39
+ out.append(cleaned)
40
+ return out
41
+
42
+
43
+ def normalise_usage(raw: Any) -> Optional[Dict[str, Any]]:
44
+ """Return normalized LLM usage metadata dict, or None when unavailable."""
45
+ if not raw or not isinstance(raw, dict):
46
+ return None
47
+ res = dict(raw)
48
+ input_tokens = raw.get("inputTokenCount") or raw.get("input_tokens") or raw.get("promptTokenCount")
49
+ output_tokens = raw.get("outputTokenCount") or raw.get("output_tokens") or raw.get("candidatesTokenCount")
50
+ total_tokens = raw.get("totalTokenCount") or raw.get("total_tokens")
51
+ if total_tokens is None and input_tokens is not None and output_tokens is not None:
52
+ total_tokens = input_tokens + output_tokens
53
+ if input_tokens is not None:
54
+ res["input_tokens"] = input_tokens
55
+ if output_tokens is not None:
56
+ res["output_tokens"] = output_tokens
57
+ if total_tokens is not None:
58
+ res["total_tokens"] = total_tokens
59
+ return res
60
+
61
+
62
+ def _chunk_text(text: str, max_chars: int = _CHUNK_MAX_CHARS, step_chars: int = _CHUNK_STEP_CHARS) -> List[str]:
63
+ cleaned = re.sub(r"\s+", " ", text or "").strip()
64
+ if not cleaned:
65
+ return []
66
+ if len(cleaned) <= max_chars:
67
+ return [cleaned]
68
+ chunks: List[str] = []
69
+ start = 0
70
+ while start < len(cleaned):
71
+ chunks.append(cleaned[start : start + max_chars])
72
+ if start + max_chars >= len(cleaned):
73
+ break
74
+ start += step_chars
75
+ return chunks
76
+
77
+
78
+ def _build_relevant_section_chunks(
79
+ raw_sections: List[Dict[str, Any]],
80
+ summary: str,
81
+ experience: List[Dict[str, Any]],
82
+ projects: List[Dict[str, Any]],
83
+ ) -> List[Tuple[str, str, str]]:
84
+ """Build (title, type, chunk_text) entries from cleaned LLM sections, skipping education/awards."""
85
+ chunks: List[Tuple[str, str, str]] = []
86
+ for sec in raw_sections or []:
87
+ if not isinstance(sec, dict):
88
+ continue
89
+ sec_type = str(sec.get("type") or "other").strip().lower()
90
+ sec_title = str(sec.get("title") or sec_type.title()).strip()
91
+ content = str(sec.get("content") or "").strip()
92
+ if not content:
93
+ continue
94
+ if sec_type in _SKIP_SECTION_TYPES or _SKIP_SECTION_TITLE_RE.search(sec_title):
95
+ continue
96
+ for piece in _chunk_text(f"{sec_title}: {content}"):
97
+ chunks.append((sec_title, sec_type, piece))
98
+ if len(chunks) >= _MAX_SECTION_CHUNKS:
99
+ return chunks
100
+
101
+ if chunks:
102
+ return chunks
103
+
104
+ # Fallback if raw_sections was empty
105
+ if summary and summary.strip():
106
+ for piece in _chunk_text(f"Summary: {summary.strip()}"):
107
+ chunks.append(("Summary", "summary", piece))
108
+ for exp in experience or []:
109
+ if len(chunks) >= _MAX_SECTION_CHUNKS:
110
+ break
111
+ role = str(exp.get("role") or "").strip()
112
+ company = str(exp.get("company") or "").strip()
113
+ resps = " ".join(str(r).strip() for r in (exp.get("responsibilities") or []) if r)
114
+ text = f"{role} at {company}. {resps}".strip(" .")
115
+ for piece in _chunk_text(text):
116
+ chunks.append((role or "Experience", "experience", piece))
117
+ if len(chunks) >= _MAX_SECTION_CHUNKS:
118
+ break
119
+ for proj in projects or []:
120
+ if len(chunks) >= _MAX_SECTION_CHUNKS:
121
+ break
122
+ name = str(proj.get("name") or "").strip()
123
+ desc = str(proj.get("description") or "").strip()
124
+ tech = ", ".join(str(t).strip() for t in (proj.get("tech_stack") or []) if t)
125
+ text = f"{name}: {desc}. Tech: {tech}".strip(" .:")
126
+ for piece in _chunk_text(text):
127
+ chunks.append((name or "Projects", "projects", piece))
128
+ if len(chunks) >= _MAX_SECTION_CHUNKS:
129
+ break
130
+ return chunks
131
+
132
+
133
+ def _compute_legacy_seniority(
134
+ markdown: str,
135
+ cv_title: str,
136
+ experience: List[Dict[str, Any]],
137
+ years_experience: int,
138
+ ) -> str:
139
+ """Use the same legacy seniority extractor as jobs for uniform CV/job labels."""
140
+ exp_lines: List[str] = []
141
+ for entry in experience or []:
142
+ if not isinstance(entry, dict):
143
+ continue
144
+ role = str(entry.get("role") or "").strip()
145
+ company = str(entry.get("company") or "").strip()
146
+ resps = " ".join(str(r).strip() for r in (entry.get("responsibilities") or []) if r)
147
+ exp_lines.append(" ".join(p for p in (role, company, resps) if p))
148
+ exp_text = "\n".join(exp_lines)
149
+ seniority_text = f"{cv_title}\n{exp_text}\n{(markdown or '')[:600]}"
150
+ detected = _detect_seniority(seniority_text, is_cv=True)
151
+ return refine_seniority_with_years(detected, int(years_experience or 0))
152
+
153
+
154
+ def _encode_texts(
155
+ texts: List[str],
156
+ embedder: Optional[Callable[[str], Any]] = None,
157
+ batch_embedder: Optional[Callable[[List[str]], Any]] = None,
158
+ ) -> List[List[float]]:
159
+ if not texts:
160
+ return []
161
+ if batch_embedder is not None:
162
+ vecs = batch_embedder(texts)
163
+ return [list(map(float, v)) for v in vecs]
164
+ if embedder is not None:
165
+ return [list(map(float, embedder(t))) for t in texts]
166
+ return []
167
+
168
+
169
+ def build_section_embeddings(
170
+ *,
171
+ cv_title: str,
172
+ summary: str,
173
+ skills_raw: List[str],
174
+ raw_sections: List[Dict[str, Any]],
175
+ experience: List[Dict[str, Any]],
176
+ projects: List[Dict[str, Any]],
177
+ embedder: Optional[Callable[[str], Any]] = None,
178
+ batch_embedder: Optional[Callable[[List[str]], Any]] = None,
179
+ title_batch_embedder: Optional[Callable[[List[str]], Any]] = None,
180
+ ) -> Optional[Dict[str, Any]]:
181
+ """Build DB-ready section, skill, and title embeddings using the job matcher's ONNX models."""
182
+ if embedder is None and batch_embedder is None:
183
+ return None
184
+
185
+ section_chunks = _build_relevant_section_chunks(raw_sections, summary, experience, projects)
186
+
187
+ profile_parts = [p for p in (cv_title.strip(), summary.strip()) if p and p != "Unknown"]
188
+ if skills_raw:
189
+ profile_parts.append("Skills: " + ", ".join(skills_raw[:35]))
190
+ profile_text = "\n".join(profile_parts)[:1800]
191
+ skills_text = ", ".join(skills_raw)[:1800] if skills_raw else ""
192
+
193
+ norm_skills: List[str] = []
194
+ seen_norm = set()
195
+ for s in skills_raw[:_MAX_SKILL_EMBEDDINGS]:
196
+ ns = re.sub(r"\s+", " ", s.strip().lower())
197
+ if ns and ns not in seen_norm:
198
+ seen_norm.add(ns)
199
+ norm_skills.append(ns)
200
+
201
+ # Pack all bge-small texts into a single batched encode call
202
+ texts_to_embed: List[str] = []
203
+ profile_idx = None
204
+ if profile_text:
205
+ profile_idx = len(texts_to_embed)
206
+ texts_to_embed.append(profile_text)
207
+
208
+ skills_idx = None
209
+ if skills_text:
210
+ skills_idx = len(texts_to_embed)
211
+ texts_to_embed.append(skills_text)
212
+
213
+ chunk_start = len(texts_to_embed)
214
+ texts_to_embed.extend(chunk_text for _, _, chunk_text in section_chunks)
215
+ chunk_end = len(texts_to_embed)
216
+
217
+ skill_start = len(texts_to_embed)
218
+ texts_to_embed.extend(norm_skills)
219
+ skill_end = len(texts_to_embed)
220
+
221
+ vectors = _encode_texts(texts_to_embed, embedder=embedder, batch_embedder=batch_embedder)
222
+
223
+ result: Dict[str, Any] = {
224
+ "profile_vec": vectors[profile_idx] if profile_idx is not None else None,
225
+ "skills_vec": vectors[skills_idx] if skills_idx is not None else None,
226
+ "raw_sections_chunks": [
227
+ {
228
+ "title": section_chunks[i][0],
229
+ "type": section_chunks[i][1],
230
+ "vector": vectors[chunk_start + i],
231
+ }
232
+ for i in range(chunk_end - chunk_start)
233
+ ],
234
+ "skill_embeddings": {
235
+ norm_skills[i]: vectors[skill_start + i]
236
+ for i in range(skill_end - skill_start)
237
+ },
238
+ }
239
+
240
+ # Precompute JobBERT title + recent experience role vectors
241
+ if title_batch_embedder is not None:
242
+ titles: List[str] = []
243
+ has_cv_title = bool(cv_title and cv_title.strip() and cv_title.strip() != "Unknown")
244
+ if has_cv_title:
245
+ titles.append(cv_title.strip())
246
+ role_titles = [
247
+ str(e.get("role") or "").strip()
248
+ for e in (experience or [])[:4]
249
+ if isinstance(e, dict) and str(e.get("role") or "").strip()
250
+ ]
251
+ titles.extend(role_titles)
252
+ if titles:
253
+ t_vecs = [list(map(float, v)) for v in title_batch_embedder(titles)]
254
+ offset = 0
255
+ if has_cv_title:
256
+ result["title_vec"] = t_vecs[0]
257
+ offset = 1
258
+ else:
259
+ result["title_vec"] = None
260
+ result["role_title_vecs"] = t_vecs[offset:]
261
+
262
+ return result
263
+
264
+
265
+ def finalize_v2_cv_payload(
266
+ *,
267
+ markdown: str,
268
+ extracted: Dict[str, Any],
269
+ model_used: str,
270
+ file_type: str,
271
+ is_scanned: bool,
272
+ page_count: int,
273
+ warnings: List[str],
274
+ embedder: Optional[Callable[[str], Any]] = None,
275
+ batch_embedder: Optional[Callable[[List[str]], Any]] = None,
276
+ title_batch_embedder: Optional[Callable[[List[str]], Any]] = None,
277
+ ) -> Dict[str, Any]:
278
+ """Assemble standardized V2 CV response payload from Gemini LLM extraction."""
279
+ cv_title = str(extracted.get("cv_title") or "").strip()
280
+ years_experience = int(extracted.get("years_experience") or 0)
281
+ summary = str(extracted.get("summary") or "").strip()
282
+ experience = extracted.get("experience") or []
283
+ projects = extracted.get("projects") or []
284
+ raw_sections = extracted.get("raw_sections") or []
285
+
286
+ # Preserve Gemini extracted skills directly under skills.items (skills_raw);
287
+ # leave canonical/technical/soft null so they are never overridden by regex.
288
+ llm_skills = extracted.get("skills") or {}
289
+ skills_raw = dedupe_preserving_case(llm_skills.get("items") or [])
290
+ skills_payload = {
291
+ "items": skills_raw,
292
+ "canonical": None,
293
+ "technical": None,
294
+ "soft": None,
295
+ }
296
+
297
+ skills_for_category = {s.lower() for s in skills_raw}
298
+ if not skills_for_category:
299
+ skills_for_category = _extract_skills(markdown, DEFAULT_SKILL_ALIASES)
300
+ inferred_category = _infer_category(
301
+ text=markdown,
302
+ skills=skills_for_category,
303
+ category_skills=DEFAULT_CATEGORY_SKILLS,
304
+ )
305
+
306
+ seniority = _compute_legacy_seniority(
307
+ markdown=markdown,
308
+ cv_title=cv_title,
309
+ experience=experience,
310
+ years_experience=years_experience,
311
+ )
312
+
313
+ t_chunk_start = datetime.now(timezone.utc)
314
+ section_embeddings = build_section_embeddings(
315
+ cv_title=cv_title,
316
+ summary=summary,
317
+ skills_raw=skills_raw,
318
+ raw_sections=raw_sections,
319
+ experience=experience,
320
+ projects=projects,
321
+ embedder=embedder,
322
+ batch_embedder=batch_embedder,
323
+ title_batch_embedder=title_batch_embedder,
324
+ )
325
+ t_chunk_end = datetime.now(timezone.utc)
326
+ chunking_ms = int((t_chunk_end - t_chunk_start).total_seconds() * 1000)
327
+
328
+ return {
329
+ "method_used": "google",
330
+ "model_used": model_used,
331
+ "markdown": markdown,
332
+ "cv_title": cv_title,
333
+ "seniority": seniority,
334
+ "years_experience": years_experience,
335
+ "category": inferred_category,
336
+ "completeness_score": extracted.get("completeness_score", 0),
337
+ "summary": summary,
338
+ "contact": extracted.get("contact", {}),
339
+ "links": extracted.get("links", []),
340
+ "skills": skills_payload,
341
+ "experience": experience,
342
+ "education": extracted.get("education", []),
343
+ "projects": projects,
344
+ "awards": extracted.get("awards", []),
345
+ "raw_sections": raw_sections,
346
+ "usage_metadata": normalise_usage(extracted.get("usage_metadata", {})),
347
+ "section_embeddings": section_embeddings,
348
+ "file_type": file_type,
349
+ "is_scanned": is_scanned,
350
+ "page_count": page_count,
351
+ "warnings": warnings,
352
+ "chunking_ms": chunking_ms,
353
+ }
services/new/cv_processor_v2.py CHANGED
@@ -11,13 +11,8 @@ from typing import Any, Callable, Dict, Optional
11
  from config import GEMINI_MODEL, LLM_PROVIDERS
12
  from services.cv_chunker import chunk_cv
13
  from services.new.cv_converter_v2 import CvConverterV2
 
14
  from services.new.google.gemini_extractor import GeminiCvExtractor
15
- from services.processor_utils import (
16
- DEFAULT_CATEGORY_SKILLS,
17
- DEFAULT_SKILL_ALIASES,
18
- _extract_skills,
19
- _infer_category,
20
- )
21
 
22
  logger = logging.getLogger(__name__)
23
 
@@ -34,6 +29,8 @@ class CvProcessorV2:
34
  file_bytes: bytes,
35
  filename: str,
36
  embedder: Optional[Callable] = None,
 
 
37
  ) -> Dict[str, Any]:
38
  """
39
  Convert file → markdown, then extract fields using LLM_PROVIDERS in order.
@@ -69,74 +66,18 @@ class CvProcessorV2:
69
  if p == "google":
70
  try:
71
  extracted = self.gemini_extractor.extract(markdown)
72
-
73
- # category: use _infer_category for consistency (same as fallback)
74
- skills_found = set(
75
- s.lower() for s in extracted.get("skills", {}).get("items", [])
76
- )
77
- if not skills_found:
78
- skills_found = _extract_skills(markdown, DEFAULT_SKILL_ALIASES)
79
- inferred_category = _infer_category(
80
- text=markdown,
81
- skills=skills_found,
82
- category_skills=DEFAULT_CATEGORY_SKILLS,
83
- )
84
-
85
- # skills: use LLM output as-is for items/technical/soft (accurate)
86
- # but add canonical from _extract_skills for matching compatibility
87
- llm_skills = extracted.get("skills", {})
88
- canonical_skills = sorted(
89
- _extract_skills(markdown, DEFAULT_SKILL_ALIASES)
90
  )
91
- skills_payload = {
92
- "items": llm_skills.get("items", []),
93
- "canonical": canonical_skills,
94
- "technical": llm_skills.get("technical", []),
95
- "soft": llm_skills.get("soft", []),
96
- }
97
-
98
- # section_embeddings — always via chunk_cv with embedder
99
- t_chunk_start = datetime.now(timezone.utc)
100
- section_embeddings = None
101
- chunking_ms = 0
102
- if embedder is not None:
103
- chunk_result = chunk_cv(markdown, embedder=embedder)
104
- section_embeddings = chunk_result.get("section_embeddings")
105
- t_chunk_end = datetime.now(timezone.utc)
106
- chunking_ms = int(
107
- (t_chunk_end - t_chunk_start).total_seconds() * 1000
108
- )
109
-
110
- # usage_metadata: extract input/output/total tokens
111
- raw_usage = extracted.get("usage_metadata", {})
112
- usage_metadata = _normalise_usage(raw_usage)
113
-
114
- return {
115
- "method_used": "google",
116
- "model_used": GEMINI_MODEL,
117
- "markdown": markdown,
118
- "cv_title": extracted.get("cv_title", ""),
119
- "seniority": extracted.get("seniority", "entry"),
120
- "years_experience": extracted.get("years_experience", 0),
121
- "category": inferred_category,
122
- "completeness_score": extracted.get("completeness_score", 0),
123
- "summary": extracted.get("summary", ""),
124
- "contact": extracted.get("contact", {}),
125
- "links": extracted.get("links", []),
126
- "skills": skills_payload,
127
- "experience": extracted.get("experience", []),
128
- "education": extracted.get("education", []),
129
- "projects": extracted.get("projects", []),
130
- "awards": extracted.get("awards", []),
131
- "raw_sections": extracted.get("raw_sections", []),
132
- "usage_metadata": usage_metadata,
133
- "section_embeddings": section_embeddings,
134
- "file_type": conversion.file_type,
135
- "is_scanned": conversion.is_scanned,
136
- "page_count": conversion.page_count,
137
- "warnings": warnings,
138
- "chunking_ms": chunking_ms,
139
- }
140
  except Exception as exc:
141
  next_provider = (
142
  LLM_PROVIDERS[idx + 1] if idx + 1 < len(LLM_PROVIDERS) else None
@@ -207,23 +148,3 @@ class CvProcessorV2:
207
  tmp_path.unlink(missing_ok=True)
208
  except Exception:
209
  pass
210
-
211
-
212
- def _normalise_usage(raw: Any) -> Optional[Dict[str, Any]]:
213
- """Return usage metadata dict preserving all raw usage keys plus normalized token counts.
214
- Returns None if no usage information is available."""
215
- if not raw or not isinstance(raw, dict):
216
- return None
217
- res = dict(raw)
218
- input_tokens = raw.get("inputTokenCount") or raw.get("input_tokens") or raw.get("promptTokenCount")
219
- output_tokens = raw.get("outputTokenCount") or raw.get("output_tokens") or raw.get("candidatesTokenCount")
220
- total_tokens = raw.get("totalTokenCount") or raw.get("total_tokens")
221
- if total_tokens is None and input_tokens is not None and output_tokens is not None:
222
- total_tokens = input_tokens + output_tokens
223
- if input_tokens is not None:
224
- res["input_tokens"] = input_tokens
225
- if output_tokens is not None:
226
- res["output_tokens"] = output_tokens
227
- if total_tokens is not None:
228
- res["total_tokens"] = total_tokens
229
- return res
 
11
  from config import GEMINI_MODEL, LLM_PROVIDERS
12
  from services.cv_chunker import chunk_cv
13
  from services.new.cv_converter_v2 import CvConverterV2
14
+ from services.new.cv_postprocessor import finalize_v2_cv_payload
15
  from services.new.google.gemini_extractor import GeminiCvExtractor
 
 
 
 
 
 
16
 
17
  logger = logging.getLogger(__name__)
18
 
 
29
  file_bytes: bytes,
30
  filename: str,
31
  embedder: Optional[Callable] = None,
32
+ batch_embedder: Optional[Callable] = None,
33
+ title_batch_embedder: Optional[Callable] = None,
34
  ) -> Dict[str, Any]:
35
  """
36
  Convert file → markdown, then extract fields using LLM_PROVIDERS in order.
 
66
  if p == "google":
67
  try:
68
  extracted = self.gemini_extractor.extract(markdown)
69
+ return finalize_v2_cv_payload(
70
+ markdown=markdown,
71
+ extracted=extracted,
72
+ model_used=GEMINI_MODEL,
73
+ file_type=conversion.file_type,
74
+ is_scanned=conversion.is_scanned,
75
+ page_count=conversion.page_count,
76
+ warnings=warnings,
77
+ embedder=embedder,
78
+ batch_embedder=batch_embedder,
79
+ title_batch_embedder=title_batch_embedder,
 
 
 
 
 
 
 
80
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
81
  except Exception as exc:
82
  next_provider = (
83
  LLM_PROVIDERS[idx + 1] if idx + 1 < len(LLM_PROVIDERS) else None
 
148
  tmp_path.unlink(missing_ok=True)
149
  except Exception:
150
  pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
services/workers.py CHANGED
@@ -175,7 +175,11 @@ class CvWorker:
175
  payload = await loop.run_in_executor(
176
  _ML_EXECUTOR,
177
  lambda: self.cv_processor_v2.process(
178
- file_bytes, filename, embedder=self.matcher._embed
 
 
 
 
179
  ),
180
  )
181
 
 
175
  payload = await loop.run_in_executor(
176
  _ML_EXECUTOR,
177
  lambda: self.cv_processor_v2.process(
178
+ file_bytes,
179
+ filename,
180
+ embedder=self.matcher._embed,
181
+ batch_embedder=self.matcher._embed_batch_texts,
182
+ title_batch_embedder=self.matcher._embed_title_batch,
183
  ),
184
  )
185