Spaces:
Sleeping
Sleeping
File size: 4,193 Bytes
14ca0c1 7972b6e 14ca0c1 7972b6e 14ca0c1 fea3c1a 14ca0c1 82d026c 7972b6e 82d026c 14ca0c1 82d026c 14ca0c1 82d026c 7972b6e 14ca0c1 82d026c 7972b6e 82d026c 7972b6e 82d026c 14ca0c1 7972b6e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | import re
import json
import logging
from typing import Dict, Optional, List
from src.utils import is_number
from src.logger import Logger
from src.consts import SKILLS, DEGREES, MATCHES, degree_map
import nltk
nltk.download('punkt')
class BaseDataProcessor:
def __init__(self):
self.skill_patterns = [
rf"\b{re.escape(skill)}\b" for skill in SKILLS if len(skill) > 1 and not is_number(skill)
]
self.skill_regex = re.compile("|".join(self.skill_patterns), re.IGNORECASE)
self.degree_patterns = [
rf"\b{re.escape(degree)}\b" for degree in DEGREES
]
self.degree_regex = re.compile("|".join(self.degree_patterns), re.IGNORECASE)
self.error_logger = Logger("Error Log", see_time=True, console_log=True, level=logging.ERROR)
def clean_text(self, cv_text: str) -> str:
# Original clean_text method
cleaned_text = re.sub(r"-{1,}\sPage\s\d+-{1,}", "", cv_text)
cleaned_text = re.sub(r"\s{13,}", "\n\n", cleaned_text)
cleaned_text = re.sub(r"[^\x00-\x7F]+", " ", cleaned_text)
cleaned_text = re.sub(r"\u0001", "", cleaned_text)
cleaned_text = re.sub(r"\n+", "\n", cleaned_text)
lines = cleaned_text.split("\n")
new_lines = []
for line in lines:
for match in MATCHES:
for pattern in MATCHES[match]:
if pattern.lower() in [
line.lower(),
line + "s" + line + ":",
] and len(line.strip().split(" ")) < (
len(pattern.strip().split(" ")) + 2
):
line = match + "\n"
new_lines.append(line)
cleaned_text = "\n".join(new_lines)
return cleaned_text
def extract_skills(self, text: str) -> List[str]:
return list(set(self.skill_regex.findall(text.lower())))
def extract_degrees(self, text: str) -> List[str]:
return list(set(self.degree_regex.findall(text.lower())))
def extract_email(self, text: str) -> Optional[str]:
match = re.search(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", text)
return match.group(0).lower() if match else None
def format_education(self, education: List[str]) -> List[str]:
# change the format of the education to a singular format such as b.s to bachelor of science
# use DEGREES List which contain all the possible degrees
formatted_education = []
for degree in education:
for key in degree_map:
if key.lower() in degree.lower():
formatted_education.append(degree_map[key])
return formatted_education
def extract_experience(self, text: str) -> List[str]:
experience = []
# Define a regex to capture everything after "experience" and before a new section
experience_match = re.search(
r"(?i)(experience|professional experience|work experience)\s*[:\n](.*?)(?=\n\s*(education|skills|certifications|projects|contact|$))",
text, re.DOTALL
)
if experience_match:
# Capture the matched text and clean it
experience_text = experience_match.group(2).strip()
experience_lines = experience_text.split("\n")
# Post-processing to filter out unwanted lines
filtered_experience = []
for line in experience_lines:
# Ignore lines that are too short or may not contain relevant content
if len(line.strip()) > 5 and not re.match(r'^\s*(education|skills|certifications|projects|contact|$)', line, re.IGNORECASE):
filtered_experience.append(line.strip())
experience.append("\n".join(filtered_experience))
return experience
# Example usage
# data_processor = BaseDataProcessor()
# processed_skills = data_processor.extract_skills(cv_text)
# processed_experience = data_processor.extract_experience(cv_text)
# Outputs
# print("Skills:", processed_skills)
# print("Experience:", processed_experience)
|