Spaces:
Sleeping
Sleeping
| import re | |
| import json | |
| import logging | |
| from typing import Dict, Optional, List | |
| from src.utils import is_number | |
| from src.logger import Logger | |
| from src.consts import SKILLS, DEGREES, MATCHES, degree_map | |
| import nltk | |
| nltk.download('punkt') | |
| class BaseDataProcessor: | |
| def __init__(self): | |
| self.skill_patterns = [ | |
| rf"\b{re.escape(skill)}\b" for skill in SKILLS if len(skill) > 1 and not is_number(skill) | |
| ] | |
| self.skill_regex = re.compile("|".join(self.skill_patterns), re.IGNORECASE) | |
| self.degree_patterns = [ | |
| rf"\b{re.escape(degree)}\b" for degree in DEGREES | |
| ] | |
| self.degree_regex = re.compile("|".join(self.degree_patterns), re.IGNORECASE) | |
| self.error_logger = Logger("Error Log", see_time=True, console_log=True, level=logging.ERROR) | |
| def clean_text(self, cv_text: str) -> str: | |
| # Original clean_text method | |
| cleaned_text = re.sub(r"-{1,}\sPage\s\d+-{1,}", "", cv_text) | |
| cleaned_text = re.sub(r"\s{13,}", "\n\n", cleaned_text) | |
| cleaned_text = re.sub(r"[^\x00-\x7F]+", " ", cleaned_text) | |
| cleaned_text = re.sub(r"\u0001", "", cleaned_text) | |
| cleaned_text = re.sub(r"\n+", "\n", cleaned_text) | |
| lines = cleaned_text.split("\n") | |
| new_lines = [] | |
| for line in lines: | |
| for match in MATCHES: | |
| for pattern in MATCHES[match]: | |
| if pattern.lower() in [ | |
| line.lower(), | |
| line + "s" + line + ":", | |
| ] and len(line.strip().split(" ")) < ( | |
| len(pattern.strip().split(" ")) + 2 | |
| ): | |
| line = match + "\n" | |
| new_lines.append(line) | |
| cleaned_text = "\n".join(new_lines) | |
| return cleaned_text | |
| def extract_skills(self, text: str) -> List[str]: | |
| return list(set(self.skill_regex.findall(text.lower()))) | |
| def extract_degrees(self, text: str) -> List[str]: | |
| return list(set(self.degree_regex.findall(text.lower()))) | |
| def extract_email(self, text: str) -> Optional[str]: | |
| match = re.search(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", text) | |
| return match.group(0).lower() if match else None | |
| def format_education(self, education: List[str]) -> List[str]: | |
| # change the format of the education to a singular format such as b.s to bachelor of science | |
| # use DEGREES List which contain all the possible degrees | |
| formatted_education = [] | |
| for degree in education: | |
| for key in degree_map: | |
| if key.lower() in degree.lower(): | |
| formatted_education.append(degree_map[key]) | |
| return formatted_education | |
| def extract_experience(self, text: str) -> List[str]: | |
| experience = [] | |
| # Define a regex to capture everything after "experience" and before a new section | |
| experience_match = re.search( | |
| r"(?i)(experience|professional experience|work experience)\s*[:\n](.*?)(?=\n\s*(education|skills|certifications|projects|contact|$))", | |
| text, re.DOTALL | |
| ) | |
| if experience_match: | |
| # Capture the matched text and clean it | |
| experience_text = experience_match.group(2).strip() | |
| experience_lines = experience_text.split("\n") | |
| # Post-processing to filter out unwanted lines | |
| filtered_experience = [] | |
| for line in experience_lines: | |
| # Ignore lines that are too short or may not contain relevant content | |
| if len(line.strip()) > 5 and not re.match(r'^\s*(education|skills|certifications|projects|contact|$)', line, re.IGNORECASE): | |
| filtered_experience.append(line.strip()) | |
| experience.append("\n".join(filtered_experience)) | |
| return experience | |
| # Example usage | |
| # data_processor = BaseDataProcessor() | |
| # processed_skills = data_processor.extract_skills(cv_text) | |
| # processed_experience = data_processor.extract_experience(cv_text) | |
| # Outputs | |
| # print("Skills:", processed_skills) | |
| # print("Experience:", processed_experience) | |