Spaces:
Sleeping
Sleeping
File size: 4,292 Bytes
be5fcd5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 |
import re
import logging
# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
class JobTitlePreprocessor():
"""Preprocesses job titles by converting to lowercase, removing unwanted words, special characters, numbers greater than 10, and content from location, states, regions, etc."""
def __init__(self):
self.unwanted_words = ['remote', 'hybrid', 'flexible location', 'location', 'open to work', 'role', 'job', 'level', 'remot']
def remove_location_unwanted_words_brackets(self, title: str) -> str:
"""Removes parts of the title based on unwanted words, bracketed content, numbers greater than 10, and also removes symbols other than alphanumeric."""
# Remove unwanted words
for word in self.unwanted_words:
pattern = r'\b{}\b'.format(re.escape(word))
title = re.sub(pattern, '', title, flags=re.IGNORECASE)
# Remove content within brackets
title = re.sub(r'\[.*?\]|\(.*?\)|\{.*?\}', '', title)
# Remove any non-alphanumeric characters (keeping spaces)
title = re.sub(r'[^a-zA-Z0-9\s]', '', title)
# Remove numbers greater than 10
title = re.sub(r'\b(?:[1-9][0-9]+|1[1-9]|[2-9][0-9])\b', '', title)
# Clean up extra spaces
title = re.sub(r'\s+', ' ', title).strip()
return title
def preprocess(self, title: str) -> str:
"""Converts title to lowercase, removes unwanted words, replaces specific terms, and standardizes job titles."""
if not isinstance(title, str):
return title
# Convert to lowercase
title = title.lower()
# Remove unwanted words
for word in self.unwanted_words:
title = re.sub(r'\b{}\b'.format(re.escape(word)), '', title, flags=re.IGNORECASE)
# Replace specific terms and Roman numerals
replacements = [
(r'\b(?:SR|sr|Sr\.?|SR\.?|Senior|senior)\b', 'Senior'),
(r'\b(?:JR|jr|Jr\.?|JR\.?|Junior|junior)\b', 'Junior'),
(r'\b(?:AIML|aiml|ML|ml|MachineLearning|machinelearning|machine[_\-]learning)\b', 'Machine Learning'),
(r'\b(?:GenAI|genai|Genai|generative[_\-]ai|GenerativeAI|generativeai)\b', 'Generative AI'),
(r'\b(?:NLP|nlp|natural[_\-]language[_\-]processing|natural language processing)\b', 'NLP'),
(r'\b(?:i|I)\b', '1'),
(r'\b(?:ii|II)\b', '2'),
(r'\b(?:iii|III)\b', '3'),
(r'\b(?:iv|IV)\b', '4'),
(r'\b(?:v|V)\b', '5')
]
for pattern, replacement in replacements:
title = re.sub(pattern, replacement, title, flags=re.IGNORECASE)
# Handle specific Data Scientist cases
title = re.sub(r'\b(director|dir\.?|dir)\b.*?(data\s*scientist|data\s*science)', 'Director Data Scientist', title, flags=re.IGNORECASE)
title = re.sub(r'\b(manager|mgr)\b.*?(data\s*scientist|data\s*science)', 'Manager Data Scientist', title, flags=re.IGNORECASE)
title = re.sub(r'\b(lead)\b.*?(data\s*scientist|data\s*science)', 'Lead Data Scientist', title, flags=re.IGNORECASE)
title = re.sub(r'\b(associate|associates?)\b.*?(data\s*scientist|data\s*science)', 'Associate Data Scientist', title, flags=re.IGNORECASE)
title = re.sub(r'\b(applied)\b.*?(data\s*scientist|data\s*science)', 'Applied Data Scientist', title, flags=re.IGNORECASE)
title = re.sub(r'\b(intern|internship|trainee)\b.*?(data\s*scientist|data\s*science)', 'Intern Data Scientist', title, flags=re.IGNORECASE)
# Clean up extra spaces
title = re.sub(r'\s+', ' ', title).strip()
return title
def preprocess_single_title(title: str) -> str:
preprocessor = JobTitlePreprocessor()
clean_title = preprocessor.remove_location_unwanted_words_brackets(title)
clean_title = preprocessor.preprocess(clean_title)
return clean_title
if __name__ == "__main__":
# Example single title
title = "Senior Remote Machine Learning Data Scientist (Manager)"
clean_title = preprocess_single_title(title)
logger.info(f"Original title: {title}")
logger.info(f"Preprocessed title: {clean_title}")
|