Spaces:
Sleeping
Sleeping
File size: 2,352 Bytes
96a2583 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | # ============================================================
# preprocess.py
# This file cleans resume and job description text
# so the AI can understand it better.
# ============================================================
import re
import nltk
# Download required NLTK data (run once)
nltk.download('stopwords', quiet=True)
nltk.download('wordnet', quiet=True)
nltk.download('punkt', quiet=True)
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
# Load English stopwords (common words like "the", "is", "and")
STOPWORDS = set(stopwords.words('english'))
# Lemmatizer converts words to their base form (e.g. "running" → "run")
lemmatizer = WordNetLemmatizer()
def clean_text(text):
"""
Cleans a single piece of text (resume or job description).
Steps: lowercase → remove URLs → remove emails →
remove punctuation → remove numbers → remove extra spaces
"""
if not isinstance(text, str):
return "" # return empty string if text is missing/NaN
# Step 1: Convert to lowercase
text = text.lower()
# Step 2: Remove URLs (like https://linkedin.com/...)
text = re.sub(r'http\S+|www\S+', '', text)
# Step 3: Remove email addresses
text = re.sub(r'\S+@\S+', '', text)
# Step 4: Remove punctuation (keep only letters and spaces)
text = re.sub(r'[^a-z\s]', '', text)
# Step 5: Remove extra whitespace
text = re.sub(r'\s+', ' ', text).strip()
return text
def remove_stopwords(text):
"""
Removes common English words that don't add meaning.
Example: 'i am a data scientist' → 'data scientist'
"""
words = text.split()
filtered = [word for word in words if word not in STOPWORDS]
return ' '.join(filtered)
def lemmatize_text(text):
"""
Reduces words to their base/root form.
Example: 'developed building created' → 'develop build create'
"""
words = text.split()
lemmatized = [lemmatizer.lemmatize(word) for word in words]
return ' '.join(lemmatized)
def full_preprocess(text):
"""
Runs the complete NLP preprocessing pipeline:
clean → remove stopwords → lemmatize
Use this function on any resume or job description text.
"""
text = clean_text(text)
text = remove_stopwords(text)
text = lemmatize_text(text)
return text
|