src/utils.py

import re
import spacy

def save_stop_words():
    nlp = spacy.load("en_core_web_sm")
    stop_words = nlp.Defaults.stop_words

    with open("./data/spacy_stop_words.txt", "w") as file:
        for word in stop_words:
            file.write(word + "\n")

def is_number(word):
    """
    Check if the given word is a number.

    Args:
        word (_type_): The word to check

    Returns:
        bool: True if the word is a number 
    """
    try:
        float(word)
        return True
    except ValueError:
        return False
    
# Read data.skills.csv and remove empty lines and duplicates and save it
def clean_skills():
    with open("./data/skills.csv", "r", errors='ignore', encoding='utf-8') as file:
        skills = file.readlines()
    
    print(len(skills))
    # ignore line if not english
    skills = list(set([skill.strip() for skill in skills if len(skill.strip()) > 3 and not is_number(skill.strip()) and not re.search(r'[^\x00-\x7F]+', skill.strip())]))
    print(len(skills))
    
    with open("./data/skills_1.csv", "w", errors="ignore", encoding='utf-8') as file:
        for skill in skills:
            file.write(skill + "\n")
            
if __name__ == "__main__":
    save_stop_words()
    clean_skills()