src/analysis/market_jsearchAPI/data_processor.py

import re

class JobProcessor:
    
    # skills to extract
    SKILLS_LIBRARY = {
        "Programming Languages": [
            'Python', 'Java', 'C++', 'C#', 'C', 'JavaScript', 'TypeScript', 'Go', 'Rust', 
            'Swift', 'Kotlin', 'PHP', 'Ruby', 'Scala', 'R', 'MATLAB', 'Dart'
        ],
        "AI & Generative AI": [
            'Machine Learning', 'Deep Learning', 'NLP', 'Computer Vision', 'LLM', 'RAG', 
            'LangChain', 'LlamaIndex', 'Hugging Face', 'OpenAI', 'PyTorch', 'TensorFlow', 
            'Keras', 'Scikit-learn', 'Pandas', 'NumPy', 'Vector Databases', 'Pinecone', 
            'Qdrant', 'Milvus', 'Weaviate', 'Prompt Engineering'
        ],
        "Data & Databases": [
            'SQL', 'PostgreSQL', 'MySQL', 'MongoDB', 'Redis', 'Cassandra', 'Elasticsearch', 
            'Oracle', 'SQL Server', 'NoSQL', 'ETL', 'Data Warehousing', 'Big Query', 
            'Snowflake', 'Spark', 'Hadoop', 'Tableau', 'Power BI'
        ],
        "Web Development (Frameworks)": [
            'React', 'Angular', 'Vue.js', 'Next.js', 'Node.js', 'Express.js', 'Django', 
            'Flask', 'FastAPI', 'Spring Boot', 'Laravel', 'ASP.NET', 'Tailwind CSS', 'Bootstrap'
        ],
        "Cloud & DevOps (Infrastructure)": [
            'AWS', 'Azure', 'Google Cloud', 'GCP', 'Docker', 'Kubernetes', 'Terraform', 
            'Ansible', 'Jenkins', 'CI/CD', 'Linux', 'Bash', 'Git', 'GitHub', 'GitLab'
        ],
        'Cybersecurity Tech (The "How")': [
            'Penetration Testing', 'Firewalls', 'SIEM', 'IDS/IPS', 'Cryptography', 
            'OWASP', 'Vulnerability Assessment', 'Network Security', 'Identity Management',
            'SOC', 'Incident Response'
        ],
        "Mobile Development": [
            'Flutter', 'React Native', 'Ionic', 'SwiftUI', 'Android SDK'
        ],
        "UI/UX & Graphic Design": [
        'Figma', 'Adobe XD', 'Sketch', 'Photoshop', 'Illustrator', 'Indesign', 
        'After Effects', 'Canva', 'User Research', 'Wireframing', 'Prototyping', 
        'Visual Design', 'Typography', 'Color Theory', 'Interaction Design'
    ],
    "Mechanical & Industrial Engineering": [
        'SolidWorks', 'AutoCAD', 'Autodesk Inventor', 'CATIA', 'MATLAB', 'Simulink', 
        'ANSYS', 'Finite Element Analysis', 'FEA', 'Computational Fluid Dynamics', 'CFD', 
        'Thermodynamics', 'Fluid Mechanics', 'Manufacturing Processes', 'CNC Programming', 
        'Robotics', 'Control Systems', 'Piping Design', 'HVAC'
    ],
    "Embedded Systems & Hardware": [
        'Arduino', 'Raspberry Pi', 'PLC', 'Microcontrollers', 'VHDL', 'Verilog', 
        'PCB Design', 'Altium Designer', 'Proteus', 'FPGA', 'RTOS', 'ARM'
    ],
    "Business & Methodology": [
        'Project Management', 'Agile', 'Scrum', 'Kanban', 'Product Management', 
        'SDLC', 'Jira', 'Confluence', 'Business Analysis', 'Six Sigma', 'Lean', 
        'Supply Chain Management', 'Logistics', 'ERP', 'SAP'
    ],
    "Marketing & Content": [
        'SEO', 'SEM', 'Social Media Marketing', 'Google Analytics', 'Content Strategy', 
        'Copywriting', 'Email Marketing', 'Market Research', 'CRM'
    ],
    "Soft Skills": [
        'Communication', 'Leadership', 'Problem Solving', 'Teamwork', 'Adaptability', 
        'Critical Thinking', 'Time Management', 'Presentation Skills', 'Negotiation',
        'Decision Making', 'Conflict Resolution'
    ]

    }

    @staticmethod
    def analyze_jobs(raw_jobs):
        if not raw_jobs:
            return {}

        results = []
        all_skills_frequency = {}
        
        flat_skills = [skill for category in JobProcessor.SKILLS_LIBRARY.values() for skill in category]

        for job in raw_jobs:
            description = job.get('job_description', '') or ''
            
            # To lower case  for better search
            full_text_lower = description.lower()
            job_skills = []

            # extracting skills from the job describtion
            for skill in flat_skills:
                if re.search(r'\b' + re.escape(skill.lower()) + r'\b', full_text_lower):
                    job_skills.append(skill)
                    all_skills_frequency[skill] = all_skills_frequency.get(skill, 0) + 1

            # Building job information
            job_info = {
                "title": job.get("job_title"),
                "company": job.get("employer_name"),
                "location": f"{job.get('job_city', 'N/A')}, {job.get('job_country', 'N/A')}",
                "is_remote": job.get("job_is_remote", False),
                "employment_type": job.get("job_employment_type"),
                "salary": f"{job.get('job_min_salary')} - {job.get('job_max_salary')}" if job.get('job_min_salary') else "Not Mentioned",
                "extracted_skills": job_skills,
                "apply_link": job.get("job_apply_link")
            }
            results.append(job_info)
        
        # Sorting skills descending depending on frequency 
        top_skills = sorted(all_skills_frequency.items(), key=lambda x: x[1], reverse=True)

        return {
            "market_summary": {
                "total_jobs_found": len(raw_jobs),
                "top_extracted_skills": [{"skill": s, "count": c} for s, c in top_skills[:20]],
                "remote_friendly_count": sum(1 for j in results if j.get('is_remote', False))
            },
            "detailed_listings": results
        }