Initial BERT backend upload

.gitattributes

@@ -0,0 +1,41 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+ner_model/transformer/model filter=lfs diff=lfs merge=lfs -text
+*.csv filter=lfs diff=lfs merge=lfs -text
+job_embeddings.pt filter=lfs diff=lfs merge=lfs -text
+ner_model/ner/model filter=lfs diff=lfs merge=lfs -text
+ner_model/tokenizer filter=lfs diff=lfs merge=lfs -text
+ner_model/vocab/lookups.bin filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+cached_models/
+__pycache__/

Dockerfile

@@ -0,0 +1,31 @@
+# Use an official Python runtime as a parent image
+FROM python:3.11-slim
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Install system dependencies required for building some Python packages
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy the requirements files into the container
+COPY requirements.txt similarity_requirements.txt ./
+
+# --- Create and populate the main virtual environment ---
+RUN python3 -m venv env
+RUN ./env/bin/pip install --no-cache-dir -r requirements.txt
+
+# --- Create and populate the similarity worker virtual environment ---
+RUN python3 -m venv similarity_env
+RUN ./similarity_env/bin/pip install --no-cache-dir -r similarity_requirements.txt
+
+# Copy the rest of the backend application files into the container
+COPY . .
+
+# Expose the port the app runs on (standard for Hugging Face Spaces)
+EXPOSE 7860
+
+# Define the command to run the application
+# Use the Python from the main virtual environment to run uvicorn
+CMD ["./env/bin/python3", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,11 @@
+---
+title: Skill Gap Bert Backend
+emoji: 🏃
+colorFrom: purple
+colorTo: indigo
+sdk: docker
+pinned: false
+license: apache-2.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

final_prototype_postings.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b253b04a9032662b233233674118a1f5306f55e2e8e7edb38ae6362eb2a3830
+size 26993136

job_embeddings.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6da951796f43a827628e5266f8a0f761f346efb56c1e97de445c3bec01f671b7
+size 36259418

main.py

@@ -0,0 +1,475 @@
+import pandas as pd
+from fastapi import FastAPI, UploadFile, File, Form, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from typing import List, Dict, Any, Optional
+import os
+import fitz  # PyMuPDF
+import torch
+import spacy
+import re
+from bs4 import BeautifulSoup
+import emoji
+import subprocess
+import json
+import sys
+import pathlib
+import uuid
+import time
+
+# --- Text Cleaning Functions ---
+
+def old_refined_text_cleaning(text: str) -> str:
+    """The OLD cleaning function used for the annotation phase. Removes #, +, / etc."""
+    if not isinstance(text, str):
+        return ""
+    text = BeautifulSoup(text, "html.parser").get_text()
+    url_pattern = r'(?:(?:https?|ftp)://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})))(?::\d{2,5})?(?:/\S*)?'
+    text = re.sub(url_pattern, '', text)
+    text = re.sub(r'\S+@\S+\s?', '', text)
+    text = emoji.demojize(text)
+    text = re.sub(r':[a-zA-Z_]+:', '', text)
+    text = text.replace('\\', ' ')
+    text = re.sub(r'[#*•]', ' ', text)
+    text = re.sub(r'\{.*?\}', ' ', text)
+    text = re.sub(r'[^a-zA-Z0-9\s.,!?-]', ' ', text)
+    text = re.sub(r'\s+', ' ', text)
+    text = re.sub(r'\s([,.!?-])', r'\1', text)
+    text = text.strip()
+    text = text.lower()
+    return text
+
+def new_refined_text_cleaning(text: str) -> str:
+    """The NEW, improved cleaning function. Keeps technical symbols."""
+    if not isinstance(text, str):
+        return ""
+    text = BeautifulSoup(text, "html.parser").get_text()
+    url_pattern = r'(?:(?:https?|ftp)://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})))(?::\d{2,5})?(?:/\S*)?'
+    text = re.sub(url_pattern, '', text)
+    text = re.sub(r'\S+@\S+\s?', '', text)
+    text = emoji.demojize(text)
+    text = re.sub(r':[a-zA-Z_]+:', '', text)
+    text = text.replace('\\', ' ')
+    text = re.sub(r'[*•]', ' ', text) # Keep '#' from old regex r'[#*•]' to preserve C#
+    text = re.sub(r'\{.*?\}', ' ', text)
+    # Keep '#', '+', '/', '()', and '_' to preserve technical terms.
+    text = re.sub(r'[^a-zA-Z0-9_#+()/\s.,!?-]', ' ', text)
+    text = re.sub(r'\s+', ' ', text)
+    text = re.sub(r'\s([,.!?-])', r'\1', text)
+    text = text.strip()
+    text = text.lower()
+    return text
+
+
+# --- Pydantic Models for API Response Structure ---
+
+class SkillCount(BaseModel):
+    skill: str
+    count: int
+
+class ToolCount(BaseModel):
+    tool: str
+    count: int
+
+class RoleSkill(BaseModel):
+    cmo_role_match: str
+    skill: str
+    count: int
+
+class RoleTool(BaseModel):
+    cmo_role_match: str
+    tool: str
+    count: int
+
+class ExperienceDistribution(BaseModel):
+    year: int
+    count: int
+
+class SkillCooccurrence(BaseModel):
+    skill_A: str
+    skill_B: str
+    count: int
+
+class ToolCooccurrence(BaseModel):
+    tool_A: str
+    tool_B: str
+    count: int
+
+class JobRoleDistribution(BaseModel):
+    cmo_role_match: str
+    count: int
+
+class RoleInsightsResponse(BaseModel):
+    top_skills: List[RoleSkill]
+    total_skills: int
+    top_tools: List[RoleTool]
+    total_tools: int
+    average_experience: Optional[float] = None
+    experience_distribution: List[ExperienceDistribution]
+    total_experience_distribution: int
+    skill_co_occurrence: List[SkillCooccurrence]
+    total_skill_co_occurrence: int
+    tool_co_occurrence: List[ToolCooccurrence]
+    total_tool_co_occurrence: int
+
+class MarketInsightsResponse(BaseModel):
+    top_overall_skills: List[SkillCount]
+    total_overall_skills: int
+    top_overall_tools: List[ToolCount]
+    total_overall_tools: int
+    experience_distribution: List[ExperienceDistribution]
+    total_experience_distribution: int
+    skill_co_occurrence: List[SkillCooccurrence]
+    total_skill_co_occurrence: int
+    tool_co_occurrence: List[ToolCooccurrence]
+    total_tool_co_occurrence: int
+    average_experience: Optional[float] = None
+
+class SimilarJob(BaseModel):
+    job_title: str
+    similarity_score: float
+    cmo_role_match: str
+    url: Optional[str] = None
+
+class SkillDetail(BaseModel):
+    name: str
+    count: int
+
+class GapAnalysis(BaseModel):
+    user_skills: List[SkillDetail]
+    user_tools: List[SkillDetail]
+    missing_skills: List[SkillDetail]
+    matching_skills: List[SkillDetail]
+    missing_tools: List[SkillDetail]
+    matching_tools: List[SkillDetail]
+    total_user_skills: int
+    total_user_tools: int
+    total_missing_skills: int
+    total_matching_skills: int
+    total_missing_tools: int
+    total_matching_tools: int
+
+class AnalysisResult(BaseModel):
+    similar_jobs: List[SimilarJob]
+    total_similar_jobs: int
+    gap_analysis: GapAnalysis
+    recommendations: Dict[str, Any]
+    session_id: str
+
+# --- App instantiation ---
+app = FastAPI(
+    title="Skill Gap Analyzer API",
+    description="API for market insights and resume analysis.",
+    version="1.3.0", # Version bump
+)
+
+# --- CORS Middleware ---
+origins = [
+    "http://localhost:5173",
+    "http://127.0.0.1:5173",
+    "http://localhost:5174",
+]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# --- In-memory storage for models and data ---
+DB = {}
+
+@app.on_event("startup")
+async def startup_event():
+    DB['similarity_cache'] = {}
+    """Load models and data into memory on application startup."""
+    print("INFO:     Loading models and data...")
+    
+    backend_dir = os.path.dirname(os.path.abspath(__file__))
+    model_path = os.path.join(backend_dir, "ner_model")
+
+    # --- Load Pre-computed Insights ---
+    insights_path = os.path.join(backend_dir, 'market_insights.json')
+    with open(insights_path, 'r') as f:
+        DB['insights'] = json.load(f)
+    print("INFO:     Market insights loaded successfully.")
+
+    # --- Load other necessary data ---
+    # This is still needed for the similarity worker and gap analysis source
+    DB['market_data'] = pd.read_csv(os.path.join(backend_dir, 'final_prototype_postings.csv'))
+
+    # --- Load Models ---
+    print(f"INFO:     Loading NER model from {model_path}...")
+    DB['ner_model'] = spacy.load(model_path)
+    print("INFO:     NER model loaded successfully.")
+
+    print("INFO:     Models and data loaded successfully.")
+
+
+@app.get("/", tags=["General"])
+async def read_root():
+    return {"message": "Welcome to the Skill Gap Analyzer API v1.3"}
+
+@app.get("/roles", response_model=List[str], tags=["Market Insights"])
+async def get_roles():
+    roles = sorted(DB['insights']['by_role'].keys())
+    return ["Overall Market"] + roles
+
+@app.get("/job_roles_distribution", response_model=List[JobRoleDistribution], tags=["Market Insights"])
+async def get_job_roles_distribution():
+    return DB['insights']['job_role_distribution']
+
+
+@app.get("/market_insights", response_model=MarketInsightsResponse, tags=["Market Insights"])
+async def get_market_insights(page: int = 1, limit: int = 20):
+    start = (page - 1) * limit
+    end = page * limit
+    
+    overall_data = DB['insights']['overall_market']
+    
+    top_skills = overall_data.get('top_skills', [])
+    top_tools = overall_data.get('top_tools', [])
+    exp_dist = overall_data.get('experience_distribution', [])
+    skill_co = overall_data.get('skill_co_occurrence', [])
+    tool_co = overall_data.get('tool_co_occurrence', [])
+    avg_exp = overall_data.get('average_experience')
+
+    return {
+        "top_overall_skills": top_skills[start:end],
+        "total_overall_skills": len(top_skills),
+        "top_overall_tools": top_tools[start:end],
+        "total_overall_tools": len(top_tools),
+        "experience_distribution": exp_dist[start:end],
+        "total_experience_distribution": len(exp_dist),
+        "skill_co_occurrence": skill_co[start:end],
+        "total_skill_co_occurrence": len(skill_co),
+        "tool_co_occurrence": tool_co[start:end],
+        "total_tool_co_occurrence": len(tool_co),
+        "average_experience": avg_exp,
+    }
+
+@app.get("/market_insights/{role:path}", response_model=RoleInsightsResponse, tags=["Market Insights"])
+async def get_role_insights(role: str, page: int = 1, limit: int = 10):
+    start = (page - 1) * limit
+    end = page * limit
+
+    role_data = DB['insights']['by_role'].get(role)
+    if not role_data:
+        raise HTTPException(status_code=404, detail="Role not found")
+
+    top_skills = role_data.get('top_skills', [])
+    top_tools = role_data.get('top_tools', [])
+    exp_dist = role_data.get('experience_distribution', [])
+    skill_co = role_data.get('skill_co_occurrence', [])
+    tool_co = role_data.get('tool_co_occurrence', [])
+    avg_exp = role_data.get('average_experience')
+
+    return {
+        "top_skills": top_skills[start:end],
+        "total_skills": len(top_skills),
+        "top_tools": top_tools[start:end],
+        "total_tools": len(top_tools),
+        "average_experience": avg_exp,
+        "experience_distribution": exp_dist[start:end],
+        "total_experience_distribution": len(exp_dist),
+        "skill_co_occurrence": skill_co[start:end],
+        "total_skill_co_occurrence": len(skill_co),
+        "tool_co_occurrence": tool_co[start:end],
+        "total_tool_co_occurrence": len(tool_co),
+    }
+
+@app.post("/analyze_resume", response_model=AnalysisResult, tags=["Resume Analysis"])
+async def analyze_resume(
+    resume_file: UploadFile = File(...),
+    target_role: Optional[str] = Form(None),
+    limit: Optional[int] = Form(10) # This limit is now for the initial page load
+):
+    # --- PDF Processing ---
+    if not resume_file or not resume_file.filename.lower().endswith('.pdf'):
+        raise HTTPException(status_code=400, detail="Invalid file type. Please upload a PDF.")
+
+    pdf_bytes = await resume_file.read()
+
+    MAX_FILE_SIZE = 1 * 1024 * 1024  # 1MB
+    if len(pdf_bytes) > MAX_FILE_SIZE:
+        raise HTTPException(
+            status_code=413,
+            detail="File is too large. Please upload a PDF under 1MB."
+        )
+
+    resume_text = ""
+    try:
+        with fitz.open(stream=pdf_bytes, filetype="pdf") as doc:
+            for page in doc:
+                resume_text += page.get_text()
+    except Exception as e:
+        raise HTTPException(status_code=422, detail=f"Failed to process PDF file: {e}")
+
+    if not resume_text or resume_text.isspace():
+        raise HTTPException(
+            status_code=422,
+            detail="Could not extract any text from the provided PDF. The document may be empty, image-based, or corrupted."
+        )
+
+    # --- Text Cleaning ---
+    ner_cleaned_text = old_refined_text_cleaning(resume_text)
+    similarity_cleaned_text = new_refined_text_cleaning(resume_text)
+
+    # --- NER Processing ---
+    start_time = time.time()
+    doc = DB['ner_model'](ner_cleaned_text)
+    end_time = time.time()
+    print(f"--- BERT NER INFERENCE TIME: {end_time - start_time:.4f} seconds ---")
+    user_skills = [ent.text for ent in doc.ents if ent.label_ == "SKILL"]
+    user_tools = [ent.text for ent in doc.ents if ent.label_ == "TOOL"]
+
+    # --- Similarity Search (via Isolated Subprocess) ---
+    # Fetch a large number of jobs to cache for pagination
+    all_similar_jobs = []
+    total_similar_jobs = 0
+    try:
+        backend_dir = pathlib.Path(__file__).parent.resolve()
+        worker_path = backend_dir / "similarity_worker.py"
+        worker_python_executable = "/app/similarity_env/bin/python3"
+        
+        process = subprocess.run(
+            [
+                str(worker_python_executable),
+                str(worker_path),
+                "--target_role",
+                target_role or "Overall Market",
+                "--limit",
+                "200"  # Fetch a large batch for caching
+            ],
+            input=similarity_cleaned_text,
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        
+        worker_output = json.loads(process.stdout)
+        all_similar_jobs = worker_output.get("similar_jobs", [])
+        total_similar_jobs = worker_output.get("total_jobs", 0)
+
+    except (subprocess.CalledProcessError, json.JSONDecodeError, FileNotFoundError) as e:
+        print(f"Similarity worker failed: {e}", file=sys.stderr)
+        if isinstance(e, subprocess.CalledProcessError):
+            print(f"Worker stderr: {e.stderr}", file=sys.stderr)
+        all_similar_jobs = []
+        total_similar_jobs = 0
+
+    # --- Cache the full results ---
+    session_id = str(uuid.uuid4())
+
+    # Simple cache eviction: Keep cache size under a limit (e.g., 50)
+    if len(DB['similarity_cache']) > 50:
+        try:
+            oldest_key = next(iter(DB['similarity_cache']))
+            del DB['similarity_cache'][oldest_key]
+        except (StopIteration, KeyError):
+            # Handle edge cases where cache might be empty or key is gone
+            pass
+
+    DB['similarity_cache'][session_id] = all_similar_jobs
+
+    # --- Gap Analysis (remains the same) ---
+    if target_role and target_role != "Overall Market":
+        role_data = DB['insights']['by_role'].get(target_role, {})
+        market_skills_list = role_data.get('top_skills', []) 
+        market_tools_list = role_data.get('top_tools', [])
+    else:
+        overall_data = DB['insights']['overall_market']
+        market_skills_list = overall_data.get('top_skills', [])
+        market_tools_list = overall_data.get('top_tools', [])
+
+    market_skill_freq = {s['skill'].lower(): s['count'] for s in market_skills_list}
+    market_tool_freq = {t['tool'].lower(): t['count'] for t in market_tools_list}
+    user_skills_lower = {s.lower() for s in user_skills}
+    user_tools_lower = {t.lower() for t in user_tools}
+    missing_skills = [{"name": s['skill'], "count": s['count']} for s in market_skills_list if s['skill'].lower() not in user_skills_lower]
+    matching_skills = [{"name": s['skill'], "count": s['count']} for s in market_skills_list if s['skill'].lower() in user_skills_lower]
+    missing_tools = [{"name": t['tool'], "count": t['count']} for t in market_tools_list if t['tool'].lower() not in user_tools_lower]
+    matching_tools = [{"name": t['tool'], "count": t['count']} for t in market_tools_list if t['tool'].lower() in user_tools_lower]
+    user_skills_with_freq = [{"name": s, "count": market_skill_freq.get(s.lower(), 0)} for s in user_skills]
+    user_tools_with_freq = [{"name": t, "count": market_tool_freq.get(t.lower(), 0)} for t in user_tools]
+    gap_analysis = {
+        "user_skills": user_skills_with_freq,
+        "user_tools": user_tools_with_freq,
+        "missing_skills": missing_skills,
+        "matching_skills": matching_skills,
+        "missing_tools": missing_tools,
+        "matching_tools": matching_tools,
+        "total_user_skills": len(user_skills),
+        "total_user_tools": len(user_tools),
+        "total_missing_skills": len(missing_skills),
+        "total_matching_skills": len(matching_skills),
+        "total_missing_tools": len(missing_tools),
+        "total_matching_tools": len(matching_tools),
+    }
+
+    # --- Recommendation Generation (remains the same) ---
+    all_user_entities = user_skills_lower.union(user_tools_lower)
+    recommendations = {
+        "message": "Based on your resume, focusing on these skills and tools could improve your market alignment. We also recommend looking at co-occurring skills for your existing strengths.",
+        "skills_to_learn": missing_skills[:5],
+        "tools_to_learn": missing_tools[:5],
+        "based_on_your_strengths": {}
+    }
+    skill_co_data = []
+    tool_co_data = []
+    if target_role and target_role != "Overall Market":
+        role_data = DB['insights']['by_role'].get(target_role, {})
+        skill_co_data = role_data.get('skill_co_occurrence', [])
+        tool_co_data = role_data.get('tool_co_occurrence', [])
+    else:
+        overall_data = DB['insights']['overall_market']
+        skill_co_data = overall_data.get('skill_co_occurrence', [])
+        tool_co_data = overall_data.get('tool_co_occurrence', [])
+    df_list = []
+    if skill_co_data:
+        skills_df = pd.DataFrame(skill_co_data)
+        if 'skill_A' in skills_df.columns and 'skill_B' in skills_df.columns:
+            skills_df = skills_df.rename(columns={'skill_A': 'entity_A', 'skill_B': 'entity_B'})
+            df_list.append(skills_df)
+    if tool_co_data:
+        tools_df = pd.DataFrame(tool_co_data)
+        if 'tool_A' in tools_df.columns and 'tool_B' in tools_df.columns:
+            tools_df = tools_df.rename(columns={'tool_A': 'entity_A', 'tool_B': 'entity_B'})
+            df_list.append(tools_df)
+    if df_list:
+        co_occurrence_df = pd.concat(df_list, ignore_index=True)
+        if 'entity_A' in co_occurrence_df.columns and 'entity_B' in co_occurrence_df.columns:
+            for entity in all_user_entities:
+                related_A = co_occurrence_df[co_occurrence_df['entity_B'].str.lower() == entity]['entity_A'].tolist()
+                related_B = co_occurrence_df[co_occurrence_df['entity_A'].str.lower() == entity]['entity_B'].tolist()
+                related_entities = related_A + related_B
+                recommended = [s for s in related_entities if s.lower() not in all_user_entities]
+                if recommended:
+                    unique_recommended = list(dict.fromkeys(recommended))
+                    recommendations["based_on_your_strengths"][entity] = unique_recommended[:3]
+
+    # --- Final Response ---
+    return {
+        "similar_jobs": all_similar_jobs[:limit], # Return only the first page
+        "total_similar_jobs": total_similar_jobs,
+        "gap_analysis": gap_analysis,
+        "recommendations": recommendations,
+        "session_id": session_id,
+    }
+
+@app.get("/similar_jobs/{session_id}", response_model=List[SimilarJob], tags=["Resume Analysis"])
+async def get_more_similar_jobs(session_id: str, page: int = 1, limit: int = 10):
+    """
+    Gets a paginated list of similar jobs from the cache.
+    """
+    if session_id not in DB['similarity_cache']:
+        raise HTTPException(status_code=404, detail="Session not found or expired.")
+
+    full_job_list = DB['similarity_cache'][session_id]
+    
+    start_index = (page - 1) * limit
+    end_index = page * limit
+    
+    return full_job_list[start_index:end_index]

market_data_with_entities.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d9544e829d6e258b26907a303488bdaf6e5ea2767dfd918d90ac3d90edeeb45
+size 27560352

ner_model/config.cfg

@@ -0,0 +1,148 @@
+[paths]
+train = "assets/train.spacy"
+dev = "assets/dev.spacy"
+vectors = null
+init_tok2vec = null
+
+[system]
+seed = 0
+gpu_allocator = "pytorch"
+
+[nlp]
+lang = "en"
+pipeline = ["transformer","ner"]
+batch_size = 128
+disabled = []
+before_creation = null
+after_creation = null
+after_pipeline_creation = null
+tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
+vectors = {"@vectors":"spacy.Vectors.v1"}
+
+[components]
+
+[components.ner]
+factory = "ner"
+incorrect_spans_key = null
+moves = null
+scorer = {"@scorers":"spacy.ner_scorer.v1"}
+update_with_oracle_cut_size = 100
+
+[components.ner.model]
+@architectures = "spacy.TransitionBasedParser.v2"
+state_type = "ner"
+extra_state_tokens = false
+hidden_width = 64
+maxout_pieces = 2
+use_upper = true
+nO = null
+
+[components.ner.model.tok2vec]
+@architectures = "spacy-transformers.TransformerListener.v1"
+pooling = {"@layers":"reduce_mean.v1"}
+grad_factor = 1.0
+upstream = "*"
+
+[components.transformer]
+factory = "transformer"
+max_batch_items = 174000
+set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"}
+
+[components.transformer.model]
+@architectures = "spacy-transformers.TransformerModel.v3"
+name = "bert-base-uncased"
+mixed_precision = true
+
+[components.transformer.model.get_spans]
+@span_getters = "spacy-transformers.strided_spans.v1"
+window = 128
+stride = 96
+
+[components.transformer.model.grad_scaler_config]
+
+[components.transformer.model.tokenizer_config]
+use_fast = true
+
+[components.transformer.model.transformer_config]
+
+[corpora]
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+gold_preproc = false
+max_length = 0
+limit = 0
+augmenter = null
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+gold_preproc = false
+max_length = 0
+limit = 0
+augmenter = null
+
+[training]
+seed = ${system.seed}
+gpu_allocator = ${system.gpu_allocator}
+dropout = 0.1
+accumulate_gradient = 1
+max_steps = 20000
+eval_frequency = 200
+frozen_components = []
+annotating_components = []
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+patience = 3200
+max_epochs = 0
+before_to_disk = null
+before_update = null
+
+[training.batcher]
+@batchers = "spacy.batch_by_words.v1"
+discard_oversize = true
+tolerance = 0.2
+get_length = null
+
+[training.batcher.size]
+@schedules = "compounding.v1"
+start = 100
+stop = 1000
+compound = 1.001
+t = 0.0
+
+[training.logger]
+@loggers = "spacy.ConsoleLogger.v1"
+progress_bar = true
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+learn_rate = 0.00001
+beta1 = 0.9
+beta2 = 0.999
+L2_is_weight_decay = true
+L2 = 0.01
+grad_clip = 1.0
+use_averages = false
+eps = 0.00000001
+
+[training.score_weights]
+ents_f = 1.0
+ents_p = 0.0
+ents_r = 0.0
+ents_per_type = null
+
+[pretraining]
+
+[initialize]
+vectors = ${paths.vectors}
+init_tok2vec = ${paths.init_tok2vec}
+vocab_data = null
+lookups = null
+before_init = null
+after_init = null
+
+[initialize.components]
+
+[initialize.tokenizer]

ner_model/meta.json

@@ -0,0 +1,64 @@
+{
+  "lang":"en",
+  "name":"pipeline",
+  "version":"0.0.0",
+  "spacy_version":">=3.7.4,<3.8.0",
+  "description":"",
+  "author":"",
+  "email":"",
+  "url":"",
+  "license":"",
+  "spacy_git_version":"bff8725f4",
+  "vectors":{
+    "width":0,
+    "vectors":0,
+    "keys":0,
+    "name":null,
+    "mode":"default"
+  },
+  "labels":{
+    "transformer":[
+
+    ],
+    "ner":[
+      "EXPERIENCE",
+      "SKILL",
+      "TOOL"
+    ]
+  },
+  "pipeline":[
+    "transformer",
+    "ner"
+  ],
+  "components":[
+    "transformer",
+    "ner"
+  ],
+  "disabled":[
+
+  ],
+  "performance":{
+    "ents_f":0.6260185445,
+    "ents_p":0.6503210741,
+    "ents_r":0.6034669556,
+    "ents_per_type":{
+      "EXPERIENCE":{
+        "p":0.8803418803,
+        "r":0.9537037037,
+        "f":0.9155555556
+      },
+      "SKILL":{
+        "p":0.6098981077,
+        "r":0.4515086207,
+        "f":0.5188854489
+      },
+      "TOOL":{
+        "p":0.6174242424,
+        "r":0.6965811966,
+        "f":0.6546184739
+      }
+    },
+    "transformer_loss":150.5976690537,
+    "ner_loss":2037.8725703364
+  }
+}

ner_model/ner/cfg.txt

@@ -0,0 +1,13 @@
+{
+  "moves":null,
+  "update_with_oracle_cut_size":100,
+  "multitasks":[
+
+  ],
+  "min_action_freq":1,
+  "learn_tokens":false,
+  "beam_width":1,
+  "beam_density":0.0,
+  "beam_update_prob":0.0,
+  "incorrect_spans_key":null
+}

ner_model/ner/model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e716df2fd6faab2abcb5035e526c58e9be2515448a6d9b576366febb0881d5e4
+size 301831

ner_model/ner/moves

@@ -0,0 +1 @@
+��moves��{"0":{},"1":{"SKILL":8953,"TOOL":8067,"EXPERIENCE":4012},"2":{"SKILL":8953,"TOOL":8067,"EXPERIENCE":4012},"3":{"SKILL":8953,"TOOL":8067,"EXPERIENCE":4012},"4":{"SKILL":8953,"TOOL":8067,"EXPERIENCE":4012,"":1},"5":{"":1}}�cfg��neg_key�

ner_model/tokenizer

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b014e8bba4958b120af2d0c1c63eabb7c00379f2bacaf10df7c5325efd2ea467
+size 77066

ner_model/transformer/cfg.txt

@@ -0,0 +1,3 @@
+{
+  "max_batch_items":174000
+}

ner_model/transformer/model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50fe4b8034556a21fe59b1d5e0e1d710c77f7b65dd8870b8ffc67c367f00e628
+size 438953871

ner_model/vocab/key2row

@@ -0,0 +1 @@
+�

ner_model/vocab/lookups.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76be8b528d0075f7aae98d6fa57a6d3c83ae480a8469e668d7b0af968995ac71
+size 1

ner_model/vocab/vectors.cfg

@@ -0,0 +1,3 @@
+{
+  "mode":"default"
+}

precompute_insights.py

@@ -0,0 +1,166 @@
+
+import pandas as pd
+import ast
+import re
+import itertools
+from collections import Counter
+import json
+import os
+
+# --- Configuration ---
+INPUT_FILE = 'market_data_with_entities.csv'
+OUTPUT_FILE = 'market_insights.json'
+OUTPUT_DIR = os.path.dirname(os.path.abspath(__file__))
+INPUT_PATH = os.path.join(OUTPUT_DIR, INPUT_FILE)
+OUTPUT_PATH = os.path.join(OUTPUT_DIR, OUTPUT_FILE)
+
+def safe_literal_eval(s):
+    try:
+        if isinstance(s, str) and s.startswith('[') and s.endswith(']'):
+            return ast.literal_eval(s)
+    except (ValueError, SyntaxError):
+        pass
+    return []
+
+def get_top_items(series):
+    """Calculates value counts for an exploded series."""
+    all_items = series.explode().dropna()
+    all_items = all_items.str.lower().str.strip()
+    counts = all_items.value_counts().reset_index()
+    counts.columns = ['item', 'count']
+    return counts
+
+def get_co_occurrence(df, column, top_n=100):
+    """Calculates co-occurrence for a given column."""
+    co_occurrence_df = df[df[column].apply(lambda x: len(set(x))) >= 2].copy()
+    co_occurrence_df[f'{column}_normalized'] = co_occurrence_df[column].apply(
+        lambda items: sorted(list(set([i.lower().strip() for i in items])))
+    )
+    pairs = co_occurrence_df[f'{column}_normalized'].apply(lambda x: list(itertools.combinations(x, 2)))
+    pair_counts = Counter(pairs.explode().dropna())
+    most_common_pairs = pair_counts.most_common(top_n)
+    
+    results = pd.DataFrame(most_common_pairs, columns=['pair', 'count'])
+    results[['item1', 'item2']] = pd.DataFrame(results['pair'].tolist(), index=results.index)
+    return results[['item1', 'item2', 'count']]
+
+def parse_experience(exp_list):
+    """Parses experience strings to find years."""
+    if not isinstance(exp_list, list) or not exp_list:
+        return None
+    for exp_string in exp_list:
+        exp_string = str(exp_string).lower()
+        numbers = re.findall(r'\d+\.?\d*', exp_string)
+        if not numbers:
+            continue
+        val = float(numbers[0])
+        return val / 12.0 if 'month' in exp_string else val
+    return None
+
+def main():
+    print("--- Starting Market Insight Pre-computation ---")
+
+    # --- Load and Prepare Data ---
+    print(f"Loading data from {INPUT_PATH}...")
+    if not os.path.exists(INPUT_PATH):
+        print(f"ERROR: Input file not found at {INPUT_PATH}")
+        return
+        
+    df = pd.read_csv(INPUT_PATH)
+    print(f"Data loaded. Found {len(df)} records.")
+
+    print("Converting stringified lists to actual lists...")
+    for col in ['extracted_skills', 'extracted_tools', 'extracted_experience']:
+        df[col] = df[col].apply(safe_literal_eval)
+
+    # --- Master Data Structure ---
+    insights = {
+        "overall_market": {},
+        "by_role": {}
+    }
+
+    # --- Overall Market Analysis ---
+    print("Analyzing overall market...")
+    # Skills
+    overall_skills = get_top_items(df['extracted_skills'])
+    insights["overall_market"]["top_skills"] = overall_skills.rename(columns={'item': 'skill'}).to_dict(orient='records')
+    
+    # Tools
+    overall_tools = get_top_items(df['extracted_tools'])
+    insights["overall_market"]["top_tools"] = overall_tools.rename(columns={'item': 'tool'}).to_dict(orient='records')
+
+    # Skill Co-occurrence
+    overall_skill_co = get_co_occurrence(df, 'extracted_skills')
+    insights["overall_market"]["skill_co_occurrence"] = overall_skill_co.rename(columns={'item1': 'skill_A', 'item2': 'skill_B'}).to_dict(orient='records')
+
+    # Tool Co-occurrence
+    overall_tool_co = get_co_occurrence(df, 'extracted_tools')
+    insights["overall_market"]["tool_co_occurrence"] = overall_tool_co.rename(columns={'item1': 'tool_A', 'item2': 'tool_B'}).to_dict(orient='records')
+
+    # Experience
+    df['min_years'] = df['extracted_experience'].apply(parse_experience)
+    exp_df = df.dropna(subset=['min_years'])
+    exp_df_filtered = exp_df[exp_df['min_years'] >= 1]
+    
+    exp_dist = exp_df_filtered['min_years'].astype(int).value_counts().sort_index().reset_index()
+    exp_dist.columns = ['year', 'count']
+    insights["overall_market"]["experience_distribution"] = exp_dist.to_dict(orient='records')
+    insights["overall_market"]["average_experience"] = exp_df['min_years'].mean()
+
+    # Job Role Distribution
+    role_counts = df['cmo_role_match'].value_counts().reset_index()
+    role_counts.columns = ['cmo_role_match', 'count']
+    insights["job_role_distribution"] = role_counts.to_dict(orient='records')
+
+    # --- Per Role Analysis ---
+    print("Analyzing data for each role...")
+    roles = df['cmo_role_match'].unique()
+    for role in roles:
+        print(f"- Processing {role}...")
+        role_df = df[df['cmo_role_match'] == role].copy()
+        insights["by_role"][role] = {}
+
+        # Skills
+        role_skills = get_top_items(role_df['extracted_skills'])
+        if not role_skills.empty:
+            role_skills['cmo_role_match'] = role
+        insights["by_role"][role]["top_skills"] = role_skills.rename(columns={'item': 'skill'}).to_dict(orient='records')
+
+        # Tools
+        role_tools = get_top_items(role_df['extracted_tools'])
+        if not role_tools.empty:
+            role_tools['cmo_role_match'] = role
+        insights["by_role"][role]["top_tools"] = role_tools.rename(columns={'item': 'tool'}).to_dict(orient='records')
+
+        # Skill Co-occurrence
+        role_skill_co = get_co_occurrence(role_df, 'extracted_skills')
+        insights["by_role"][role]["skill_co_occurrence"] = role_skill_co.rename(columns={'item1': 'skill_A', 'item2': 'skill_B'}).to_dict(orient='records')
+
+        # Tool Co-occurrence
+        role_tool_co = get_co_occurrence(role_df, 'extracted_tools')
+        insights["by_role"][role]["tool_co_occurrence"] = role_tool_co.rename(columns={'item1': 'tool_A', 'item2': 'tool_B'}).to_dict(orient='records')
+        
+        # Experience
+        role_exp_df = role_df.dropna(subset=['min_years'])
+        role_exp_df_filtered = role_exp_df[role_exp_df['min_years'] >= 1]
+        
+        if not role_exp_df.empty:
+            insights["by_role"][role]["average_experience"] = role_exp_df['min_years'].mean()
+            
+            role_exp_dist = role_exp_df_filtered['min_years'].astype(int).value_counts().sort_index().reset_index()
+            role_exp_dist.columns = ['year', 'count']
+            insights["by_role"][role]["experience_distribution"] = role_exp_dist.to_dict(orient='records')
+        else:
+            insights["by_role"][role]["average_experience"] = None
+            insights["by_role"][role]["experience_distribution"] = []
+
+
+    # --- Save to JSON ---
+    print(f"Saving aggregated insights to {OUTPUT_PATH}...")
+    with open(OUTPUT_PATH, 'w') as f:
+        json.dump(insights, f, indent=4)
+
+    print("--- Pre-computation Finished Successfully! ---")
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,29 @@
+# --- Core Application Dependencies ---
+fastapi==0.117.1
+uvicorn==0.37.0
+pandas==2.3.2
+PyMuPDF==1.26.4
+python-multipart==0.0.20
+beautifulsoup4==4.13.5
+emoji==2.15.0
+
+# --- ML/NLP Dependencies (Pinned for Stability & BERT Model Compatibility) ---
+# Pinned to match the BERT model's training environment
+spacy[transformers]==3.7.4
+
+# Pinned from training to resolve environment conflicts
+click<8.0.0
+typer<0.5.0
+numpy==1.26.4
+catalogue==2.0.10
+
+# Pinned to fix build failures on hosting platforms like Render
+blis==0.7.11
+thinc==8.2.3
+
+# PyTorch - Compatible with Spacy 3.7.4 and Python 3.11/3.12
+torch==2.2.0
+
+# Other dependencies from model training
+scikit-learn==1.4.2
+tqdm==4.66.2

similarity_requirements.txt

@@ -0,0 +1,5 @@
+sentence-transformers==5.1.1
+torch==2.8.0
+pandas==2.3.2
+beautifulsoup4==4.13.5
+emoji==2.15.0

similarity_worker.py

@@ -0,0 +1,114 @@
+
+import sys
+import json
+import pandas as pd
+import torch
+from sentence_transformers import SentenceTransformer, util
+import argparse
+import pathlib
+import re
+from bs4 import BeautifulSoup
+import emoji
+
+# --- Text Cleaning Function (copied from main.py) ---
+def new_refined_text_cleaning(text: str) -> str:
+    """The NEW, improved cleaning function. Keeps technical symbols."""
+    if not isinstance(text, str):
+        return ""
+    text = BeautifulSoup(text, "html.parser").get_text()
+    url_pattern = r'(?:(?:https?|ftp)://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})))(?::\d{2,5})?(?:/\S*)?'
+    text = re.sub(url_pattern, '', text)
+    text = re.sub(r'\S+@\S+\s?', '', text)
+    text = emoji.demojize(text)
+    text = re.sub(r':[a-zA-Z_]+:', '', text)
+    text = text.replace('\\', ' ')
+    text = re.sub(r'[*•]', ' ', text)
+    text = re.sub(r'\{.*?\}', ' ', text)
+    text = re.sub(r'[^a-zA-Z0-9_#+()/\\s.,!?-]', ' ', text)
+    text = re.sub(r'\s+', ' ', text)
+    text = re.sub(r'\s([,.!?-])', r'\1', text)
+    text = text.strip()
+    text = text.lower()
+    return text
+
+def main():
+    """
+    Main function to perform similarity search.
+    Reads resume text from stdin and target role from args.
+    Prints a JSON list of similar jobs to stdout.
+    """
+    try:
+        # 1. Setup paths
+        backend_dir = pathlib.Path(__file__).parent.resolve()
+        
+        # 2. Parse arguments
+        parser = argparse.ArgumentParser()
+        parser.add_argument("--target_role", type=str, required=True)
+        parser.add_argument("--limit", type=int, default=10)
+        args = parser.parse_args()
+        target_role = args.target_role
+
+        # 3. Read resume text from stdin
+        resume_text = sys.stdin.read()
+        if not resume_text:
+            print(json.dumps([]))
+            return
+
+        # 4. Load models and data
+        model = SentenceTransformer(
+            'TechWolf/JobBERT-v2',
+            cache_folder=str(backend_dir / "cached_models"),
+            device="cpu"
+        )
+        job_embeddings = torch.load(backend_dir / "job_embeddings.pt", map_location="cpu")
+        market_data = pd.read_csv(backend_dir / "final_prototype_postings.csv")
+
+        # 5. Filter data based on target_role
+        if target_role != "Overall Market":
+            role_specific_data = market_data[market_data["cmo_role_match"] == target_role]
+            if not role_specific_data.empty:
+                role_indices = role_specific_data.index.tolist()
+                embeddings_tensor = job_embeddings[role_indices]
+                filtered_market_data = role_specific_data
+            else:
+                embeddings_tensor = job_embeddings
+                filtered_market_data = market_data
+        else:
+            embeddings_tensor = job_embeddings
+            filtered_market_data = market_data
+
+        # 6. Perform similarity search
+        cleaned_resume_text = new_refined_text_cleaning(resume_text)
+        resume_embedding = model.encode(
+            cleaned_resume_text, convert_to_tensor=True, device="cpu"
+        )
+        
+        cosine_scores = util.cos_sim(resume_embedding, embeddings_tensor)[0]
+        top_results = torch.topk(cosine_scores, k=min(args.limit, len(filtered_market_data)))
+
+        # 7. Prepare and print results
+        similar_jobs = []
+        for score, idx in zip(top_results[0], top_results[1]):
+            job = filtered_market_data.iloc[idx.item()]
+            similar_jobs.append({
+                "job_title": job["title"],
+                "cmo_role_match": job["cmo_role_match"],
+                "url": job["job_url"],
+                "similarity_score": score.item(),
+            })
+
+        output = {
+            "total_jobs": len(filtered_market_data),
+            "similar_jobs": similar_jobs
+        }
+        print(json.dumps(output))
+
+    except Exception as e:
+        # Log any errors to stderr to be captured by the main process
+        print(f"Similarity worker error: {e}", file=sys.stderr)
+        # Output an empty list to stdout to prevent downstream JSON errors
+        print(json.dumps([]))
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()