Update app.py

app.py

@@ -6,8 +6,6 @@ import re
 import nltk
 from nltk.corpus import words, stopwords
 import urllib.parse as _url
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
 from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 from nltk.stem import PorterStemmer
 import gradio as gr
@@ -26,7 +24,7 @@ for package in ['words', 'stopwords', 'averaged_perceptron_tagger', 'punkt']:
 STOPWORDS = set(stopwords.words('english'))
 stemmer = PorterStemmer()
 
-# --- EXPANDED: Skill Whitelist with more business, finance, and consulting terms ---
+# --- Expanded Skill Whitelist ---
 SKILL_WHITELIST = {
     # Technical & Data
     'python', 'java', 'c++', 'javascript', 'typescript', 'sql', 'nosql', 'html', 'css', 'react', 'angular', 'vue',
@@ -43,6 +41,7 @@ SKILL_WHITELIST = {
     'strategy', 'stakeholder management', 'risk management', 'compliance', 'aml', 'kyc', 'reinsurance', 'finance',
     'financial modeling', 'financial analysis', 'due diligence', 'sourcing', 'procurement', 'negotiation', 'supply chain',
     'business analysis', 'business intelligence', 'presentations', 'public speaking', 'time management', 'critical thinking',
+    'design thinking', 'innovation', 'adaptability', 'supervisory', 'pmp', 'cpsm', 'cips',
     # Soft & Other
     'leadership', 'stakeholder communication', 'client communication', 'teamwork', 'collaboration', 'problem solving',
     'ui/ux design', 'figma', 'sketch', 'adobe xd', 'graphic design', 'autocad', 'solidworks', 'sales', 'marketing',
@@ -137,7 +136,6 @@ def find_job_matches(original_user_query: str, expanded_user_query: str, top_k:
     final_results_df = final_results_df.set_index('job_id', drop=False).rename(columns={'job_id': 'Job ID'})
     return final_results_df
 
-# --- REWRITTEN: Skill scoring function using semantic similarity ---
 def score_jobs_by_skills(user_skills: list[str], df_to_rank: pd.DataFrame) -> pd.DataFrame:
     if df_to_rank is None or df_to_rank.empty or not user_skills:
         return df_to_rank.sort_values(by='Similarity Score', ascending=False) if df_to_rank is not None else pd.DataFrame()
@@ -146,21 +144,17 @@ def score_jobs_by_skills(user_skills: list[str], df_to_rank: pd.DataFrame) -> pd
     if 'Skills' not in ranked_df.columns:
         return ranked_df.sort_values(by='Similarity Score', ascending=False)
 
-    # 1. Encode all user skills and all unique job skills across the dataframe ONCE for efficiency
     user_skill_embeddings = model.encode(user_skills, convert_to_tensor=True)
     all_job_skills = sorted(list(set(skill for skills_list in ranked_df['Skills'] if skills_list for skill in skills_list)))
     
-    if not all_job_skills: # No skills to compare against
+    if not all_job_skills:
         ranked_df['Skill Match Score'] = 0.0
         return ranked_df
 
     job_skill_embeddings = model.encode(all_job_skills, convert_to_tensor=True)
-    
-    # 2. Calculate the similarity matrix between every user skill and every job skill
     similarity_matrix = util.cos_sim(user_skill_embeddings, job_skill_embeddings)
 
-    # 3. Define the new scoring function
-    def calculate_semantic_match(row, threshold=0.55):
+    def calculate_semantic_match(row, threshold=0.48): # Lowered threshold for more sensitivity
         job_skills_list = row.get('Skills', [])
         if not job_skills_list:
             return [], 0, 0.0
@@ -168,9 +162,7 @@ def score_jobs_by_skills(user_skills: list[str], df_to_rank: pd.DataFrame) -> pd
         matched_skills_in_job = set()
         for job_skill in job_skills_list:
             try:
-                # Find which column in the matrix corresponds to the current job skill
                 job_skill_idx = all_job_skills.index(job_skill)
-                # Check if ANY of the user's skills meet the similarity threshold for this job skill
                 if torch.any(similarity_matrix[:, job_skill_idx] > threshold):
                     matched_skills_in_job.add(job_skill)
             except (ValueError, IndexError):
@@ -180,14 +172,10 @@ def score_jobs_by_skills(user_skills: list[str], df_to_rank: pd.DataFrame) -> pd
         match_score = len(matched_skills_in_job) / total_required if total_required > 0 else 0.0
         return list(matched_skills_in_job), len(matched_skills_in_job), match_score
 
-    # 4. Apply the new scoring function to each row
     results = ranked_df.apply(lambda row: calculate_semantic_match(row), axis=1, result_type='expand')
     ranked_df[['Skill Matches', 'Skill Match Count', 'Skill Match Score']] = results
-    
-    # 5. Sort by the new graded score
     ranked_df = ranked_df.sort_values(by=['Skill Match Score', 'Similarity Score'], ascending=[False, False]).reset_index(drop=True)
     return ranked_df.set_index('Job ID', drop=False).rename_axis(None)
-# ----------------------------------------------------------------------
 
 def initialize_data_and_model():
     global original_df, combined_df, model, combined_job_embeddings, original_job_title_embeddings
@@ -219,7 +207,7 @@ Text: "{text}"
 Extracted Skills:
 """
             try:
-                response = LLM_PIPELINE(prompt, max_new_tokens=100, do_sample=False, temperature=0.1)
+                response = LLM_PIPELINE(prompt, max_new_tokens=150, do_sample=False, temperature=0.1)
                 generated_text = response[0]['generated_text']
                 skills_part = generated_text.split("Extracted Skills:")[-1].strip()
                 skills = [skill.strip() for skill in skills_part.split(',') if skill.strip()]
@@ -238,15 +226,26 @@ Extracted Skills:
             for subtree in chunked_text.subtrees():
                 if subtree.label() == 'NP':
                     phrase = " ".join(word for word, tag in subtree.leaves())
-                    normalized_phrase = _norm_skill_token(phrase)
-                    if normalized_phrase in SKILL_WHITELIST:
-                        potential_skills.add(normalized_phrase)
+                    if _norm_skill_token(phrase) in SKILL_WHITELIST:
+                        potential_skills.add(_norm_skill_token(phrase))
             return sorted(list(potential_skills))
+            
+        # NEW: Third extraction method for maximum coverage
+        def extract_skills_direct_scan(text: str) -> list[str]:
+            if not isinstance(text, str): return []
+            found_skills = set()
+            for skill in SKILL_WHITELIST:
+                # Use word boundaries to avoid matching substrings like 'art' in 'startup'
+                if re.search(r'\b' + re.escape(skill) + r'\b', text, re.IGNORECASE):
+                    found_skills.add(skill)
+            return list(found_skills)
 
         def extract_skills_hybrid(text: str) -> list[str]:
             llm_skills = extract_skills_llm(text)
             nltk_skills = extract_skills_nltk(text)
-            combined_skills = set(llm_skills) | set(nltk_skills)
+            direct_skills = extract_skills_direct_scan(text)
+            # Combine all sources and return a unique, sorted list
+            combined_skills = set(llm_skills) | set(nltk_skills) | set(direct_skills)
             return sorted(list(combined_skills))
 
         def create_text_for_skills(row):
@@ -261,8 +260,7 @@ Extracted Skills:
         original_df.to_parquet(PROCESSED_DATA_PATH)
 
     original_df['job_id'] = original_df.index
-    def create_full_text(row):
-        return " ".join([str(s) for s in [row.get("Job title"), row.get("Company"), row.get("Duties"), row.get("qualifications"), row.get("Description")]])
+    def create_full_text(row): return " ".join([str(s) for s in [row.get("Job title"), row.get("Company"), row.get("Duties"), row.get("qualifications"), row.get("Description")]])
     original_df["full_text"] = original_df.apply(create_full_text, axis=1)
     
     ds = datasets.load_dataset("its-zion-18/Jobs-tabular-dataset")
@@ -355,7 +353,16 @@ def on_select_job(job_id, skills_text):
     if not job_skills:
         learning_plan_html = "<p><i>No specific skills could be extracted for this job.</i></p>"
         return job_details_markdown, duties, qualifications, description, learning_plan_html, gr.Accordion(visible=True), [], 0, gr.Button(visible=False)
-    all_missing_skills = sorted([s for s in job_skills if not any(util.cos_sim(model.encode(ut), model.encode(s))[0][0] > 0.55 for ut in user_skills)], key=lambda x: x.lower())
+
+    all_missing_skills = job_skills
+    if user_skills:
+        user_skill_embeddings = model.encode(user_skills, convert_to_tensor=True)
+        job_skill_embeddings = model.encode(job_skills, convert_to_tensor=True)
+        similarity_matrix = util.cos_sim(user_skill_embeddings, job_skill_embeddings)
+        
+        matched_job_skills_mask = torch.any(similarity_matrix > 0.48, dim=0)
+        all_missing_skills = [skill for i, skill in enumerate(job_skills) if not matched_job_skills_mask[i]]
+
     if not all_missing_skills:
         learning_plan_html = "<h4 style='color:green;'>🎉 You have all the required skills!</h4>"
         return job_details_markdown, duties, qualifications, description, learning_plan_html, gr.Accordion(visible=True), [], 0, gr.Button(visible=False)
@@ -365,16 +372,16 @@ def on_select_job(job_id, skills_text):
         job_details_markdown += f"\n**Your skill match:** {score_val:.1%}"
         headline = "<b>Great fit!</b>" if score_val >= 0.8 else "<b>Good progress!</b>" if score_val >= 0.5 else "<b>Solid starting point.</b>"
         learning_plan_html = f"<h4>{headline} Focus on these skills to improve your match:</h4>"
-        skills_to_display = all_missing_skills[:5]
+        skills_to_display = sorted(all_missing_skills)[:5]
         items_html = [f"<li><b>{ms}</b><br>• Learn: {_course_links_for(ms)}</li>" for ms in skills_to_display]
         learning_plan_html += f"<ul style='list-style-type: none; padding-left: 0;'>{''.join(items_html)}</ul>"
         return job_details_markdown, duties, qualifications, description, learning_plan_html, gr.Accordion(visible=True), [], 0, gr.Button(visible=False)
     else:
         headline = "<h4>To be a good fit for this role, you'll need to learn these skills:</h4>"
-        skills_to_display = job_skills[:5]
+        skills_to_display = sorted(job_skills)[:5]
         items_html = [f"<li><b>{ms}</b><br>• Learn: {_course_links_for(ms)}</li>" for ms in skills_to_display]
         learning_plan_html = f"{headline}<ul style='list-style-type: none; padding-left: 0;'>{''.join(items_html)}</ul>"
-        full_skill_list_for_state = job_skills
+        full_skill_list_for_state = sorted(job_skills)
         new_offset = len(skills_to_display)
         should_button_be_visible = len(full_skill_list_for_state) > 5
         return job_details_markdown, duties, qualifications, description, learning_plan_html, gr.Accordion(visible=True), full_skill_list_for_state, new_offset, gr.Button(visible=should_button_be_visible)
@@ -424,14 +431,4 @@ with gr.Blocks(theme=gr.themes.Soft()) as ui:
             with gr.TabItem("Duties"): duties_markdown = gr.Markdown()
             with gr.TabItem("Qualifications"): qualifications_markdown = gr.Markdown()
             with gr.TabItem("Full Description"): description_markdown = gr.Markdown()
-        learning_plan_output = gr.HTML(label="Learning Plan")
-        load_more_btn = gr.Button("Load More Skills", visible=False)
-    search_btn.click(fn=find_matches_and_rank_with_check, inputs=[dream_text, topk_slider, skills_text], outputs=[status_text, initial_matches_state, df_output, job_selector, details_accordion, spelling_alert, spelling_row])
-    search_anyway_btn.click(fn=find_matches_and_rank_anyway, inputs=[dream_text, topk_slider, skills_text], outputs=[status_text, initial_matches_state, df_output, job_selector, details_accordion, spelling_alert, spelling_row])
-    retype_btn.click(lambda: ("Status: Ready for you to retype.", None, pd.DataFrame(), gr.Dropdown(visible=False), gr.Accordion(visible=False), gr.Markdown(visible=False), gr.Row(visible=False)), outputs=[status_text, initial_matches_state, df_output, job_selector, details_accordion, spelling_alert, spelling_row])
-    reset_btn.click(fn=on_reset, outputs=[dream_text, topk_slider, skills_text, df_output, initial_matches_state, job_selector, details_accordion, status_text, job_details_markdown, duties_markdown, qualifications_markdown, description_markdown, spelling_alert, spelling_row, missing_skills_state, skills_offset_state, load_more_btn], queue=False)
-    rerank_btn.click(fn=rerank_current_results, inputs=[initial_matches_state, skills_text, topk_slider], outputs=[status_text, df_output, job_selector])
-    job_selector.change(fn=on_select_job, inputs=[job_selector, skills_text], outputs=[job_details_markdown, duties_markdown, qualifications_markdown, description_markdown, learning_plan_output, details_accordion, missing_skills_state, skills_offset_state, load_more_btn])
-    load_more_btn.click(fn=load_more_skills, inputs=[missing_skills_state, skills_offset_state], outputs=[learning_plan_output, skills_offset_state, load_more_btn])
-
-ui.launch()
+        learning_plan_output = gr.HTML(label="Learning