Update utility/utils.py

utility/utils.py

@@ -402,7 +402,34 @@ def process_extracted_text(extracted_text):
 
 # Process the model output for parsed result
 def process_resume_data(LLMdata,cont_data,extracted_text):
+
+    # Removing duplicate emails
+    unique_emails = []
+    for email in cont_data['emails']:
+        if not any(email.lower() == existing_email.lower() for existing_email in LLMdata['Email']):
+            unique_emails.append(email)
+    
+    # Removing duplicate links (case insensitive)
+    unique_links = []
+    for link in cont_data['links_RE']:
+        if not any(link.lower() == existing_link.lower() for existing_link in LLMdata['Link']):
+            unique_links.append(link)
     
+    # Removing duplicate phone numbers
+    normalized_contact = [num[-10:] for num in LLMdata['Contact']]
+    unique_numbers = []
+    for num in cont_data['phone_numbers']:
+        if num[-10:] not in normalized_contact:
+            unique_numbers.append(num)
+       
+    # Add unique emails, links, and phone numbers to the original LLMdata
+    LLMdata['Email'] += unique_emails
+    LLMdata['Link'] += unique_links
+    LLMdata['Contact'] += unique_numbers
+
+    # Apply the function to the data
+    LLMdata=remove_duplicates_case_insensitive(LLMdata)
+        
     # Initialize the processed data dictionary
     processed_data = {            
             "name": [],
@@ -415,15 +442,32 @@ def process_resume_data(LLMdata,cont_data,extracted_text):
             "extracted_text": extracted_text
             }
     #LLM
+    
     processed_data['name'].extend(LLMdata.get('Name', []))
-    processed_data['contact_number'].extend(LLMdata.get('Contact', []))
+    #processed_data['contact_number'].extend(LLMdata.get('Contact', []))
     processed_data['Designation'].extend(LLMdata.get('Designation', []))
-    processed_data['email'].extend(LLMdata.get("Email", []))
+    #processed_data['email'].extend(LLMdata.get("Email", []))
     processed_data['Location'].extend(LLMdata.get('Address', []))
-    processed_data['Link'].extend(LLMdata.get('Link', []))
+    #processed_data['Link'].extend(LLMdata.get('Link', []))
     processed_data['Company'].extend(LLMdata.get('Company', []))
+    
     #Contact
-    processed_data['email'].extend(cont_data.get("emails", [])) 
-    processed_data['contact_number'].extend(cont_data.get("phone_numbers", []))
-    processed_data['Link'].extend(cont_data.get("links_RE", []))
-    return processed_data
+    #processed_data['email'].extend(cont_data.get("emails", [])) 
+    #processed_data['contact_number'].extend(cont_data.get("phone_numbers", []))
+    #processed_data['Link'].extend(cont_data.get("links_RE", []))
+    
+    #New_merge_data
+    processed_data['email'].extend(LLMdata['Email']) 
+    processed_data['contact_number'].extend(LLMdata['Contact'])
+    processed_data['Link'].extend(LLMdata['Link'])
+
+    #to remove not found fields
+    # List of keys to check for 'Not found'
+    keys_to_check = ["name", "contact_number", "Designation", "email", "Location", "Link", "Company"]
+
+    # Replace 'Not found' with an empty list for each key
+    for key in keys_to_check:
+        if processed_data[key] == ['Not found'] or processed_data[key] == ['not found'] or processed_data[key] == []:
+            del processed_data[key]
+    
+    return processed_data