Update HF_processor.py

HF_processor.py

@@ -1,6 +1,6 @@
 import pandas as pd
 import numpy as np
-import re
+import json
 from fuzzywuzzy import process
 
 class FMEADataPipeline:
@@ -22,11 +22,13 @@ class FMEADataPipeline:
         self.new_fmea = None
     
     def read_catalog_profile(self):
+
         self.cp = pd.read_json(self.catalog_profile['cp'],orient='split')
         self.object_part = pd.read_json(self.catalog_profile['object part'],orient='split')
         self.symptom = pd.read_json(self.catalog_profile['symptom'],orient='split')
         self.damage = pd.read_json(self.catalog_profile['damage'],orient='split')
-        self.cause = pd.read_json(self.catalog_profile['cause'],orient='split')
+        self.cause = pd.read_json(self.catalog_profile['cause'],orient='split') 
+
 
     def build_connector(self):
         self.code_group = self.cp[self.cp['Catalog profile']==self.catalog_code][['Catalog','Code group']]
@@ -36,10 +38,6 @@ class FMEADataPipeline:
         self.code_group = pd.merge(self.code_group,self.fmea_code,how='left',on='Catalog')
         self.fmea['Catalog Profile (SAP)'] = self.catalog_code
 
-    def column_dropper(self):
-        cols = [6,7,8,9,10,11,17,18,19,20,21,22,23]
-        self.fmea.drop(self.fmea.columns[cols],axis=1,inplace=True)
-        self.fmea = self.fmea.iloc[1:]
 
     def column_matcher(self):
         for code,sap in zip(self.fmea_code['fmea code'],[self.object_part,self.symptom,self.damage,self.cause]):
@@ -66,8 +64,7 @@ class FMEADataPipeline:
 
             self.fmea[f"{name}_description"] = self.fmea[name].apply(
                 lambda x: mapping_dict_short_text.get(x) if x in mapping_dict_short_text else None)
-
-        print('Column Matcher Done')     
+        
             
     def column_arranger(self):
         catalog_profile = self.fmea.pop('Catalog Profile (SAP)')
@@ -91,17 +88,11 @@ class FMEADataPipeline:
         self.fmea.insert(12,cause_5.name,cause_5)
         self.fmea.insert(13,cause_5_desc.name,cause_5_desc)
 
-        print('Column Arranger Done')
-
 
     def process_and_split_excel(self):
         print('Got into process_and_split_excel')
         new_rows = []
         columns = ['Proposed Task', 'Task Type', 'Frequency', 'Action Party', 'TA (Y/N)']
-        clean_columns = ['Frequency', 'Action Party', 'TA (Y/N)']  # Columns to clean bullet points
-
-        # Enhanced regex for bullet points
-        bullet_pattern = r'^\s*[\da-zA-Z]+[)\.\-•]?\s*'
 
         for _, row in self.fmea.iterrows():
             cell_value = row[columns[0]]
@@ -114,25 +105,14 @@ class FMEADataPipeline:
                     for column in columns:
                         column_values = row[column].split('\n') if isinstance(row[column], str) else [row[column]]
                         if idx < len(column_values):
-                            new_value = column_values[idx]
-                            
-                            # Remove bullet points if column is in clean_columns
-                            if column in clean_columns:
-                                new_value = re.sub(bullet_pattern, '', new_value).strip()
-
-                            new_row[column] = new_value
+                            new_row[column] = column_values[idx]
                         else:
                             new_row[column] = np.nan  # Fill with NaN if the split is not aligned
                     new_rows.append(new_row)
             else:
-                # Clean up bullet points for non-split rows in clean_columns
-                for column in clean_columns:
-                    if isinstance(row[column], str):
-                        row[column] = re.sub(bullet_pattern, '', row[column]).strip()
                 new_rows.append(row)
 
         self.new_fmea = pd.DataFrame(new_rows)
         self.new_fmea.to_excel('processed_excel.xlsx', index=False)
         
-        print('process_and_split_excel done')
         return self.new_fmea