sm UI v1.11

UI_input_output.py

@@ -0,0 +1,102 @@
+import pandas as pd
+import numpy as np
+from scipy.sparse import csr_matrix
+import gensim.downloader as api
+from sentence_transformers import SentenceTransformer, util
+from sklearn.feature_extraction.text import CountVectorizer
+from gensim.utils import simple_preprocess
+from concurrent.futures import ThreadPoolExecutor
+import torch
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+from sentence_transformers import SentenceTransformer
+
+
+def read_clean_model(input_df, mdr_df, model_path):
+
+    input_columns_to_check = ["variable", "description"]
+    mdr_columns_to_check = ["name", "definition"]
+
+    # drop missing, drop dups
+    input_df = input_df.dropna(subset=input_columns_to_check)
+    mdr_df = mdr_df.dropna(subset=mdr_columns_to_check)
+    
+    input_df = input_df.drop_duplicates(subset=['variable'])
+    mdr_df = mdr_df.drop_duplicates(subset=['name'])
+    
+    # Remove place holders like TBD
+    
+    placeholder_vars = ['tbd'] # lower case
+    
+    for var in placeholder_vars:
+        for col in mdr_columns_to_check:
+            to_drop = mdr_df[mdr_df[col].str.lower()==var].index
+            mdr_df = mdr_df.drop(to_drop)
+    
+    for var in placeholder_vars:
+        for col in input_columns_to_check:
+            to_drop = input_df[input_df[col].str.lower()==var].index
+            input_df = input_df.drop(to_drop)
+
+    # removed derived
+    derived_data_type_list = ['_DVAL', '_DSUM']
+    
+    mdr_df['derived'] = 'no'
+    mask_derived = (mdr_df['name'].str.contains('|'.join(derived_data_type_list), case=False, na=False))
+    mdr_df.loc[mask_derived, 'derived'] = 'yes'
+    mdr_df = mdr_df[mdr_df['derived'] == 'no']
+    
+    input_df['derived'] = 'no'
+    mask_derived = (input_df['variable'].str.contains('|'.join(derived_data_type_list), case=False, na=False)) 
+    input_df.loc[mask_derived, 'derived'] = 'yes'
+    input_df = input_df[input_df['derived'] == 'no']
+
+    print(input_df.shape)
+    print(mdr_df.shape)
+
+    ## Model learning ...
+    # model = SentenceTransformer('all-MiniLM-L6-v2').to(device)
+    # mannually load the model due to not able to connect to huggingface.co at running time from census laptop :
+    model = SentenceTransformer(f'{model_path}/all-MiniLM-L6-v2').to(device)
+    
+    # Encode the existing descriptions from the MDR dataset into embeddings
+    definitionEmbeddingsB = model.encode(input_df['description'].tolist(), convert_to_tensor=True).to(device)
+    definitionEmbeddingsA = model.encode(mdr_df["definition"].tolist(), convert_to_tensor=True).to(device)
+    
+    # Initialize a list to store results
+    results = []
+    
+    for i, embeddingA in enumerate(definitionEmbeddingsA):
+        similarities = util.pytorch_cos_sim(embeddingA, definitionEmbeddingsB)
+        most_similar_idx = similarities.argmax().item()
+        similarity_score = round(similarities[0][most_similar_idx].item() * 100, 1)
+        result = {
+            "input_name": input_df.iloc[most_similar_idx][["variable"][0]] if ["variable"] and ["variable"][0] in input_df.columns else None,
+            "input_descr": input_df['description'].iloc[most_similar_idx],
+            "mdr_name": mdr_df.iloc[i][["name"][0]] if ["name"] and ["name"][0] in mdr_df.columns else None,
+            "mdr_descr": mdr_df["definition"].iloc[i],
+            "similarity_score": similarity_score
+        }
+    
+        results.append(result)
+    
+    results_df = pd.DataFrame(results)
+    sorted_df = results_df.sort_values(by=['similarity_score'], ascending=[False])
+    # sorted_df['similarity_score'] = sorted_df['similarity_score'].apply(lambda x: str(x) + '%') not converting to string, otherwise slider function doesn't work
+    sorted_df = sorted_df.drop_duplicates(subset=["input_descr", "mdr_descr"]).reset_index(drop=True)
+
+    return sorted_df
+
+
+if __name__ == '__main__':
+
+    print('something')
+    
+    # # Load datasets 
+    # file_path_input = "data/raw/ABS-MOPS Variables - December 11 2024.xlsm"
+    # file_path_mdr = "data/raw/mdr Variables 1.xlsx"
+    
+    # input_df = pd.read_excel(file_path_input, sheet_name="Data Sheet", header=12).rename(columns={'Unnamed: 3':'Legacy Variable'})
+    # mdr_df = pd.read_excel(file_path_mdr)
+
+    # output_df = read_clean_model(input_df, mdr_df)

requirements.txt

@@ -0,0 +1,10 @@
+pandas==2.2.3
+numpy==1.26.4 
+torch==2.6.0
+sentence_transformers==4.1.0 
+scipy==1.13.1 
+scikit-learn==1.6.1 
+openpyxl==3.1.5 
+gradio==5.27.1 
+chardet==5.2.0
+gensim==4.3.3

sm_ui.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+import pandas as pd
+import numpy as np
+import gradio as gr
+
+import os, tempfile
+from scipy.sparse import csr_matrix
+import gensim.downloader as api
+from sentence_transformers import SentenceTransformer, util
+from sklearn.feature_extraction.text import CountVectorizer
+from gensim.utils import simple_preprocess
+from concurrent.futures import ThreadPoolExecutor
+import chardet
+import torch
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+from UI_input_output import read_clean_model
+pd.set_option('display.max_columns', None)
+pd.set_option('display.max_colwidth',500)  
+
+# In[2]:
+
+# where we saved the models. due to not able to connect to huggingface.co, we are running the models offline mode
+model_path = './models'
+
+# Declare global placeholder for dataframes
+input_df = pd.DataFrame()
+mdr_df = pd.DataFrame()
+result_df = pd.DataFrame()
+
+# to load the csv files as input and mdr file; return a list of columns for users to select
+def load_csvs(file_path_input, file_path_mdr):
+    global input_df, mdr_df
+    
+    file_missing_msg =  """<p>  
+        <span style="font-weight:bold; color:red; font-size:20px;">At least one of the files is missing from upload</span>.
+        </p>"""
+        
+    if file_path_input is None or file_path_mdr is None:
+        return None, None, file_missing_msg
+
+    # Read both files into DataFrames with their own encoding
+    with open(file_path_input, 'rb') as f:
+        result = chardet.detect(f.read())
+    encoding_input = result['encoding']
+    
+    with open(file_path_mdr, 'rb') as f:
+        result = chardet.detect(f.read())
+    encoding_mdr = result['encoding']
+
+    input_df = pd.read_csv(file_path_input, encoding=encoding_input)
+    mdr_df = pd.read_csv(file_path_mdr, encoding=encoding_mdr)
+    
+    return gr.update(choices=input_df.columns.tolist(), value=None), gr.update(choices=input_df.columns.tolist(), value=None), gr.update(choices=mdr_df.columns.tolist(), value=None), gr.update(choices=mdr_df.columns.tolist(), value=None), None
+
+
+# Once files are ready, users to select the columns from each, to run through the models to create a final dataframe with similarity scores
+## will prompt users to select at least the key columns from each file. because the models will use the key columns
+def process_two_csvs(input_cols1, input_cols2, mdr_cols1, mdr_cols2):
+    global input_df, mdr_df, result_df
+
+    # # the key columns from input and mdr.  we will rename whatever users selected to these names later
+    # input_key_columns: 'variable','description'
+    # mdr_key_columns: 'name','definition'
+
+    select_2_msg =  """<p>Please select:  
+        <span style="font-weight:bold; color:red; font-size:20px;">1 column (only 1)</span>
+        for each dropdown
+        </p>"""
+        
+    if not input_cols1 or not input_cols2 or not mdr_cols1 or not mdr_cols2:
+        return None, None, select_2_msg
+        
+    else:
+        if len(input_cols1)>1 or len(input_cols2)>1 or len(mdr_cols1)>1 or len(mdr_cols2)>1:
+            return None, None, select_2_msg
+
+    try:
+        
+        # print('\n input selected columns:')
+        # print(input_cols1[0], type(input_cols1[0]))
+        # print(input_cols2[0], type(input_cols2[0]))
+        # print('\n mdr selected columns:')
+        # print(mdr_cols1[0], type(mdr_cols1[0]))
+        # print(mdr_cols2[0], type(mdr_cols2[0]))
+    
+        # print(input_df[[input_cols1[0], input_cols2[0]]].head())
+        # print(mdr_df[[mdr_cols1[0], mdr_cols2[0]]].head())
+
+        # input col1 is the equivalent of variable, input col2 is equivalent of description
+        # mdr col1 is the equivalent of name, mdr col2 is equivalent of definition
+        input_df = input_df[[input_cols1[0], input_cols2[0]]].rename(columns={input_cols1[0]:'variable',input_cols2[0]:'description'})
+        mdr_df = mdr_df[[mdr_cols1[0], mdr_cols2[0]]].rename(columns={mdr_cols1[0]:'name',mdr_cols2[0]:'definition'})
+        # print('\n')
+        # print(input_df.head())
+        # print(mdr_df.head())
+        
+        print('\n')
+        print(input_df.shape)
+        print(mdr_df.shape)
+    
+        # clean and run through models
+        result_df = read_clean_model(input_df, mdr_df, model_path)
+
+        # result_df = pd.read_csv(r"C:\Users\hamme040\Documents\work\ECON\sematch\data_for_testing\tmpmwtn5lgn_abs_num.csv") # for demo, faster display
+    
+        # # flag where one input variable matched multiple name from MDR
+        result_df['Flag'] = (result_df['input_name'].duplicated(keep=False) | result_df['mdr_name'].duplicated(keep=False)) \
+                              .map(lambda x: '🔴 Duplicate' if x else '✅ Unique')
+    
+        # sort for displaying table
+        result_df = result_df.sort_values(by=['input_name','mdr_name'])
+        
+        # Use the system temp directory to save result dataframe as a CSV temp file
+        temp_dir = tempfile.gettempdir()
+        custom_filename = "cos_similarity.csv"
+        file_path = os.path.join(temp_dir, custom_filename)
+        result_df.to_csv(file_path, index=False)
+        
+        return result_df, file_path, None
+        
+    except Exception as e:
+        return None, None, f"Error during processing: {e}"
+
+
+# silder: to filter the similarity score based on a threshold
+def filter_data(threshold):
+    global result_df
+    df = result_df.copy()
+    df["similarity_score"] = df["similarity_score"].astype(float)
+    filtered_df = df[df["similarity_score"] >= threshold]
+    return filtered_df
+
+
+######################################## APP ###########################################
+with gr.Blocks() as demo:
+    
+    ################## app layout ####################
+    gr.Markdown("# Semantic Matching between Input and MDR")
+    
+    with gr.Row():
+        
+        # box to load the input file
+        l1 = gr.Markdown(
+        """
+        Upload Input file
+        (file must contains at least two columns (not case sensitive): **variable** and **description**)
+        """)
+        csv_input = gr.File(label="Upload Input File",
+                    file_types=[".csv"])
+        
+        # box to load the MDR file
+        l2 = gr.Markdown(
+        """
+        Upload MDR file
+        (file must contains at least two columns (not case sensitive): **name** and **definition**)
+        """)
+        csv_mdr = gr.File(label="Upload MDR File", 
+                    file_types=[".csv"])
+        
+    errmsg_load = gr.HTML(label="Error message") # to display error message for load csvs
+
+    # button to click and initiate loading the files
+    load_button = gr.Button("Load CSVs")
+
+    # dropdowns for users to select the columns from their uploaded files, for further processing
+    with gr.Row():
+        
+        input_cols_dropdown1 = gr.Dropdown(label="Select VAR column from Input", choices=[], multiselect=True, interactive=True) # mutiselect=True automatically return a list
+        input_cols_dropdown2 = gr.Dropdown(label="Select Description column from Input", choices=[], multiselect=True, interactive=True) # mutiselect=True automatically return a list
+        
+        mdr_cols_dropdown1 = gr.Dropdown(label="Select VAR column from MDR", choices=[], multiselect=True, interactive=True)
+        mdr_cols_dropdown2 = gr.Dropdown(label="Select Description column from MDR", choices=[], multiselect=True, interactive=True)
+
+
+    errmsg_proc = gr.HTML(label="Error message") # to display error message for process csvs
+
+    # button to click and initiate processing the files, i.e. clean the data sets, run through models
+    process_btn = gr.Button("Process Files")
+    
+    # table that displays the final result dataframe
+    output_table = gr.Dataframe(
+        interactive=True,   # True Enables sorting, and editing;  need False in order to do the filtering
+        row_count=10,
+        column_widths="auto"
+    )
+
+    # the link that enables download of this result table
+    download_link = gr.File(label="⬇️ Download Result CSV")
+    
+    # slider, for filtering similarity score as a threshold
+    slider = gr.Slider(minimum=0, maximum=100, step=1, value=70, label="Similarity Score")
+
+
+    ################## app functions ####################
+    # link load button to its function and input output
+    load_button.click(
+        fn=load_csvs,
+        inputs=[csv_input, csv_mdr],
+        outputs=[input_cols_dropdown1, input_cols_dropdown2, mdr_cols_dropdown1, mdr_cols_dropdown2, errmsg_load]
+    )
+
+    # link process button to its function and input output
+    process_btn.click(
+        fn=process_two_csvs,
+        inputs=[input_cols_dropdown1, input_cols_dropdown2, mdr_cols_dropdown1, mdr_cols_dropdown2],
+        outputs=[output_table, download_link, errmsg_proc]
+    )
+
+    # link slider to its function and input output
+    slider.change(fn=filter_data, inputs=slider, outputs=output_table)
+
+
+demo.launch()
+
+
+
+# watchmedo auto-restart --patterns="*.py" -- python sm_ui.py