Initial commit

functions/extract_function.py

@@ -0,0 +1,101 @@
+import os
+import yaml
+import requests
+import pandas as pd
+
+def internal_data(type):
+    """
+    Extract internal data from either catalog or query.
+
+    :param type: str, 'catalog' or 'query'
+
+    :return: pandas.DataFrame, dataframe containing product name and category name
+    """
+    if type == 'catalog':
+        dfs = []
+        for file in os.listdir('catalog'):
+            if file.endswith('.xlsx'):
+                df = pd.read_excel('catalog/' + file)
+                dfs.append(df)
+        catalog = pd.concat(dfs, ignore_index=True)
+        return catalog
+    
+    elif type == 'query':
+        dfs = []
+        for file in os.listdir('query'):
+            if file.endswith('.xlsx'):
+                df = pd.read_excel('query/' + file)
+                dfs.append(df)
+        query = pd.concat(dfs, ignore_index=True)
+        return query
+    
+    else:
+        return 'Error: type must be either catalog or query'
+
+def registered_fertilizer_data():
+    """
+    Scrape registered fertilizer data in Ministry of Agriculture website.
+
+    :param type: str, 'organik' or 'anorganik'
+
+    :return: pandas.DataFrame, dataframe containing registered fertilizer data
+    """
+    # check if the "external" folder is empty
+    if os.listdir('external') == []:
+        print('External folder is empty. Extracting data from Ministry of Agriculture website...')
+        print('Extracting Organic Fertilizer Data...')
+        dfs1 = []
+        # Scrape every table in every page: Organic
+        i = 1
+        while True:
+            url = yaml.load(open('config.yaml'), Loader=yaml.FullLoader)['scraping_url']['organik'][0] + str(i)
+            result = requests.get(url).content
+            try:
+                df = pd.read_html(result)[5].iloc[2:-1, [2, 3, 6]].rename(columns={2: 'Merek', 3: 'Jenis', 6: 'Nomor Pendaftaran'})
+                df['Page Number'] = i
+                dfs1.append(df)
+                i += 1
+            except IndexError:
+                break
+
+        registered_organic_fertilizers = pd.concat(dfs1, ignore_index=True).dropna()
+        
+        print('Extracting Inorganic Fertilizer Data...')
+        dfs2 = []
+        # Scrape every table in every page: Inorganic
+        i = 1
+        while True:
+            url = yaml.load(open('config.yaml'), Loader=yaml.FullLoader)['scraping_url']['anorganik'][0] + str(i)
+            result = requests.get(url).content
+            try:
+                df = pd.read_html(result)[5].iloc[2:-1, 5:8].rename(columns={5: 'Merek', 6: 'Jenis', 7: 'Nomor Pendaftaran'})
+                df['Page Number'] = i
+                dfs2.append(df)
+                i += 1
+            except IndexError:
+                break
+
+        registered_inorganic_fertilizers = pd.concat(dfs2, ignore_index=True).dropna()
+
+        registered_fertilizers = pd.concat([registered_organic_fertilizers, registered_inorganic_fertilizers], ignore_index=True)
+        registered_fertilizers['Nama Lengkap'] = registered_fertilizers['Jenis'] + ' ' + registered_fertilizers['Merek']
+        return registered_fertilizers
+
+    else :
+        return pd.read_csv('external/registered_fertilizers.csv')
+    
+def scrape_result():
+    """
+    Extract scraped result data.
+
+    :return: pandas.DataFrame, dataframe containing scraped result data
+    """
+    dfs = []
+
+    for filename in os.listdir('scrape_result'):
+        df = pd.read_csv('scrape_result/'+filename)
+        dfs.append(df)
+
+    # combine
+    final_df = pd.concat(dfs, ignore_index=True)
+    return final_df

functions/modelling_function.py

@@ -0,0 +1,191 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+import yaml
+import os
+import warnings
+from rapidfuzz import fuzz, utils
+from simpletransformers.classification import ClassificationModel, ClassificationArgs
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import confusion_matrix, classification_report
+from scipy.special import softmax
+
+
+def generate_training_data(df, text_column, label_column, external_table = None, external_column = None, add_external_table=False, sampling=True):
+    """
+    This function generates training data for the model.
+
+    :param df: pandas.DataFrame, dataframe containing product name and category name
+    :param text_column: str, column name containing product name
+    :param label_column: str, column name containing category name
+    :param external_table: pandas.DataFrame, dataframe containing product name and category name
+    :param external_column: str, column name containing product name
+    :param add_external_table: bool, whether to add external table or not
+    :param sampling: bool, whether to do sampling or not
+
+    :return: pandas.DataFrame, dataframe containing product name and category name
+    """
+    if os.listdir('training') == []:
+        print('Training folder is empty. Generating training data...')
+        units = yaml.load(open('config.yaml'), Loader=yaml.FullLoader)['excluded_words']
+        
+        df['category_name'] = df[label_column].apply(lambda x: 'Fertilizer - High' if isinstance(x, list) and len(x) == 1 and 'Garden Soil & Fertilizers' in x else 'Pesticide - High' if isinstance(x, list) and len(x) == 1 and 'Weeds & Pest Control' in x else 'Fertilizer - Medium' if isinstance(x, list) and len(x) > 1 and 'Garden Soil & Fertilizers' in x else 'Pesticide - Medium' if isinstance(x, list) and len(x) > 1 and 'Weeds & Pest Control' in x else 'Others')
+        df = df[[text_column, 'category_name']]
+        
+        # take only where category_name is Ferilizer - High or Pesticide - High or Others
+        df = df[df['category_name'].isin(['Fertilizer - High', 'Pesticide - High', 'Others'])]
+        # exclude product name that contains units AND category_name is Others
+        df = df[~(df[text_column].str.contains('|'.join(units)) & (df['category_name'] == 'Others'))]
+
+        if add_external_table:
+            external_table['category_name'] = 'Fertilizer - High'
+            external_table = external_table[[external_column, 'category_name']]
+            external_table.columns = [text_column, 'category_name']
+
+            training_df = pd.concat([external_table, df])
+            training_df.columns = ['product_name','category_name']
+
+            training_df['category_name'] = training_df['category_name'].apply(lambda x: 0 if x == 'Fertilizer - High' else 1 if x == 'Pesticide - High' else 2)
+            if sampling:
+                return pd.concat([training_df[training_df['category_name'] == 0].sample(n=1250), training_df[training_df['category_name'] == 1].sample(n=1250), training_df[training_df['category_name'] == 2].sample(n=1500)])
+            else:
+                return training_df
+        else:
+            return df
+    else:
+        training_df = pd.read_csv('training/training_data.csv')
+        return training_df
+
+def category_reassign(row, reference_df, checked_category, threshold=70):
+    """
+    This function reassigns the category name of a product based on the similarity score between the product name and the reference dataframe.
+
+    :param row: pandas.Series, row of dataframe
+    :param reference_df: pandas.DataFrame, dataframe containing product name and category name
+    :param checked_category: str, category name to be checked
+    :param threshold: int, threshold for similarity score
+
+    :return: str, category name
+    """
+    if row['category_name'] == checked_category:
+        for i in range(len(reference_df)):
+            row2 = reference_df.iloc[i]
+            if row2['category_name'] != checked_category:
+                if fuzz.ratio(row['product_name'], row2['product_name'], processor= utils.default_process) >= threshold:
+                    return row2['category_name']
+        return checked_category
+    else:
+        return row['category_name']
+    
+def train_model(df, stratify=True, model_type='bert', use_existing_model=False, model_name=None):
+    """
+    This function trains the model using the configuration in config.yaml
+
+    :param df: pandas.DataFrame, dataframe containing product name and category name
+    :param stratify: bool, whether to do stratified sampling or not
+    :param model_type: str, type of model to use
+    :param use_existing_model: bool, whether to use existing model or not
+    :param model_name: str, name of existing model
+
+    :return: simpletransformers.classification.ClassificationModel, model
+    :return: numpy.ndarray, predictions
+    :return: str, classification report
+    :return: pandas.DataFrame, training dataframe
+    :return: pandas.DataFrame, testing dataframe
+    :return: list, list of class names
+    """
+    warnings.filterwarnings('ignore')
+    
+    test_size = yaml.load(open('config.yaml'), Loader=yaml.FullLoader)['parameters']['training_args']['test_size']
+    train_df, test_df = train_test_split(df, test_size=test_size, stratify=df['category_name'])
+
+    # Optional model configuration  
+    model_config = yaml.load(open('config.yaml'), Loader=yaml.FullLoader)['parameters']['model_args']
+    model_args = ClassificationArgs()
+    model_args.num_train_epochs = model_config['num_train_epochs']
+    model_args.train_batch_size = model_config['train_batch_size']
+    model_args.eval_batch_size = model_config['eval_batch_size']
+    model_args.overwrite_output_dir = model_config['overwrite_output_dir']
+    model_args.fp16 = model_config['fp16']
+    model_args.do_lower_case = model_config['do_lower_case']
+
+    # Create a ClassificationModel
+    model_detail = yaml.load(open('config.yaml'), Loader=yaml.FullLoader)['parameters']['model_types']
+    class_names = yaml.load(open('config.yaml'), Loader=yaml.FullLoader)['parameters']['class_names']
+
+    if use_existing_model:
+        model = ClassificationModel(model_type, model_name, num_labels=len(class_names), args=model_args, use_cuda=False)
+    else:
+        model = ClassificationModel(model_type, model_detail[model_type], num_labels=len(class_names), args=model_args, use_cuda=False)
+
+    # Train the model
+    model.train_model(train_df)
+
+    # Evaluate the model
+    result, model_outputs, wrong_predictions = model.eval_model(test_df)
+    preds = np.argmax(model_outputs, axis=1)
+    class_report =classification_report(test_df['category_name'], preds, target_names=class_names)
+
+    return model, preds, class_report, train_df, test_df, class_names
+
+def save_model(model, model_name):
+    """
+    This function saves the model.
+
+    :param model: simpletransformers.classification.ClassificationModel, model
+    :param model_name: str, name of model
+
+    :return: None
+    """
+    model.model.save_pretrained(model_name)
+    model.tokenizer.save_pretrained(model_name)
+    model.config.save_pretrained(model_name + '/')
+    print('Model saved to ' + model_name + '/')
+
+def show_confusion_matrix(test_category, preds, class_names):
+    """
+    This function shows the confusion matrix.
+
+    :param test_category: numpy.ndarray, array of category name
+    :param preds: numpy.ndarray, array of predictions
+    :param class_names: list, list of class names
+
+    :return: matplotlib.axes._subplots.AxesSubplot, confusion matrix
+    """
+    cm = confusion_matrix(test_category, preds)
+    df_cm = pd.DataFrame(cm, index=class_names, columns=class_names)
+    hmap = sns.heatmap(df_cm, annot=True, fmt="d", cmap="Blues")
+    hmap.yaxis.set_ticklabels(hmap.yaxis.get_ticklabels(), rotation=0, ha='right')
+    hmap.xaxis.set_ticklabels(hmap.xaxis.get_ticklabels(), rotation=30, ha='right')
+    plt.ylabel('True Topics')
+    plt.xlabel('Predicted Topics')
+
+def predict_proba(model,text):
+    """
+    This function predicts the probability of each class (in a text form).
+
+    :param model: simpletransformers.classification.ClassificationModel, model
+    :param text: str, text to predict
+
+    :return: numpy.ndarray, array of probabilities
+    """
+    proba = softmax(model.predict([text])[1])[0]
+    print('-----------------------------')
+    print('Text to Predict: ', text)
+    print('Probability of each class:')
+    print('Fertilizer: ', proba[0])
+    print('Pesticide: ', proba[1])
+    print('Others: ', proba[2])
+
+def predict_proba_array(model,text):
+    """
+    This function predicts the probability of each class (in an array form).
+
+    :param model: simpletransformers.classification.ClassificationModel, model
+    :param text: str, text to predict
+
+    :return: numpy.ndarray, array of probabilities
+    """
+    proba = softmax(model.predict([text])[1])[0]
+    return proba

functions/preprocessing_function.py

@@ -0,0 +1,203 @@
+import numpy as np
+import pandas as pd
+from rapidfuzz import process, fuzz, utils
+
+def clean_dataframe(df, column, remove_na=True, remove_non_words=True, remove_symbols=True, remove_duplicates=True):
+    """
+    This function cleans the given dataframe by removing NaN, non-words, symbols, and duplicates.
+
+    Parameters:
+    df (pandas.DataFrame): The dataframe to clean.
+    column (str): The column to clean.
+    remove_na (bool): Whether to remove NaN or not.
+    remove_non_words (bool): Whether to remove non-words or not.
+    remove_symbols (bool): Whether to remove symbols or not.
+    remove_duplicates (bool): Whether to remove duplicates or not.
+
+    Returns:
+    pandas.DataFrame: The cleaned dataframe.
+    """
+    # Lowercase the column
+    df[column + ' Clean'] = df[column].apply(lambda x: str(x).lower())
+    
+    # Remove non words (symbols, numbers, etc.)
+    if remove_non_words:
+        df[column + ' Clean'] = ''
+        for i in range(len(df)):
+            row = df.iloc[i]
+            clean_word_list = []
+            for word in str(row[column]).lower().split():
+                if not any(char.isdigit() for char in word):
+                    clean_word_list.append(word)
+            df.at[i, column + ' Clean'] = ' '.join(clean_word_list)
+    
+    # Remove symbols, but keep numbers
+    if remove_symbols:
+        df[column + ' Clean'] = df[column + ' Clean'].apply(lambda x: ''.join(letter for letter in x if letter.isalnum() or letter.isspace()))
+    
+    # Drop if the new column is NaN or empty string (when the whitespace is removed, it is '')
+    if remove_na:
+        df = df[df[column + ' Clean'].notna()]
+        df = df[df[column + ' Clean'].replace(' ','') != '']
+    
+    # Remove duplicates
+    if remove_duplicates:
+        df = df.drop_duplicates(subset=[column + ' Clean'])
+
+    return df
+
+def fuzzy_join(row, df_reference, column_reference, column_matched_to, take_regist_number=False, take_source = False, set_ratio_weight=0.5, ratio_weight=0.5):
+    """
+    This function applies fuzzy join to the given row and returns the matched product name and nomor pendaftaran
+    based on the maximum similarity score between the two columns.
+
+    Parameters:
+    row (pandas.Series): The row to apply fuzzy join on.
+    df_reference (pandas.DataFrame): The dataframe to compare with.
+    column_reference (str): The column to use for fuzzy join.
+    column_matched_to (str): The column to compare with.
+    take_regist_number (bool): Whether to take the nomor pendaftaran from the registered fertilizer dataset.
+    set_ratio_weight (int): The weight to set for the ratio-based similarity metric.
+    ratio_weight (int): The weight to set for the weighted average of the two similarity metrics.
+
+    Returns:
+    pandas.DataFrame: The input dataframe with additional columns for matched product name and nomor pendaftaran.
+    """
+    similar_product_name = ''
+    similarity_score = 0
+    nomor_pendaftaran = ''
+    source = ''
+    for product_name in df_reference[column_reference]:
+        if set_ratio_weight == 0:
+            score = fuzz.ratio(product_name.lower(), row[column_matched_to].lower(), processor=utils.default_process)
+        elif ratio_weight == 0:
+            score = fuzz.token_set_ratio(product_name, row[column_matched_to], processor=utils.default_process)
+        else:
+            score = set_ratio_weight * fuzz.token_set_ratio(product_name, row[column_matched_to], processor=utils.default_process) + ratio_weight * fuzz.ratio(product_name.lower(), row[column_matched_to].lower(), processor=utils.default_process)
+            
+        if score > similarity_score:
+            similarity_score = score
+            similar_product_name = product_name
+            if take_regist_number:
+                nomor_pendaftaran = df_reference[df_reference[column_reference] == product_name]['Nomor Pendaftaran'].iloc[0]
+            if take_source:
+                source = df_reference[df_reference[column_reference] == product_name]['Source'].iloc[0]
+
+    if take_regist_number and take_source:
+        return similar_product_name, similarity_score, nomor_pendaftaran, source
+    elif take_regist_number:
+        return similar_product_name, similarity_score, nomor_pendaftaran
+    elif take_source:
+        return similar_product_name, similarity_score, source
+    else:
+        return similar_product_name, similarity_score
+    
+
+def fuzzy_join_compare(df, first_column, second_column, registered_fertilizers, take_regist_number=True, set_ratio_weight=1, ratio_weight=0):
+    """
+    This function applies fuzzy join to the given dataframe and returns the matched product name and nomor pendaftaran
+    based on the maximum similarity score between the two columns.
+
+    Parameters:
+    df (pandas.DataFrame): The dataframe to apply fuzzy join on.
+    first_column (str): The first column to use for fuzzy join.
+    second_column (str): The second column to compare with.
+    registered_fertilizers (pandas.DataFrame): The dataframe containing the registered fertilizers.
+    take_regist_number (bool): Whether to take the nomor pendaftaran from the registered fertilizer dataset.
+    set_ratio_weight (int): The weight to set for the ratio-based similarity metric.
+    ratio_weight (int): The weight to set for the weighted average of the two similarity metrics.
+
+    Returns:
+    pandas.DataFrame: The input dataframe with additional columns for matched product name and nomor pendaftaran.
+    """
+    df['Matched Product Name 1'], df['Similarity Score 1'], df['Nomor Pendaftaran 1'] = zip(*df.apply(lambda row: fuzzy_join(row, registered_fertilizers, 'Nama Lengkap', first_column, take_regist_number=take_regist_number, set_ratio_weight=set_ratio_weight, ratio_weight=ratio_weight), axis=1))
+    df['Matched Product Name 2'], df['Similarity Score 2'], df['Nomor Pendaftaran 2'] = zip(*df.apply(lambda row: fuzzy_join(row, registered_fertilizers, 'Nama Lengkap', second_column, take_regist_number=take_regist_number, set_ratio_weight=set_ratio_weight, ratio_weight=ratio_weight), axis=1))
+
+    # Take the maximum similarity score and take the matched product name and nomor pendaftaran based on that
+    df['Max Similarity Score'] = df[['Similarity Score 1', 'Similarity Score 2']].max(axis=1)
+    # If condition: if similarity score 1 is higher than equal to similarity score 2, take the matched product name 1 as matched product name, else take matched product name 2
+    df['Matched Product Name'] = np.where(df['Similarity Score 1'] >= df['Similarity Score 2'], df['Matched Product Name 1'], df['Matched Product Name 2'])
+    # If condition: if similarity score 1 is higher than equal to similarity score 2, take the nomor pendaftaran 1 as nomor pendaftaran, else take nomor pendaftaran 2
+    df['Nomor Pendaftaran'] = np.where(df['Similarity Score 1'] >= df['Similarity Score 2'], df['Nomor Pendaftaran 1'], df['Nomor Pendaftaran 2'])
+    # Remove the columns that are no longer needed such as the matched product name 1 and 2, similarity score 1 and 2, and nomor pendaftaran 1 and 2
+    df.drop(columns=['Matched Product Name 1', 'Matched Product Name 2', 'Similarity Score 1', 'Similarity Score 2', 'Nomor Pendaftaran 1', 'Nomor Pendaftaran 2'], inplace=True)
+    
+    return df
+
+def slice_with_filter(df, column, ref_df, use_filter=False, filter_condition=None):
+    """
+    This function slices the given dataframe based on the given reference dataframe.
+
+    :param df: pandas.DataFrame, dataframe to be sliced
+    :param column: str, column to be sliced
+    :param ref_df: pandas.DataFrame, reference dataframe
+    :param use_filter: bool, whether to use filter or not
+    :param filter_condition: str, filter condition
+
+    :return: pandas.DataFrame, sliced dataframe
+    """
+    if use_filter:
+        ref_df = ref_df[filter_condition]
+
+    return df[~df[column].isin(ref_df[column].to_list())]
+
+def combine_catalog(column_1, column_2, source_1, source_2):
+    """
+    This function combines two columns into one dataframe.
+
+    :param column_1: pandas.Series, first column
+    :param column_2: pandas.Series, second column
+    :param source_1: str, source of first column
+    :param source_2: str, source of second column
+
+    :return: pandas.DataFrame, combined dataframe
+    """
+    combined_catalog = pd.concat([column_1, column_2])
+    combined_catalog = combined_catalog.to_frame(name='Registered Product')
+    combined_catalog['Source'] = pd.concat([column_1.apply(lambda x: source_1), column_2.apply(lambda x: source_2)])
+    combined_catalog.reset_index(drop=True, inplace=True)
+
+    return combined_catalog
+
+def clean_category_dataframe(df, category_column, product_name_column, reference_table, reference_column, split=False):
+    """
+    This function cleans the given dataframe by removing NaN, non-words, symbols, and duplicates.
+
+    Parameters:
+    df (pandas.DataFrame): The dataframe to clean.
+    category_column (str): The column containing category name.
+    product_name_column (str): The column containing product name.
+    reference_table (pandas.DataFrame): The reference table to be used for fuzzy join.
+    reference_column (str): The column to be used for fuzzy join.
+    split (bool): Whether to split the dataframe into two or not.
+
+    Returns:
+    pandas.DataFrame: The cleaned dataframe.
+    """
+    # If column does not contain "Category", fill it with "Unknown" (including those that are NaN)
+    df[category_column] = df[category_column].apply(lambda x: x if isinstance(x, str) and 'Category' in x else 'Unknown')
+    # If column contains "Category", remove the word "Category" and replace "\n" with ","
+    df[category_column] = df[category_column].apply(lambda x: x.replace('Category', '').replace('\n', ',') if isinstance(x, str) else x)
+    # Replace "Lihat Lebih Banyak" with empty string
+    df[category_column] = df[category_column].apply(lambda x: x.replace('Lihat Lebih Banyak', '') if isinstance(x, str) else x)
+    # Add category_list column
+    df['category_list'] = df[category_column].apply(lambda x: x.split(',') if isinstance(x, str) else x)
+    # Add product_name_clean
+    df['product_name_clean'] = df[product_name_column].apply(lambda x: str(x).lower().strip())
+    # Remove duplicates
+    df = df.drop_duplicates(subset=['product_name_clean'], keep = 'last')
+    # Left join with product query
+    df_reference = reference_table.merge(df[['product_name_clean','category_list']], how='left', left_on=reference_table[reference_column].str.lower().str.strip(), right_on=df['product_name_clean'])
+    # convert category_list that contains 'Unknown' to NaN
+    df_reference['category_list'] = df_reference['category_list'].apply(lambda x: np.nan if isinstance(x, list) and 'Unknown' in x else x)
+    # if the list in category_list contains empty string element, drop that element from the list
+    df_reference['category_list'] = df_reference['category_list'].apply(lambda x: [i for i in x if i != ''] if isinstance(x, list) else x)
+    # Choose final columns
+    df_reference = df_reference[['Product Name', 'Product Name Clean', 'category_list']]
+    # Strip
+    df_reference['category_list'] = df_reference['category_list'].apply(lambda x: [i.strip() for i in x] if isinstance(x, list) else x)
+
+    if split:
+        return df_reference, df_reference.dropna(subset=['category_list'])
+    else:
+        return df_reference