chatbotlib/train_chatbot.py

#!/usr/bin/env python

"""The chatbot model training app is the "Train the chatbot model page" in the main page"""

# the code for the page
def run_app():
    """
    Application code for the Chatbot Training app
    """
    ############################
    # :: IMPORTS AND CONSTANTS #
    ############################

    # machine learning modules
    import numpy as np
    import random

    # Deep Learning modules
    from keras.models import Sequential
    from keras.layers import Dense, Dropout
    from keras.optimizers import SGD

    import streamlit as st

    # suppress warnings
    import warnings
    warnings.filterwarnings('ignore')

    # for file processing
    import json
    import pickle

    # natural language processiong modules
    import nltk
    from nltk.stem import WordNetLemmatizer
    lemmatizer = WordNetLemmatizer()

    # Load NLTK dependencies
    nltk.download('punkt')
    nltk.download('wordnet')

    ######################################
    # :: HELPER FUNCTIONS FOR PROCESSING #
    ######################################

    def read_json_file(uploaded_file):
        """
        Reads a JSON file and returns its contents.

        Args:
            uploaded_file (_UploadedFile): The uploaded JSON file.

        Returns:
            dict: The contents of the JSON file.
        """
        data = json.loads(uploaded_file.read().decode("utf-8"))
        return data

    def preprocess_data(data):
        """
        Preprocesses the JSON data.

        Args:
            data (dict): The JSON data.

        Returns:
            tuple: The pre-processed data (words, classes, documents).
        """

        # creating lists for NLP
        words=[]
        classes = []
        documents = []
        ignore_words = ['?', '!']

        lemmatizer = WordNetLemmatizer()
        for command in data['intents']:
            for pattern in command['patterns']:
                # Tokenize each word
                w = nltk.word_tokenize(pattern)
                words.extend(w)
                # Add documents to the corpus
                documents.append((w, command['tag']))
                # Add to classes list
                if command['tag'] not in classes:
                    classes.append(command['tag'])

        # Lemmatize, convert to lowercase, and remove duplicates
        words = [lemmatizer.lemmatize(w.lower()) for w in words if w not in ignore_words]
        words = sorted(list(set(words)))
        classes = sorted(list(set(classes)))

        return words, classes, documents

    def load_pickle_data():
        """
        Loads the pickle data.

        Returns:
            tuple: The loaded pickle data (words, classes, documents).
        """
        with open('words.pkl', 'rb') as f:
            words = pickle.load(f)
        with open('classes.pkl', 'rb') as f:
            classes = pickle.load(f)
        with open('documents.pkl', 'rb') as f:
            documents = pickle.load(f)
        return words, classes, documents

    def create_training_data(words, classes, documents):
        """
        Create the training data for the chatbot model.

        Args:
            words (list): List of words in the vocabulary.
            classes (list): List of classes/intents.
            documents (list): List of (pattern, intent) pairs.

        Returns:
            tuple: Tuple containing the training data as NumPy arrays (train_x, train_y).
        """
        training = []
        output_empty = np.zeros(len(classes), dtype=int)

        for doc in documents:
            bag = np.zeros(len(words), dtype=int)
            pattern_words = doc[0]
            pattern_words = [lemmatizer.lemmatize(word.lower()) for word in pattern_words]

            for i, w in enumerate(words):
                if w in pattern_words:
                    bag[i] = 1

            output_row = np.copy(output_empty)
            output_row[classes.index(doc[1])] = 1

            training.append([bag, output_row])

        random.shuffle(training)
        training = np.array(training)

        train_x = list(training[:, 0])
        train_y = list(training[:, 1])

        return np.array(train_x), np.array(train_y)

    ###############
    # :: MAIN APP #
    ###############

    st.markdown("<h1 style='text-align: left;'>Train the chatbot model ⚙️</h1>", unsafe_allow_html=True)
    st.subheader(
    """
    Let's train the chatbot model by following the sequence of steps provided below:
    """
    )
    # Summary of steps
    st.markdown(
        """
        **Summary of Steps:**
        - Upload the `commands.json` file for processing. The file should contain the commands and their corresponding tags.
        - Load the preprocessed data from pickle files (optional if you have already processed the data previously).
        - Create the training data by converting the commands into numerical vectors.
        - Build the model by specifying the number of layers, epochs, batch size, and activation function.

        Once the model is built, the training loss and accuracy will be displayed.
        """
    )
    st.write("---")

    if st.checkbox("Upload the commands.json file for processing"):
        st.subheader("JSON File Uploader")
        uploaded_file = st.file_uploader("Upload JSON file", type="json")

        if uploaded_file is not None:
            try:
                data = read_json_file(uploaded_file)

                st.json(data)

                # Preprocess the data
                words, classes, documents = preprocess_data(data)

                # Save the preprocessed data as pickle files
                with open('words.pkl', 'wb') as f:
                    pickle.dump(words, f)
                with open('classes.pkl', 'wb') as f:
                    pickle.dump(classes, f)
                with open('documents.pkl', 'wb') as f:
                    pickle.dump(documents, f)

                # Display the processed data
                st.write("Preprocessing Results:")
                st.write(len(documents), "documents")
                st.write(len(classes), "classes", classes)
                st.write(len(words), "unique lemmatized words", words)

            except json.JSONDecodeError:
                st.error("Invalid JSON file.")
    
    if st.checkbox("Load pickle data"):
        # Initialize the progress bar
        progress_bar = st.progress(0) 
        
        with st.spinner("Creating training data ..."):

            words, classes, documents = load_pickle_data()
            
        # Update the progress bar
        progress_bar.progress(100)

        st.write("Words:")
        st.write(words)

        st.write("Classes:")
        st.write(classes)

        st.write("Documents:")
        st.write(documents)

    if st.checkbox("Create training data"):
        try:
            # Initialize the progress bar
            progress_bar = st.progress(0)  

            with st.spinner("Creating training data ..."):
                train_x, train_y = create_training_data(words, classes, documents)

            # Update the progress bar
            progress_bar.progress(100)

            st.success("Training data created")

            st.write(f"Training data (train_x): {len(train_x)} samples")
            st.write(f"Training data (train_y): {len(train_y)} samples")
    
        except Exception as e:
            st.error("An error occurred during training data creation.")
            st.error(str(e))

    if st.checkbox("Build the model"):
        # Get user inputs
        num_layers = st.number_input("Number of layers", min_value=1, max_value=10, value=3)
        epochs = st.number_input("Number of epochs", min_value=1, max_value=1000, value=200)
        batch_size = st.number_input("Batch size", min_value=1, max_value=100, value=5)
        activation_functions = ['relu', 'sigmoid', 'softmax']
        activation_function = st.selectbox("Activation function", options=activation_functions)

        try:
            # Initialize the progress bar
            progress_bar = st.progress(0)  

            with st.spinner("Building the model ..."):
                # Create model
                model = Sequential()

                # Add layers to the model based on user input
                for i in range(num_layers):
                    if i == 0:
                        # Input layer
                        model.add(Dense(128, input_shape=(len(train_x[0]),), activation=activation_function))
                    else:
                        # Hidden layers
                        model.add(Dense(64, activation=activation_function))
                    model.add(Dropout(0.5))

                # Output layer
                model.add(Dense(len(train_y[0]), activation='softmax'))

                # Compile model
                sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
                model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])

                # Fit the model
                hist = model.fit(np.array(train_x), np.array(train_y), epochs=epochs, batch_size=batch_size, verbose=1)

                # Save the model
                model.save('chatbot_model.h5', hist)

            # Update the progress bar
            progress_bar.progress(100)

            st.success("The chatbot model is created")

            # Display training loss and accuracy summary
            st.subheader("Training Summary")
            st.write("Training Loss:", hist.history['loss'][-1])
            st.write("Training Accuracy:", hist.history['accuracy'][-1])
    
        except Exception as e:
            st.error("An error occurred during model building.")
            st.error(str(e))

# End of app
if __name__ == "__main__":

    run_app()