#!/usr/bin/env python """The chatbot model training app is the "Train the chatbot model page" in the main page""" # the code for the page def run_app(): """ Application code for the Chatbot Training app """ ############################ # :: IMPORTS AND CONSTANTS # ############################ # machine learning modules import numpy as np import random # Deep Learning modules from keras.models import Sequential from keras.layers import Dense, Dropout from keras.optimizers import SGD import streamlit as st # suppress warnings import warnings warnings.filterwarnings('ignore') # for file processing import json import pickle # natural language processiong modules import nltk from nltk.stem import WordNetLemmatizer lemmatizer = WordNetLemmatizer() # Load NLTK dependencies nltk.download('punkt') nltk.download('wordnet') ###################################### # :: HELPER FUNCTIONS FOR PROCESSING # ###################################### def read_json_file(uploaded_file): """ Reads a JSON file and returns its contents. Args: uploaded_file (_UploadedFile): The uploaded JSON file. Returns: dict: The contents of the JSON file. """ data = json.loads(uploaded_file.read().decode("utf-8")) return data def preprocess_data(data): """ Preprocesses the JSON data. Args: data (dict): The JSON data. Returns: tuple: The pre-processed data (words, classes, documents). """ # creating lists for NLP words=[] classes = [] documents = [] ignore_words = ['?', '!'] lemmatizer = WordNetLemmatizer() for command in data['intents']: for pattern in command['patterns']: # Tokenize each word w = nltk.word_tokenize(pattern) words.extend(w) # Add documents to the corpus documents.append((w, command['tag'])) # Add to classes list if command['tag'] not in classes: classes.append(command['tag']) # Lemmatize, convert to lowercase, and remove duplicates words = [lemmatizer.lemmatize(w.lower()) for w in words if w not in ignore_words] words = sorted(list(set(words))) classes = sorted(list(set(classes))) return words, classes, documents def load_pickle_data(): """ Loads the pickle data. Returns: tuple: The loaded pickle data (words, classes, documents). """ with open('words.pkl', 'rb') as f: words = pickle.load(f) with open('classes.pkl', 'rb') as f: classes = pickle.load(f) with open('documents.pkl', 'rb') as f: documents = pickle.load(f) return words, classes, documents def create_training_data(words, classes, documents): """ Create the training data for the chatbot model. Args: words (list): List of words in the vocabulary. classes (list): List of classes/intents. documents (list): List of (pattern, intent) pairs. Returns: tuple: Tuple containing the training data as NumPy arrays (train_x, train_y). """ training = [] output_empty = np.zeros(len(classes), dtype=int) for doc in documents: bag = np.zeros(len(words), dtype=int) pattern_words = doc[0] pattern_words = [lemmatizer.lemmatize(word.lower()) for word in pattern_words] for i, w in enumerate(words): if w in pattern_words: bag[i] = 1 output_row = np.copy(output_empty) output_row[classes.index(doc[1])] = 1 training.append([bag, output_row]) random.shuffle(training) training = np.array(training) train_x = list(training[:, 0]) train_y = list(training[:, 1]) return np.array(train_x), np.array(train_y) ############### # :: MAIN APP # ############### st.markdown("

Train the chatbot model ⚙️

", unsafe_allow_html=True) st.subheader( """ Let's train the chatbot model by following the sequence of steps provided below: """ ) # Summary of steps st.markdown( """ **Summary of Steps:** - Upload the `commands.json` file for processing. The file should contain the commands and their corresponding tags. - Load the preprocessed data from pickle files (optional if you have already processed the data previously). - Create the training data by converting the commands into numerical vectors. - Build the model by specifying the number of layers, epochs, batch size, and activation function. Once the model is built, the training loss and accuracy will be displayed. """ ) st.write("---") if st.checkbox("Upload the commands.json file for processing"): st.subheader("JSON File Uploader") uploaded_file = st.file_uploader("Upload JSON file", type="json") if uploaded_file is not None: try: data = read_json_file(uploaded_file) st.json(data) # Preprocess the data words, classes, documents = preprocess_data(data) # Save the preprocessed data as pickle files with open('words.pkl', 'wb') as f: pickle.dump(words, f) with open('classes.pkl', 'wb') as f: pickle.dump(classes, f) with open('documents.pkl', 'wb') as f: pickle.dump(documents, f) # Display the processed data st.write("Preprocessing Results:") st.write(len(documents), "documents") st.write(len(classes), "classes", classes) st.write(len(words), "unique lemmatized words", words) except json.JSONDecodeError: st.error("Invalid JSON file.") if st.checkbox("Load pickle data"): # Initialize the progress bar progress_bar = st.progress(0) with st.spinner("Creating training data ..."): words, classes, documents = load_pickle_data() # Update the progress bar progress_bar.progress(100) st.write("Words:") st.write(words) st.write("Classes:") st.write(classes) st.write("Documents:") st.write(documents) if st.checkbox("Create training data"): try: # Initialize the progress bar progress_bar = st.progress(0) with st.spinner("Creating training data ..."): train_x, train_y = create_training_data(words, classes, documents) # Update the progress bar progress_bar.progress(100) st.success("Training data created") st.write(f"Training data (train_x): {len(train_x)} samples") st.write(f"Training data (train_y): {len(train_y)} samples") except Exception as e: st.error("An error occurred during training data creation.") st.error(str(e)) if st.checkbox("Build the model"): # Get user inputs num_layers = st.number_input("Number of layers", min_value=1, max_value=10, value=3) epochs = st.number_input("Number of epochs", min_value=1, max_value=1000, value=200) batch_size = st.number_input("Batch size", min_value=1, max_value=100, value=5) activation_functions = ['relu', 'sigmoid', 'softmax'] activation_function = st.selectbox("Activation function", options=activation_functions) try: # Initialize the progress bar progress_bar = st.progress(0) with st.spinner("Building the model ..."): # Create model model = Sequential() # Add layers to the model based on user input for i in range(num_layers): if i == 0: # Input layer model.add(Dense(128, input_shape=(len(train_x[0]),), activation=activation_function)) else: # Hidden layers model.add(Dense(64, activation=activation_function)) model.add(Dropout(0.5)) # Output layer model.add(Dense(len(train_y[0]), activation='softmax')) # Compile model sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) # Fit the model hist = model.fit(np.array(train_x), np.array(train_y), epochs=epochs, batch_size=batch_size, verbose=1) # Save the model model.save('chatbot_model.h5', hist) # Update the progress bar progress_bar.progress(100) st.success("The chatbot model is created") # Display training loss and accuracy summary st.subheader("Training Summary") st.write("Training Loss:", hist.history['loss'][-1]) st.write("Training Accuracy:", hist.history['accuracy'][-1]) except Exception as e: st.error("An error occurred during model building.") st.error(str(e)) # End of app if __name__ == "__main__": run_app()