all code files

.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

Dockerfile

@@ -0,0 +1,14 @@
+# read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.9
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . .
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 binaychandra
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

app.py

@@ -0,0 +1,242 @@
+# Standard Library Imports
+from flask import ( Flask, jsonify, render_template, request,
+                    url_for,
+                    make_response,
+                    session,
+                    send_file,
+                    Response,
+                    render_template_string,
+                    redirect)
+import pandas as pd
+import plotly.graph_objects as go
+from sqlalchemy import create_engine
+from config import tbl_mapping
+from data_connector.sqlite_connector import get_db_connection
+from lang_assistant.langhelper import chat_response, summary_extractor_from_df, chat_with_df, generate_graphdata
+from utilities.plotting import (get_validation_json, 
+                                badges_get_pillar_dougnutdata, 
+                                badges_get_badgecompletion_monthwise, 
+                                get_wfrankwise_countmom, 
+                                get_lst_topdepartment, 
+                                get_wfrankwise_count,
+                                get_topfive_badgetitle)
+import sqlite3
+import time
+import json
+
+app = Flask(__name__, static_url_path='/static')
+app.secret_key = 'test'
+
+@app.route("/validate_learning", methods=['GET'])
+def load_learning():
+    # con = sqlite3.connect("database.db")
+    # df = pd.read_sql_query(f"SELECT * from learning", con)
+    df = pd.DataFrame({'ID':[12,13], 'Status':['Done', 'In Progress']})
+    no_rows, no_cols = df.shape
+    n_gui = df.GUI.nunique() if 'GUI' in df.columns else 'GUI not Found'
+
+    tablevalues = {'n_rows':no_rows, 'n_cols':no_cols, 'unique_gui':n_gui}
+    print('Calculation done')
+    # Data for the bar chart
+    bar_chart_data = {
+        'labels': ['Label 1', 'Label 2', 'Label 3', 'Label 4', 'Label 5'],
+        'values': [30, 40, 30, 21, 34]
+    }
+    # Data for the pie chart
+    pie_chart_data = {
+        'labels': ['Label A', 'Label B', 'Label C', 'Label D', 'Label E'],
+        'values': [45, 30, 25, 9, 34]
+    }
+
+    # Create the bar chart figure
+    bar_chart_figure = go.Figure(
+        data=[
+            go.Bar(
+                x=bar_chart_data['labels'],
+                y=bar_chart_data['values'],
+                marker_color='rgba(54, 162, 235, 0.5)',
+                marker_line_color='rgba(54, 162, 235, 1)',
+                marker_line_width=1
+            )
+        ],
+        layout=go.Layout(
+            title='Bar Chart',
+            yaxis=dict(title='Values'),
+            margin=dict(l=20, r=20, t=40, b=20)
+        )
+    )
+
+    # Create the pie chart figure
+    pie_chart_figure = go.Figure(
+        data=[
+            go.Pie(
+                labels=pie_chart_data['labels'],
+                values=pie_chart_data['values'],
+                hole=0.3,
+                marker=dict(colors=['rgba(255, 99, 132, 0.5)', 'rgba(54, 162, 235, 0.5)', 'rgba(255, 206, 86, 0.5)'],
+                            line=dict(color='rgba(0, 0, 0, 0.5)', width=1))
+            )
+        ],
+        layout=go.Layout(
+            title='Pie Chart',
+            margin=dict(l=20, r=20, t=40, b=20)
+        )
+    )
+    # Convert the figures to HTML
+    bar_chart_html = bar_chart_figure.to_html(full_html=False)
+    pie_chart_html = pie_chart_figure.to_html(full_html=False)
+
+    data = {
+        'Regex issue': [-90, -10, -5, 0],
+        'Null percentage': [-10, -35, 0, 0],
+        'Seems ok': [40, 45, 90, 100],
+        'data mismatch': [0, -10, -5, 0]
+    }
+
+    df = pd.DataFrame(data, index=['GTE', 'SMU', 'Service_Line', 'Sub_SL'])
+
+    labels = df.index.to_list()
+    reg_issue = df['Regex issue'].to_list()
+    null_issue = df['Null percentage'].to_list()
+    ok_data = df['Seems ok'].to_list()
+    mismatch_issue = df['data mismatch'].to_list()
+    tbl_selected = session.get('tbl_selected', [])
+
+    return render_template("validate_learning.html",
+                            req_tables = tbl_selected,
+                            bar_chart_html=bar_chart_html, 
+                            pie_chart_html=pie_chart_html, 
+                            table_info = tablevalues,
+                            labels = labels, reg_issue=reg_issue, 
+                            null_issue=null_issue, ok_data=ok_data , mismatch_issue=mismatch_issue,
+                            show_sidebar=True)
+
+@app.route("/validate_badges", methods=['GET', 'POST'])
+def load_badges():
+    con = sqlite3.connect("database.db")
+    df = pd.read_sql_query(f"SELECT * from badges", con)
+    no_rows, no_cols = df.shape
+    n_gui = df.GUI.nunique() if 'GUI' in df.columns else 'GUI not Found'
+
+    tablevalues = {'n_rows':no_rows, 'n_cols':no_cols, 'unique_gui':n_gui}
+    print('Calculation done')
+    
+    tbl_selected = session.get('tbl_selected', [])
+
+    json_data = get_validation_json('badges')
+    json_pillar_data = badges_get_pillar_dougnutdata()
+    json_badgecompletion_data = badges_get_badgecompletion_monthwise()
+    lst_topfive_badgetitle = get_topfive_badgetitle()
+    
+    return render_template("validate_badges.html",
+                            lst_topfive_badgetitle = lst_topfive_badgetitle,
+                            req_tables = tbl_selected,
+                            table_info = tablevalues,
+                            json_data = json_data,
+                            json_pillar_data=json_pillar_data,
+                            json_badgecompletion_data = json_badgecompletion_data,
+                            show_sidebar=True)
+
+@app.route("/validation", methods=['GET', 'POST'])
+@app.route("/validate_workforce", methods=['GET', 'POST'])
+def load_workforce():
+    # con = sqlite3.connect("database.db")
+    # df = pd.read_sql_query(f"SELECT * from workforce", con)
+    df = pd.read_csv(r"referencefiles\workforce.csv")
+    no_rows, no_cols = df.shape
+    n_gui = df.GUI.nunique() if 'GUI' in df.columns else 'GUI not Found'
+
+    tablevalues = {'n_rows':no_rows, 'n_cols':no_cols, 'unique_gui':n_gui}
+    
+    json_val_data = get_validation_json('workforce')
+    json_empdist = get_wfrankwise_countmom()
+    lst_topfive_dept = get_lst_topdepartment()
+    json_rankwise_empdist = get_wfrankwise_count()
+    tbl_selected = session.get('tbl_selected', []) #['Badges', 'learning']
+    gpt_response = summary_extractor_from_df("""{"male": 56, "female": 44 }""")
+
+    return render_template("validate_workforce.html",
+                            req_tables = tbl_selected,
+                            table_info = tablevalues,
+                            json_data = json_val_data,
+                            json_empdist = json_empdist,
+                            lst_topfive_dept = lst_topfive_dept,
+                            json_rankwise_empdist = json_rankwise_empdist,
+                            aicontent_genderanalysis = gpt_response, 
+                            show_sidebar = True)
+
+@app.route("/validate_miscellaneous", methods=['GET', 'POST'])
+def load_miscellaneous():
+    tbl_selected = session.get('tbl_selected', [])    
+    return render_template("validate_miscellaneous.html", 
+                           req_tables = tbl_selected,
+                           show_sidebar = True)
+
+@app.route("/timecard.html", methods=['GET', 'POST'])
+def load_timecard():
+    return render_template("timecard.html")
+
+@app.route("/get_llmresponse")
+def get_bot_response():
+    user_message = request.args.get('msg')
+    dd_table_selected = request.args.get('table_selected')
+    print(f"user message and table selected : {user_message}, {dd_table_selected}")
+    print(f"request args : {request.args.get('msg')}")
+    response_usrmsg = chat_with_df(user_message, table_name = dd_table_selected)
+    return response_usrmsg
+
+@app.route("/get_val_llmresponse")
+def get_bot_valresponse():
+    user_message = request.args.get('msg')
+    table_selected = request.args.get('table_selected')
+    print(f"user message and table selected : {user_message}, {table_selected}")
+    print(f"request args : {request.args.get('msg')}")
+
+    try:
+        llm_response_dict = generate_graphdata(user_message, table_name = table_selected)
+    except Exception as e:
+        llm_response_dict = dict(success=False, 
+                                chart_type='text', 
+                                chart_label=None,
+                                chart_json_data=None,
+                                text_to_display="Exception : Some error occured while processing, "+str(e)[:50] + "..")
+    
+    print(llm_response_dict)
+    output_gendata = json.dumps(llm_response_dict)
+    return output_gendata
+
+@app.route("/data.html", methods=['GET', 'POST'])
+@app.route("/data", methods=['GET', 'POST'])
+def data():
+    tbl_htmls = {}
+    tbl_selected = session.get('tbl_selected', [])
+    print(tbl_selected)
+    for tblname in tbl_selected:
+        # Read sqlite query results into a pandas DataFrame
+        # con = sqlite3.connect("database.db")
+        # df = pd.read_sql_query(f"SELECT * from {tblname}", con)
+        df = pd.read_csv(f"referencefiles\{tblname}.csv")
+        top_records = df.copy()
+        # con.close()
+        html_top_records = top_records.to_html(index=False, table_id= f'dtable_{tblname}', classes='display nowrap table table-bordered table-striped table-condensed small p-1', justify='left')
+        html_top_records = html_top_records.replace('<thead>', '<thead class="thead-light" style="top: 0;position: sticky;">')
+        tbl_htmls[tblname] = html_top_records
+
+    return render_template('data.html', table_htmls = tbl_htmls, req_tables = json.dumps(tbl_selected[0]))
+
+@app.route("/", methods=['GET', 'POST'])
+@app.route("/home", methods=['GET', 'POST'])
+def hometest():
+    if request.method == 'GET':
+        return render_template('home.html')
+    elif request.method == 'POST':
+        session['start_date'] = request.form.get('calendar_value').split(":")[0]
+        session['end_date'] = request.form.get('calendar_value').split(":")[1]
+        session['sl_subsl'] = request.form.get('sl_subsl')
+        session['tbl_selected'] = request.form.getlist('tbl_selected')
+        return redirect(url_for('data'))
+
+if __name__ == '__main__':
+    #app.run(debug=True)
+    from waitress import serve
+    serve(app, host="0.0.0.0", port=8080)

config.py

@@ -0,0 +1,9 @@
+tbl_mapping = {
+    'workforce' : '[dbo].[workforce]',
+    'badges' : '[dbo].[badges]',
+    'learning' : '[dbo].[learning]',
+    'lat' : '[dbo].[lat]',
+    'mct' : '[dbo].[mct]',
+    'profqual' : '[dbo].[profqual]',
+    'workexperience' : '[dbo].[workexperience]'
+}

data_connector/fakedata_generator.py

@@ -0,0 +1,14 @@
+import pandas as pd
+#from faker import Faker
+from datetime import date, timedelta
+import random
+
+def generate_fakedata_badges(num_rows = 1000):
+    pass
+
+def generate_fakedata_workforce(num_rows = 1000):
+    # Initialize the Faker object
+    pass
+
+def upload_to_sqlitedb():
+    pass

data_connector/sqlite_connector.py

@@ -0,0 +1,15 @@
+import sqlite3
+
+def get_db_connection():
+    conn = sqlite3.connect('database.db')
+    conn.row_factory = sqlite3.Row
+    return conn
+
+def push_to_sqlitedb(df, db_tblname):
+    # Get the database connection object
+    conn = get_db_connection()
+    try:
+        df.to_sql(db_tblname, conn, index=False, if_exists = 'replace')
+    except:
+        raise Exception("Insertion Error : DB Insertion failed..")
+    return None

lang_assistant/langhelper.py

@@ -0,0 +1,301 @@
+import langchain
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts import ChatPromptTemplate
+import openai
+from langchain.memory import ConversationBufferMemory
+import pandas as pd
+from langchain import memory
+import sqlite3
+import os
+import json
+from langchain.agents import create_pandas_dataframe_agent
+from langchain.agents.agent_types import AgentType
+import io
+import logging
+import requests
+import urllib3
+from langchain.prompts import ChatPromptTemplate
+from langchain.output_parsers import ResponseSchema
+from langchain.output_parsers import StructuredOutputParser
+from langchain.callbacks import LLMonitorCallbackHandler
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts.chat import SystemMessage, HumanMessagePromptTemplate, HumanMessage, AIMessage
+from langchain.agents import create_pandas_dataframe_agent
+
+model_id = 'gpt-3.5-turbo'
+
+from dotenv import load_dotenv, find_dotenv
+_ = load_dotenv(find_dotenv()) # read local .env file
+
+def summary_extractor_from_df(df:pd.DataFrame)-> str:
+    chatmodel = ChatOpenAI()
+    template = "You are an AI assistant and your task is to summarize the workforce distribution based on gender delimited by triple backticks \
+                and output should clearly indicate how much percentage one gender is higher than other one, and based on the findings make some comments on includsiveness \
+                if this is good for company or not.. limit the output in 50 words```{json_genderdf}```"
+    prompt_template = ChatPromptTemplate.from_template(template)
+
+    user_message = prompt_template.format_messages(json_genderdf = df)
+    response = chatmodel(user_message)
+    # print(response.content)
+    return response.content
+
+def chat_response(text:str)->str:
+    chatmodel = ChatOpenAI(max_tokens=50)
+    response = chatmodel.predict(text)
+    return response
+
+def chat_with_df(query, table_name = None):
+    if table_name is None:
+        df = pd.read_csv(r"https://raw.githubusercontent.com/datasciencedojo/datasets/master/titanic.csv")
+    else:
+        #get data
+        con = sqlite3.connect("database.db")
+        df = pd.read_sql_query(f"SELECT * from {table_name}", con)
+
+    agent = create_pandas_dataframe_agent(                      
+                ChatOpenAI(temperature=0, model="gpt-3.5-turbo-0613"),
+                df,
+                agent_type=AgentType.OPENAI_FUNCTIONS,
+                verbose=True
+            )
+    try:
+        res_query = agent.run(query)
+    except Exception as e:
+        print(str(e))
+        res_query = "ERROR"
+
+    return res_query
+
+def gen_chartdata(df_out, label, chart_type):
+    
+    print("Inside gen_chartdata.....")
+    print(f"df_out --> {df_out}")
+    data_dict = {
+                    "labels": None,
+                    "datasets": [
+                        {
+                            "label": "Default Label",
+                            "data": None,
+                            "type" : chart_type,
+                            "backgroundColor": ["rgba(250, 240, 230, 0.7)"],
+                            "hoverBackgroundColor": ["rgba(250, 240, 230, 1)"],
+                            "borderColor": ["rgba(250, 240, 230, 1)"]
+                        }
+                    ]
+                }
+    print(f"df_out.iloc[:,0].to_list() ---> {df_out.iloc[:,0].to_list()}")
+    print(f"data_dict[\"datasets\"][0][\"label\"] --> {data_dict['datasets'][0]['label']}")
+    print(f"label : {label}")
+    print(f"data_dict['datasets'][0]['data'] --> {data_dict['datasets'][0]['data']}")
+    print(f"df_out.iloc[:,1].to_list() --> {df_out.iloc[:,1].to_list()}")
+
+    data_dict["labels"] = df_out.iloc[:,0].to_list()
+    data_dict["datasets"][0]["label"] = label
+    data_dict["datasets"][0]["data"] = df_out.iloc[:,1].to_list()
+    if chart_type == 'doughnut':
+        data_dict["datasets"][0]["backgroundColor"] = ["rgba(120, 214, 198, 0.7)", "rgba(255, 105, 105, 0.7)", "rgba(150, 194, 145, 0.7)", "rgba(250, 240, 230, 0.7)"]
+        data_dict["datasets"][0]["hoverBackgroundColor"] = ["rgba(120, 214, 198, 1)", "rgba(255, 105, 105, 1)", "rgba(150, 194, 145, 1)", "rgba(250, 240, 230, 1)"]
+        data_dict["datasets"][0]["borderColor"] = ["rgba(120, 214, 198, 1)", "rgba(255, 105, 105, 1)", "rgba(150, 194, 145, 1)", "rgba(250, 240, 230, 1)"]
+    
+    # Serializing json  
+    json_object = json.dumps(data_dict, indent = 4) 
+    
+    return json_object
+
+def generate_graphdata(query, table_name = None):
+    if table_name is None:
+        df = pd.read_csv(r"https://raw.githubusercontent.com/datasciencedojo/datasets/master/titanic.csv")
+    else:
+        #get data from sqlite db
+        con = sqlite3.connect("database.db")
+        df = pd.read_sql_query(f"SELECT * from {table_name}", con)
+
+    jsondf = df.head(5).to_json(orient='records')
+    
+    prompt_template = ChatPromptTemplate.from_messages(
+        [
+        SystemMessage(
+            content=(
+                "You are a data manager and you have to assign the task to a pandas coder \
+            who will be working on the steps provided by you. Assuming the pandas coder has already loaded \
+            the dataset in pandas dataframe, your task is to properly provide the steps with \
+            proper column names so that pandas coder can easily understand the ask and code till the processing of dataset\
+            which can be used for graph creation. include steps only to process the data \
+            THE STEPS SHOULD NOT CONTAIN ANY GRAPH CREATION OR PLOTTING INSTRUCTIONS.. \
+            always select columns only that are mentioned \
+            not any extra columns. also think through if output contains two columns if not then create the steps to \
+            only include two appropriate columns in output. Also do not include any steps to select top few records.. \
+            the output should be all records from the steps execution.")
+        ),
+        HumanMessage(content=("\
+                user_input = ```plot the distribution of males and females who survived```\
+            use below dataset for reference delimited by <<<>>>\
+            <<<[{\"PassengerId\":1,\"Survived\":0,\"Pclass\":3,\"Name\":\"Braund, Mr. Owen Harris\",\"Sex\":\"male\",\"Age\":22.0,\"SibSp\":1,\"Parch\":0,\"Ticket\":\"A\/5 21171\",\"Fare\":7.25,\"Cabin\":null,\"Embarked\":\"S\"},{\"PassengerId\":2,\"Survived\":1,\"Pclass\":1,\"Name\":\"Cumings, Mrs. John Bradley (Florence Briggs Thayer)\",\"Sex\":\"female\",\"Age\":38.0,\"SibSp\":1,\"Parch\":0,\"Ticket\":\"PC 17599\",\"Fare\":71.2833,\"Cabin\":\"C85\",\"Embarked\":\"C\"},{\"PassengerId\":3,\"Survived\":1,\"Pclass\":3,\"Name\":\"Heikkinen, Miss. Laina\",\"Sex\":\"female\",\"Age\":26.0,\"SibSp\":0,\"Parch\":0,\"Ticket\":\"STON\/O2. 3101282\",\"Fare\":7.925,\"Cabin\":null,\"Embarked\":\"S\"},{\"PassengerId\":4,\"Survived\":1,\"Pclass\":1,\"Name\":\"Futrelle, Mrs. Jacques Heath (Lily May Peel)\",\"Sex\":\"female\",\"Age\":35.0,\"SibSp\":1,\"Parch\":0,\"Ticket\":\"113803\",\"Fare\":53.1,\"Cabin\":\"C123\",\"Embarked\":\"S\"},{\"PassengerId\":5,\"Survived\":0,\"Pclass\":3,\"Name\":\"Allen, Mr. William Henry\",\"Sex\":\"male\",\"Age\":35.0,\"SibSp\":0,\"Parch\":0,\"Ticket\":\"373450\",\"Fare\":8.05,\"Cabin\":null,\"Embarked\":\"S\"}]>>>\
+            {format_instructions}\
+            from these inputs extract the following information\
+            steps: the steps to be performed for the execution of task\
+            label: suggest label to be displayed on the graph for the required ask in less than 3 words\
+            chart_type: extract what type of chart user want to see.. if not specified then suggest which one \
+            would be suitable for the task.. should be one of these ['bar', 'line', 'doughnut', 'scatter', 'text']\
+            output text if graph cant be plotted or user required the answer in text format")),
+        AIMessage(content=("{\n\t\"steps\": \"1. Filter the dataframe to include only the rows where \'Survived\' column is equal to 1.\\n\
+        2. Group the filtered dataframe by \'Sex\' column.\\n\
+        3. Count the number of occurrences of each unique value in the \'Sex\' column.\\n\
+        4. Create a new dataframe with two columns: \'Sex\' and \'Count\', where \'Sex\' contains the unique values from the \'Sex\' column and \'Count\' contains the corresponding counts.\\n\
+        5. Select top two records from the dataframe.\",\\n\
+        6. Convert the dataset to Json and create the bar graph.\"t\
+        \"label\": \"Distribution of Males and Females who Survived\",\\n\t\
+        \"chart_type\": \"bar\"\\n}")),
+        HumanMessage(content=("The output seems not correct. the provided steps include importing matplotlib and creating the bar diagram\
+        which was clearly mentioned not to include. Also steps include selecting top two records where it was mentioned \
+        not to select top few records for output. you should output all records unless asked otherwise. Also steps include \
+        the conversion to json which should not have been included. the output should be only pandas dataframe..\
+        can you correct these and provide the response.")),
+        AIMessage(content=("{\n\t\"steps\": 1. Filter the dataframe to include only the rows where \'Survived\' column is equal to 1.\\n\
+        2. Group the filtered dataframe by \'Sex\' column.\\n\
+        3. Count the number of occurrences of each unique value in the \'Sex\' column.\\n\
+        4. Create a new dataframe with two columns: \'Sex\' and \'Count\', where \'Sex\' contains the unique values from the \'Sex\' column and \'Count\' contains the corresponding counts.\\n\
+        5. Return the new dataframe.\",\\n\t\
+        \"label\": \"Distribution of Males and Females who Survived\",\\n\t\
+        \"chart_type\": \"bar\"\\n}")),
+        HumanMessage(content=("Now the output looks perfect. This was just the way I wanted. Thanks a lot..")),
+        AIMessage(content=("Thanks for the feedback, I will double check these things next time..")),
+        HumanMessage(content=("\
+                user_input = ```draw the bar chart for distribution of pclass and for only sex=males```\
+            use below dataset for reference delimited by <<<>>>\
+            <<<[{\"PassengerId\":1,\"Survived\":0,\"Pclass\":3,\"Name\":\"Braund, Mr. Owen Harris\",\"Sex\":\"male\",\"Age\":22.0,\"SibSp\":1,\"Parch\":0,\"Ticket\":\"A\/5 21171\",\"Fare\":7.25,\"Cabin\":null,\"Embarked\":\"S\"},{\"PassengerId\":2,\"Survived\":1,\"Pclass\":1,\"Name\":\"Cumings, Mrs. John Bradley (Florence Briggs Thayer)\",\"Sex\":\"female\",\"Age\":38.0,\"SibSp\":1,\"Parch\":0,\"Ticket\":\"PC 17599\",\"Fare\":71.2833,\"Cabin\":\"C85\",\"Embarked\":\"C\"},{\"PassengerId\":3,\"Survived\":1,\"Pclass\":3,\"Name\":\"Heikkinen, Miss. Laina\",\"Sex\":\"female\",\"Age\":26.0,\"SibSp\":0,\"Parch\":0,\"Ticket\":\"STON\/O2. 3101282\",\"Fare\":7.925,\"Cabin\":null,\"Embarked\":\"S\"},{\"PassengerId\":4,\"Survived\":1,\"Pclass\":1,\"Name\":\"Futrelle, Mrs. Jacques Heath (Lily May Peel)\",\"Sex\":\"female\",\"Age\":35.0,\"SibSp\":1,\"Parch\":0,\"Ticket\":\"113803\",\"Fare\":53.1,\"Cabin\":\"C123\",\"Embarked\":\"S\"},{\"PassengerId\":5,\"Survived\":0,\"Pclass\":3,\"Name\":\"Allen, Mr. William Henry\",\"Sex\":\"male\",\"Age\":35.0,\"SibSp\":0,\"Parch\":0,\"Ticket\":\"373450\",\"Fare\":8.05,\"Cabin\":null,\"Embarked\":\"S\"}]>>>\
+            {format_instructions}\
+            from these inputs extract the following information\
+            steps: the steps to be performed for the execution of task\
+            label: suggest label to be displayed on the graph for the required ask in less than 3 words\
+            chart_type: extract what type of chart user want to see.. if not specified then suggest which one \
+            would be suitable for the task.. should be one of these ['bar', 'line', 'doughnut', 'scatter', 'text']\
+            output text if graph cant be plotted or user required the answer in text format")),
+        AIMessage(content=("{\"\n\t\"steps\": \"1. Filter the dataframe to include only rows where Sex is \'male\'\\n\
+        2. Group the data by Pclass and count the number of occurrences\\n\
+        3. Create a new dataframe with the Pclass and count columns\\n\
+        4. Sort the dataframe by Pclass\\n\
+        5. Output the resulting dataframe\",\n\t\
+        \"label\": \"Distribution of Pclass for males\",\n\t\
+        \"chart_type\": \"bar\"\n}")),
+        HumanMessage(content=("The output looks perfect. This is exactly the response I was looking for.")),
+        AIMessage(content=("Thanks! Now I am more confident about the requirement. ")),
+        HumanMessagePromptTemplate.from_template(
+            "user_input = ```{user_input}```\
+            use sample dataset below for reference delimited by <<<>>>\
+            <<<{reference_data}>>>\
+            {format_instructions}\
+            from these inputs extract the following information\
+            steps: the steps to be performed for the execution of task. stritcly remember to not include any graph or plotting related instructions..\
+            label: suggest label to be displayed on the graph for the required ask in less than 3 words\
+            chart_type: extract what type of chart user want to see.. if not specified then suggest which one \
+            would be suitable for the task.. should be one of these ['bar', 'line', 'doughnut', 'scatter', 'text']\
+            output text if graph cant be plotted or user required the answer in text format"
+        ),
+        ]
+    )
+
+    step_schema = ResponseSchema(name="steps",
+                             description="steps to be performed for the execution of task. stritcly remember to not include any graph or plotting related instructions..")
+    label_schema = ResponseSchema(name="label",
+                                description="suggest label to be displayed on the graph for the required ask")
+    chart_type_schema = ResponseSchema(name="chart_type",
+                                        description="Suggest the type of chart to be created.. if user specified\
+                                        any chart type, select that.. otherwise suggest what would be suitable for the user ask\
+                                        From the user input.. should be one of these ['bar', 'line', 'doughnut', 'scatter', 'text']\
+                                        output text if graph cant be plotted or user required the answer in text format")
+
+    response_schemas = [step_schema, label_schema, chart_type_schema]
+    output_parser = StructuredOutputParser.from_response_schemas(response_schemas)
+    format_instructions = output_parser.get_format_instructions()
+    
+    customer_messages = prompt_template.format_messages(user_input=query, 
+                                                    reference_data=jsondf, 
+                                                    format_instructions=format_instructions)
+
+    llm = ChatOpenAI(model = 'gpt-3.5-turbo-0613', temperature = 0)
+    response = llm(customer_messages)
+    json_pandassteps_charttype = output_parser.parse(response.content)
+    print("pandas steps and charttype" , json_pandassteps_charttype)
+    agent = create_pandas_dataframe_agent(                 
+                ChatOpenAI(temperature=0, model="gpt-3.5-turbo-0613"),
+                df, 
+                agent_type=AgentType.OPENAI_FUNCTIONS,
+                verbose=True,
+                return_intermediate_steps=True
+            )
+    try:
+        if json_pandassteps_charttype['chart_type'] != "text":
+            agent_response = agent({
+                'input': f"Remove any steps related to charts or plotting from the steps mentioned in triple backticks\
+                        and Execute the Python script combining all the steps after remoing the chart related steps\
+                        Please don't use any print statements. Output should be a pandas dataframe only \
+                            steps to be performed : {json_pandassteps_charttype['steps']}"
+            })
+        else:
+            chart_type, chart_label, chart_json_data = "text", None, None
+            print("The text block has been activated")
+            text_to_display = chat_with_df(query, table_name)
+            success = True
+    except langchain.schema.output_parser.OutputParserException as e:
+        success = False
+        chart_type, chart_label, chart_json_data = "text", None, None
+        text_to_display = "OutputParserException : " + str(e)[:25] + "...."
+        print(f"OutputParserException Occurred --> {str(e)}")
+    except Exception as e:
+        success = False
+        chart_type, chart_label, chart_json_data = "text", None, None
+        text_to_display = "AgentException : " + str(e)[:25] + "...."
+        print(f"Exception Occurred --> {str(e)}")
+    else:
+        graph_data_df = agent_response['intermediate_steps'][-1][-1]
+        print("intermediate steps " , agent_response['intermediate_steps'])
+        if isinstance(graph_data_df, pd.DataFrame):
+            final_data = dataframe_sanitizer(graph_data_df)
+            chart_label = json_pandassteps_charttype['label']
+            chart_type = json_pandassteps_charttype['chart_type']
+            chart_json_data = gen_chartdata(final_data, chart_label, chart_type)
+            success = True
+            text_to_display = None
+        else:
+            chart_type, chart_label, chart_json_data = "text", None, None
+            text_to_display = graph_data_df
+            success = True
+    finally:
+        final_response = dict(success=success, 
+                              chart_type=chart_type, 
+                              chart_label=chart_label,
+                              chart_json_data=chart_json_data,
+                              text_to_display=text_to_display)
+        return final_response
+
+def dataframe_sanitizer(raw_dataframe):
+    # Check if reset index is required or not
+    new_df = raw_dataframe.copy()
+    if new_df.index.name:
+        new_df = new_df.reset_index()
+    
+    print("new_df --> ", new_df)
+    # Rearrange the columns as object and int
+    object_columns = [col for col, dtype in new_df.dtypes.items() if dtype not in ['int8','int16','int32','int64', 'float64', 'float32']]
+    int_columns = [col for col, dtype in new_df.dtypes.items() if dtype in ['int8','int16','int32','int64', 'float64', 'float32']]
+    reordered_columns = object_columns + int_columns
+    print(f"object Columns , reordered columns : {object_columns}, {reordered_columns}")
+    new_df_reordered = new_df[reordered_columns]
+    print(f"New ordered columns : {new_df_reordered}")
+
+    #Check if first column is object and second column is int
+    try:
+        if (new_df_reordered.shape[1] != 2) & (new_df_reordered.dtypes[1] not in ('int8','int16','int32','int64', 'float64', 'float32')):
+            print("========Exception in shape and type==========")
+            return_string = f"The output dataframe has columns count as {new_df_reordered.shape[1]}..\
+                    and the datatypes of columns present as {new_df_reordered.dtypes}..\
+                    so cant proceed further with plotting the same.."
+            print(return_string)
+            raise Exception(return_string)
+        
+        else: 
+            new_df_reordered[new_df_reordered.columns[0]] = new_df_reordered[new_df_reordered.columns[0]].astype(str)
+            return new_df_reordered
+    except Exception as e:
+        print("Exception occurred" , str(e))

requirements.txt

@@ -0,0 +1,13 @@
+Flask==3.0.0
+Flask_Bootstrap==3.3.7.1
+langchain==0.0.281
+openai==0.27.8
+pandas==1.3.5
+plotly==5.14.1
+pyodbc==4.0.32
+python-dotenv==0.21.1
+Requests==2.31.0
+SQLAlchemy==1.4.42
+urllib3==1.26.14
+waitress==2.1.2
+tabulate==0.9.0

static/css/chat_style.css

@@ -0,0 +1,191 @@
+.sidechatpanel {
+    width: 400px;
+    position: fixed;
+    top: 56px;
+    opacity: 0.98;
+    z-index: 3;
+    right: -400px;
+    padding-bottom: 60px;
+    height: calc(100vh - 55px);
+    background-color: #2e2e2e;
+    transition: right 0.3s ease-in-out;
+    overflow: hidden;
+    display: flex;
+    flex-direction: column;
+    scroll-behavior: auto;
+    box-shadow: -5px 4px 8px 0 rgba(0, 0, 0, 0.2);
+}
+
+.sidechatpanel.show {
+    right: 0;
+}
+
+.msger-header {
+    background-color: revert;
+    color: cornsilk;
+    text-align: center;
+    margin-top: -5px;
+    padding: 12px 5px 5px 20px;
+    /* font-weight: bolder; */
+    border-radius: 15px;
+    box-shadow: 0px 4px 4px 0 rgba(22, 0, 0, 0.2);
+    font-style: oblique;
+    padding-top: -5px;
+}
+
+.msger-send-btn {
+    background-color: transparent;
+    border: none;
+    padding: 10px;
+    width: 24px;
+    height: 24px;
+    background-image: url('/static/images/send5.png');
+    background-size: cover;
+    background-repeat: no-repeat;
+}
+
+.msger-inputarea {
+    position: absolute;
+    bottom: 0;
+    left: 0;
+    width: 100%;
+    padding: 6px;
+    background-color: #000000;
+}
+
+.msger-input {
+    display: flex;
+    align-items: center;
+    border: 1px solid #ccc;
+    border-radius: 5px;
+    background-color: #cdcdca;
+    overflow: hidden;
+}
+
+.msger-input input {
+    flex: 1;
+    padding: 10px;
+    border: none;
+    background-color: transparent;
+    width: 100%;
+}
+
+.msger-input button {
+    border: none;
+    margin: 0;
+    padding: 0;
+}
+
+
+.chat-container {
+    padding-left: 15px;
+    padding-right: 15px;
+    padding-top: 10px;
+    display: flex;
+    flex-direction: column;
+    max-height: calc(100vh - 110px);
+    max-width: auto;
+    margin: 10 auto;
+    scroll-behavior: auto;
+    overflow-y: auto;
+}
+
+.chat-container::-webkit-scrollbar {
+    width: 5px;
+}
+
+.chat-container::-webkit-scrollbar-thumb {
+    background-color: #888;
+    border-radius: 5px;
+}
+
+/* Hide the scrollbar track */
+.chat-container::-webkit-scrollbar-track {
+    display: none;
+}
+
+.user-message {
+    display: flex;
+    font-size: 500;
+    margin-bottom: 10px;
+    color: gold;
+    border-radius: 5px;
+}
+
+.bot-message {
+    display: flex;
+    color: rgb(209, 243, 244);
+    margin-bottom: 10px;
+}
+
+.user-icon {
+    margin-top: 8px;
+    justify-content: center;
+    width: 24px;
+    height: 24px;
+    color: white;
+    text-align: center;
+    line-height: 40px;
+    background-image: url('/static/images/user6.png');
+    background-size: cover;
+    margin-right: 10px;
+}
+
+.bot-icon {
+    margin-top: 8px;
+    justify-content: center;
+    width: 24px;
+    height: 24px;
+    color: white;
+    text-align: center;
+    line-height: 40px;
+    background-image: url('/static/images/bot.png');
+    background-size: cover;
+    margin-right: 10px;
+}
+
+.bot-message-bubble {
+    flex: 1;
+    font-size: small;
+    font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+    background-color: transparent;
+    padding: 10px;
+    border-radius: 10px;
+    box-shadow: #1f2b37;
+    box-shadow: -4px 4px 8px 0 rgba(0, 0, 0, 0.2);
+}
+
+.user-message-bubble {
+    flex: 1;
+    font-size: small;
+    font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+    background-color: transparent;
+    padding: 10px;
+    border-radius: 10px;
+    box-shadow: #1f2b37;
+    box-shadow: -4px 4px 8px rgba(0, 0, 0, 0.2);
+}
+
+.closebtn {
+    cursor: pointer;
+    color: darkgrey;
+    font-size: 20px;
+    font-weight: bold;
+}
+
+.closebtn:hover {
+    color: coral;
+}
+
+.toggle-button {
+    position: fixed;
+    top: 65px;
+    height: 37px;
+    right: -10px;
+    background-color: #8fadc7;
+    color: white;
+    /* padding: 10px 10px; */
+    border: none;
+    border-radius: 10px;
+    cursor: pointer;
+}

static/css/data_style.css

@@ -0,0 +1,72 @@
+.tab {
+    position: relative;
+    overflow: hidden;
+    background-color: #f1f1f1;
+}
+
+.tab button {
+    background-color: inherit;
+    font-weight: bolder;
+    color:darkslategray;
+    float: left;
+    border: none;
+    outline: none;
+    cursor: pointer;
+    padding: 14px 12px;
+    transition: background-color 0.3s ease;
+}
+
+.tab button:hover {
+    background-color: #ddd;
+}
+
+.tab button.active {
+    background-color: #ccc;
+}
+
+div.dataTables_wrapper {
+    margin: 0 auto;
+}
+
+.page-item.active .page-link {
+    z-index: 1;
+    color: #fff;
+    background-color: #84888d;
+    border-color: #929ba4;
+}
+
+.fixed-button {
+    position: fixed;
+    bottom: 0;
+    left: 0;
+    width: 100%;
+    /* margin-top: 50px; */
+    padding: 15px;
+    padding-right: 50px;
+    background-color: #f8f9fa;
+    box-shadow: 0 -2px 6px rgba(0, 0, 0, 0.1);
+    z-index: 2;
+}
+
+/* Define your custom button color */
+.btn-custom-color {
+    background-color:  rgb(17, 67, 86); /* Replace 'specific-color' with your desired color */
+    color: rgb(168, 167, 167); /* Set text color to white to ensure readability on colored backgrounds */
+    border-color:  rgb(17, 67, 86); /* Set border color to match the background color */
+}
+
+/* If you want to change the color of the button on hover and active states as well */
+.btn-custom-color:hover,
+.btn-custom-color:active,
+.btn-custom-color:focus {
+    background-color:  rgb(17, 67, 86); /* Replace 'specific-hover-color' with the desired hover color */
+    color: rgb(209, 212, 214);
+    border-color:  rgb(38, 144, 185);
+}
+
+.content {
+    padding-left: 0px;
+    padding-right: 0px;
+}
+
+

static/css/home_style.css

@@ -0,0 +1,32 @@
+/* body {
+    font-family: Arial, sans-serif;
+} */
+.container {
+    max-width: 1100px;
+    margin: 0 auto;
+    padding: 0 auto;
+}
+
+.row {
+    margin-bottom: 20px;
+}
+
+.left-column {
+    background-color: #f8f9fa;
+    padding: 20px;
+    position: relative;
+}
+  
+.right-column {
+    padding: 20px;
+}
+
+h1 {
+    text-align: center;
+    margin-bottom: 30px;
+}
+
+form {
+    margin-bottom: 20px;
+}
+

static/css/layout_style.css

@@ -0,0 +1,50 @@
+/* .navbar {
+    background-color: black;
+} */
+
+/* .navbar-toggler {
+    border-color: rgba(255, 255, 255, 0.5);
+}
+
+.navbar-toggler-icon {
+    background-image: url("data:image/svg+xml,%3Csvg viewBox='0 0 30 30' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath stroke='rgba(255, 255, 255, 0.5)' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3E%3C/svg%3E");
+}
+
+*/
+
+/* .navbar-toggler {
+    border-color: rgba(255, 18, 18, 0.5);
+} */
+
+/* .navbar-nav .nav-link {
+    padding: 0.5rem 0.75rem;
+    margin-left: 10px;
+    color: rgb(221, 234, 239);
+} */
+
+/* .navbar-nav .nav-link:hover {
+    color: rgb(234, 128, 41);
+} */
+
+/* .navbar-nav .nav-link.active {
+    font-weight: bold;  
+}  */
+/* 
+.navbar-nav .nav-link.active {
+    font-weight: bold !important;
+    color: rgb(235, 80, 13) !important;
+    background-color: red;
+  } */
+
+/* .navbar-brand {
+    padding-left: 100px;
+} */
+
+#navbarNav {
+    padding-right: 110px;
+}
+
+.disabled {
+    pointer-events: none;  /* Disable click events */
+    opacity: 0.6;         /* Visually indicate disabled state */
+}

static/css/validate_style.css

@@ -0,0 +1,78 @@
+.tab {
+    position: relative;
+    overflow: hidden;
+    background-color: #f1f1f1;
+}
+
+.tab button {
+    background-color: inherit;
+    float: left;
+    border: none;
+    outline: none;
+    cursor: pointer;
+    padding: 14px 16px;
+    transition: background-color 0.3s ease;
+}
+
+.tab button:hover {
+    background-color: #ddd;
+}
+
+.tab button.active {
+    background-color: #ccc;
+}
+
+body {
+    background-color: #f0f0f0;
+  }
+
+  .left_tblsidebar {
+    background-color:cadetblue;
+    height: 100vh;
+    position: fixed;
+    top: 0;
+    left: 0px;
+    width: 100px;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    padding-top: 80px;
+    z-index:0;
+  }
+
+  /* .nav-link { */
+  .val_leftsidebarcls {
+    color: #fff;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    text-decoration:none;
+    padding-top: 10px;
+    padding-bottom: 10px;
+  }
+
+  /* .nav-link span { */
+  .val_leftsidebarcls span{
+    display: block;
+    width: 0;
+    text-align: center;
+    overflow: hidden;
+    white-space: nowrap;
+  }
+
+  /* .nav-link:hover span{ */
+  .val_leftsidebarcls:hover span{
+    display: block;
+    width:fit-content;
+  }
+
+  .dashboard-content{
+    margin-left: 20px;
+  }
+
+  div.headstats{
+    text-align: center;
+    margin-left: 10px;
+    margin-right: 10px;
+    /* background-color: floralwhite; */
+  }

static/js/data_script.js

@@ -0,0 +1,83 @@
+function openTab(evt, tabName) {
+    var i, tabcontent, tablinks;
+    
+    // Hide all content
+    tabcontent = document.getElementsByClassName("content");
+    for (i = 0; i < tabcontent.length; i++) {
+      tabcontent[i].style.display = "none";
+    }
+    
+    // Remove "active" class from all buttons
+    tablinks = document.getElementsByClassName("tablinks");
+    for (i = 0; i < tablinks.length; i++) {
+      tablinks[i].className = tablinks[i].className.replace(" active", "");
+    }
+    
+    // Show the selected content and mark the button as active
+    document.getElementById(tabName).style.display = "block";
+    evt.currentTarget.className += " active";
+  };
+
+function redirecttovalidation() {
+    window.location.href = '/validate_' + req_tables;
+}
+
+// chat function which loads user message and gets bot response by invoking the URI
+document.addEventListener("DOMContentLoaded", function () {
+  const msgerForm = document.querySelector(".msger-inputarea");
+  const msgerInput = document.querySelector("#textInput");
+  const msgerChat = document.querySelector(".chat-container");
+
+  msgerForm.addEventListener("submit", function (event) {
+    event.preventDefault();
+    const msgText = msgerInput.value;
+    if (!msgText) return;
+
+    appendMessage(msgText);
+    msgerInput.value = "";
+    botResponse(msgText);
+  });
+
+  function appendMessage(text) {
+    const msgHTML = `
+      <div class="user-message">
+        <div class="user-icon"></div>
+        <div class="user-message-bubble">
+          ${text}
+        </div>
+      </div>
+    `;
+    msgerChat.insertAdjacentHTML("beforeend", msgHTML);
+    msgerChat.scrollTop += 500;
+  }
+
+  function botResponse(rawText) {
+    const selectElement = document.getElementById("table-dropdown");
+    const desiredValue = document.getElementById("table-dropdown").value;
+    let selected_tbl_name = '';
+    for (let i = 0; i < selectElement.options.length; i++) {
+      const option = selectElement.options[i];
+      if (option.value === desiredValue) {
+        // Found a match, get the innerHTML
+        selected_tbl_name = option.innerHTML.toLowerCase();
+        break; // Exit the loop since we found a match
+      }
+    }
+
+
+    $.get("/get_llmresponse", { msg: rawText, table_selected: selected_tbl_name }).done(function (data) {
+      console.log(rawText);
+      console.log(data);
+      const msgHTML = `
+        <div class="bot-message">
+          <div class="bot-icon"></div>
+          <div class="bot-message-bubble">
+            ${data}
+          </div>
+        </div>
+      `;
+      msgerChat.insertAdjacentHTML("beforeend", msgHTML);
+      msgerChat.scrollTop += 500;
+    });
+  }
+});

static/js/home_script.js

@@ -0,0 +1,47 @@
+$(function() {
+    var start = moment().subtract(29, 'days');
+    var end = moment();
+    function cb(start, end) {
+        $('#reportrange span').html(start.format('MMM D, YYYY') + ' - ' + end.format('MMM D, YYYY'));
+        $('input[name="calendar_value"]').val(start.format('YYYY-MM-DD') + ':' + end.format('YYYY-MM-DD'));
+    }
+    // Set the value for the calendar_value input field
+    $('#reportrange').daterangepicker({
+        startDate: start,
+        endDate: end,
+        ranges: {
+        //    'Today': [moment(), moment()],
+        //    'Yesterday': [moment().subtract(1, 'days'), moment().subtract(1, 'days')],
+           'Last 7 Days': [moment().subtract(6, 'days'), moment()],
+           'Last 30 Days': [moment().subtract(29, 'days'), moment()],
+           'This Month': [moment().startOf('month'), moment().endOf('month')],
+           'Last Month': [moment().subtract(1, 'month').startOf('month'), moment().subtract(1, 'month').endOf('month')],
+           'This Quarter': [moment().startOf('quarter'), moment().endOf('quarter')],
+           'Last Quarter': [moment().subtract(1, 'quarter').startOf('quarter'), moment().subtract(1, 'quarter').endOf('quarter')]
+        }
+    }, cb);
+    cb(start, end);
+});
+
+function toggleFormElements() {
+    var toggleSwitch = document.getElementById('toggleSwitch');
+    var dropdowns = document.querySelectorAll('form select');
+    var textbox = document.querySelector('form textarea');
+    var submitButton = document.querySelector('form button[type="submit"]');
+
+    if (toggleSwitch.checked) {
+        for (var i = 0; i < dropdowns.length; i++) {
+            dropdowns[i].disabled = true;
+        }
+        //reportrange.disabled = true;
+        textbox.disabled = false;
+        submitButton.disabled = false;
+    } else {
+        for (var i = 0; i < dropdowns.length; i++) {
+            dropdowns[i].disabled = false;
+        }
+        //reportrange.disabled = false;
+        textbox.disabled = true;
+        submitButton.disabled = false;
+    }
+};

static/js/mytest.js

@@ -0,0 +1,79 @@
+function openTab(evt, tabName) {
+    var i, tabcontent, tablinks;
+    
+    // Hide all content
+    tabcontent = document.getElementsByClassName("content");
+    for (i = 0; i < tabcontent.length; i++) {
+      tabcontent[i].style.display = "none";
+    }
+    
+    // Remove "active" class from all buttons
+    tablinks = document.getElementsByClassName("tablinks");
+    for (i = 0; i < tablinks.length; i++) {
+      tablinks[i].className = tablinks[i].className.replace(" active", "");
+    }
+    
+    // Show the selected content and mark the button as active
+    document.getElementById(tabName).style.display = "block";
+    evt.currentTarget.className += " active";
+  };
+
+// document.getElementById('slssl').value = "option8";;
+
+$(function() {
+    var start = moment().subtract(29, 'days');
+    var end = moment();
+    function cb(start, end) {
+        $('#reportrange span').html(start.format('MMM D, YYYY') + ' - ' + end.format('MMM D, YYYY'));
+        $('input[name="calendar_value"]').val(start.format('YYYY-MM-DD') + ':' + end.format('YYYY-MM-DD'));
+    }
+    // Set the value for the calendar_value input field
+    $('#reportrange').daterangepicker({
+        startDate: start,
+        endDate: end,
+        ranges: {
+        //    'Today': [moment(), moment()],
+        //    'Yesterday': [moment().subtract(1, 'days'), moment().subtract(1, 'days')],
+           'Last 7 Days': [moment().subtract(6, 'days'), moment()],
+           'Last 30 Days': [moment().subtract(29, 'days'), moment()],
+           'This Month': [moment().startOf('month'), moment().endOf('month')],
+           'Last Month': [moment().subtract(1, 'month').startOf('month'), moment().subtract(1, 'month').endOf('month')],
+           'This Quarter': [moment().startOf('quarter'), moment().endOf('quarter')],
+           'Last Quarter': [moment().subtract(1, 'quarter').startOf('quarter'), moment().subtract(1, 'quarter').endOf('quarter')]
+        }
+    }, cb);
+    cb(start, end);
+});
+
+function downloadTableData() {
+    window.location.href = "/downloaddata";
+  };
+
+function toggleFormElements() {
+    var toggleSwitch = document.getElementById('toggleSwitch');
+    var dropdowns = document.querySelectorAll('form select');
+    var textbox = document.querySelector('form textarea');
+    var submitButton = document.querySelector('form button[type="submit"]');
+
+    if (toggleSwitch.checked) {
+        for (var i = 0; i < dropdowns.length; i++) {
+            dropdowns[i].disabled = true;
+        }
+        //reportrange.disabled = true;
+        textbox.disabled = false;
+        submitButton.disabled = false;
+    } else {
+        for (var i = 0; i < dropdowns.length; i++) {
+            dropdowns[i].disabled = false;
+        }
+        //reportrange.disabled = false;
+        textbox.disabled = true;
+        submitButton.disabled = false;
+    }
+};
+
+function redirecttovalidation() {
+    window.location.href = '/validation';
+}
+
+document.getElementById("tab1").click();

static/js/validate_script.js

@@ -0,0 +1,723 @@
+// Chart js | script to populate stacked horizontal bar chart for validation results
+document.addEventListener('DOMContentLoaded', function() {
+    var labels = [];
+    var pct_pattern_mismatch = [];
+    var pct_null_values = [];
+    var pct_good_data = [];
+    var pct_column_notfound = [];
+    var pct_dtype_issue = [];
+    var pct_neg_values = [];
+    var pct_value_notdatetime = [];
+    var pct_value_unknown = [];
+
+    // Extract data from JSON
+    jsonData.forEach(function(item) {
+        labels.push(item.column);
+        pct_pattern_mismatch.push(item.pct_pattern_mismatch);
+        pct_null_values.push(item.pct_null_values);
+        pct_good_data.push(item.pct_good_data);
+        pct_column_notfound.push(item.pct_column_notfound);
+        pct_dtype_issue.push(item.pct_dtype_issue);
+        pct_neg_values.push(item.pct_neg_values);
+        pct_value_notdatetime.push(item.pct_value_notdatetime);
+        pct_value_unknown.push(item.pct_value_unknown);
+    });
+
+    new Chart(
+        document.getElementById('validationbar'), {
+        type: 'bar',
+        data: {
+        labels: labels,
+        datasets: [{
+            label: 'Pattern Issue',
+            data: pct_pattern_mismatch,
+            backgroundColor: "rgba(152, 134, 123, 0.75)", // Maroon shade
+            hoverBackgroundColor: "rgba(152, 134, 123, 1)",
+            barPercentage:0.65,
+        }, {
+            label: 'Null Values',
+            data: pct_null_values,
+            backgroundColor: "rgba(109, 104, 117, 0.75)", // Gray shade
+            hoverBackgroundColor: "rgba(109, 104, 117, 1)",
+            barPercentage:0.65,
+        }, {
+            label: "Good Data",
+            data: pct_good_data,
+            backgroundColor: "rgba(40, 75, 99, 0.75)", // Blue shade
+            hoverBackgroundColor: "rgba(40, 75, 99, 1)",
+            barPercentage:0.65,
+        }, {
+            label: "Column_unavailable",
+            data: pct_column_notfound,
+            backgroundColor: "rgba(255, 205, 178, 0.75)", // Light Maroon shade
+            hoverBackgroundColor: "rgba(255, 205, 178, 1)",
+            barPercentage:0.65,
+        }, {
+            label: "DataType Issue",
+            data: pct_dtype_issue,
+            backgroundColor: "rgba(111, 67, 76,0.75)", // The original color
+            hoverBackgroundColor: "rgba(111, 67, 76,1)",
+            barPercentage:0.65,
+        }, {
+            label: "Found Negative Values",
+            data: pct_neg_values,
+            backgroundColor: "rgba(255, 180, 162, 0.75)", // Reddish shade
+            hoverBackgroundColor: "rgba(255, 180, 162, 1)",
+            barPercentage:0.65,
+        }, {
+            label: "Not Datetime",
+            data: pct_value_notdatetime,
+            backgroundColor: "rgba(229, 152, 155, 0.75)", // Reddish shade
+            hoverBackgroundColor: "rgba(229, 152, 155, 1)",
+            barPercentage:0.65,
+        }, {
+            label: "Unknown value",
+            data: pct_value_unknown,
+            backgroundColor: "rgba(235, 152, 155, 0.75)", // Reddish shade
+            hoverBackgroundColor: "rgba(235, 152, 155, 1)",
+            barPercentage:0.65,
+        }]
+        
+        },
+        options: {
+        indexAxis: 'y',
+        scales: {
+            x: {
+            stacked: true
+            },
+            y: {
+            stacked: true
+            }
+        },
+        plugins: {
+            legend: {
+            display: true,
+            position: 'top', // Place the legend to the top
+            align: 'end', // Place the legend to the right end 
+            labels: {
+                // Use 'dataset' to display dataset labels instead of 'undefined'
+                generateLabels: function (chart) {
+                var datasets = chart.data.datasets;
+                var labels = chart.data.labels;
+                var legends = [];
+
+                datasets.forEach(function (dataset, datasetIndex) {
+                    // Check if any non-zero value is present in the dataset
+                    var hasNonZeroValue = dataset.data.some(function(value) {
+                        return value !== 0;
+                    });
+    
+                    if (hasNonZeroValue) {
+                        legends.push({
+                            text: dataset.label,
+                            fillStyle: dataset.backgroundColor,
+                            hidden: !chart.isDatasetVisible(datasetIndex),
+                            lineCap: dataset.borderCapStyle,
+                            lineDash: dataset.borderDash,
+                            lineDashOffset: dataset.borderDashOffset,
+                            lineJoin: dataset.borderJoinStyle,
+                            lineWidth: dataset.borderWidth,
+                            strokeStyle: dataset.borderColor,
+                            pointStyle: dataset.pointStyle,
+                            rotation: dataset.rotation,
+                            datasetIndex: datasetIndex
+                        });
+                    }
+                });
+                return legends;
+                }
+            },
+            }
+        },
+        },
+    })
+});
+
+// Chart js | script to populate doughnut chart 
+document.addEventListener('DOMContentLoaded', function() {
+    
+    var data = {
+        labels: ['Structured', 'Unstructured'],
+        datasets: [{
+            data: [58, 42],
+            backgroundColor: ['rgba(63,103,126,1)', 'rgba(163,103,126,1)', 'rgba(63,203,226,1)', 'rgba(90,34,21,1)', 'rgba(200,150,50,1)', 'rgba(235,91,56,1)', 'rgba(137,196,244,1)', 'rgba(245,203,83,1)', 'rgba(142,69,173,1)', 'rgba(76,175,80,1)'],
+            hoverBackgroundColor: ['rgba(50,90,100,1)', 'rgba(140,85,100,1)', 'rgba(46,185,235,1)', 'rgba(45,21,231,1)', 'rgba(190,120,40,1)', 'rgba(205,80,47,1)', 'rgba(101,154,204,1)', 'rgba(221,183,60,1)', 'rgba(110,49,147,1)', 'rgba(60,136,63,1)']
+        }]
+    };
+
+    new Chart(document.getElementById('doughnutchart_strvsunstr'), {
+        type: 'doughnut',
+        data: data,
+        options: {
+            plugins: {
+                legend: {
+                    display: true,
+                    position: 'bottom',
+                    align: 'center',
+                }
+            },
+            tooltips: {
+                callbacks: {
+                    label: function (context) {
+                        var label = context.label || '';
+                        var value = context.formattedValue;
+                        return label + ': ' + value;
+                    }
+                }
+            }
+        }
+    });
+});
+
+// Chart js | script to populate doughnut chart 
+document.addEventListener('DOMContentLoaded', function() {
+    var ranks = [];
+    var cnt_rank_employee = [];
+
+    // Extract data from JSON
+    json_rankwise_empdist.forEach(function(item) {
+        ranks.push(item.Rank);
+        cnt_rank_employee.push(item.count_rank);
+    });
+
+
+    var data = {
+        labels: ranks,
+        datasets: [{
+            data: cnt_rank_employee,
+            backgroundColor: ['rgba(63,103,126,1)', 'rgba(163,103,126,1)', 'rgba(63,203,226,1)', 'rgba(90,34,21,1)', 'rgba(200,150,50,1)', 'rgba(235,91,56,1)', 'rgba(137,196,244,1)', 'rgba(245,203,83,1)', 'rgba(142,69,173,1)', 'rgba(76,175,80,1)'],
+            hoverBackgroundColor: ['rgba(50,90,100,1)', 'rgba(140,85,100,1)', 'rgba(46,185,235,1)', 'rgba(45,21,231,1)', 'rgba(190,120,40,1)', 'rgba(205,80,47,1)', 'rgba(101,154,204,1)', 'rgba(221,183,60,1)', 'rgba(110,49,147,1)', 'rgba(60,136,63,1)']
+        }]
+    };
+
+    new Chart(document.getElementById('doughnutChart'), {
+        type: 'doughnut',
+        data: data,
+        options: {
+            plugins: {
+                legend: {
+                    display: true,
+                    position: 'bottom',
+                    align: 'center',
+                }
+            },
+            tooltips: {
+                callbacks: {
+                    label: function (context) {
+                        var label = context.label || '';
+                        var value = context.formattedValue;
+                        return label + ': ' + value;
+                    }
+                }
+            }
+        }
+    });
+});
+
+// Badges | Doughnut chart script for pillar distribution 
+document.addEventListener('DOMContentLoaded', function() {
+    var pillar = [];
+    var pct_gui = [];
+    json_pillar_data.forEach(function(item) {
+        pillar.push(item.Pillar);
+        pct_gui.push(item.pct_gui);
+    });
+
+    var data = {
+        labels: pillar,
+        datasets: [{
+            data: pct_gui,
+            backgroundColor: ['rgba(63,103,126,1)', 'rgba(163,103,126,1)', 'rgba(63,203,226,1)', 'rgba(90,34,21,1)', 'rgba(200,150,50,1)'],
+            hoverBackgroundColor: ['rgba(50,90,100,1)', 'rgba(140,85,100,1)', 'rgba(46,185,235,1)', 'rgba(45,21,231,1)', 'rgba(190,120,40,1)']
+        }]
+    };
+
+    new Chart(document.getElementById('doughnutChart2'), {
+        type: 'doughnut',
+        data: data,
+        options: {
+            plugins: {
+                legend: {
+                    display: true,
+                    position: 'bottom',
+                    align: 'center',
+                }
+            },
+            tooltips: {
+                callbacks: {
+                    label: function (context) {
+                        var label = context.label || '';
+                        var value = context.formattedValue;
+                        return label + ': ' + value;
+                    }
+                }
+            }
+        }
+    });
+});
+
+// Badges | Bar chart for badge earned per month
+document.addEventListener('DOMContentLoaded', function() {
+    var month = [];
+    var cnt_gui_badgeinitiated = [];
+    var cnt_gui_badgeawarded = [];
+
+    json_badgecompletion_data.forEach(function(item) {
+        month.push(item.Month);
+        cnt_gui_badgeinitiated.push(item.cnt_gui_badgeinitiated);
+        cnt_gui_badgeawarded.push(item.cnt_gui_badgeawarded);
+    }); 
+
+    new Chart(document.getElementById('barchart_badgecompletionpermonth'), {
+        type: 'line',
+        data: {
+            labels: month,
+            datasets: [{
+                label: 'BadgeInitiated',
+                data: cnt_gui_badgeinitiated,
+                borderColor: 'rgba(154, 59, 59, 0.55)',
+                backgroundColor: 'rgba(154, 59, 59, 0.2)',
+                borderWidth: 7,
+                pointRadius: 4,
+                pointHoverRadius: 7,
+                fill:true,
+                
+            },{
+                label: 'BadgeAwarded',
+                data: cnt_gui_badgeawarded,
+                borderColor: 'rgba(38,87,124, 0.55)',
+                backgroundColor: 'rgba(38,87,124,0.2)',
+                borderWidth: 7,
+                pointRadius: 4,
+                pointHoverRadius: 7,
+                fill:true,
+            },
+            ]
+        },
+        options: {
+            scales: {
+                y: {
+                    beginAtZero: true
+                }
+            },
+        },
+        
+    });
+});
+
+document.addEventListener('DOMContentLoaded', function() {
+    new Chart(document.getElementById('doughnutChart4'), {
+        type: 'doughnut',
+        data: data,
+        options: {
+            plugins: {
+                legend: {
+                    display: true,
+                    position: 'top',
+                    align: 'right',
+                }
+            },
+            tooltips: {
+                callbacks: {
+                    label: function (context) {
+                        var label = context.label || '';
+                        var value = context.formattedValue;
+                        return label + ': ' + value;
+                    }
+                }
+            }
+        }
+    });
+});
+
+document.addEventListener('DOMContentLoaded', function() {
+    // Chart js | populating the line chart across each month with their rank
+    // Get the canvas element and context for the line chart
+    var labels = [];
+    var dist_staff = [];
+    var dist_senior = [];
+    var dist_manager = [];
+    var dist_seniormanager = [];
+    var dist_director = [];
+    var dist_partner = [];
+
+    // Extract data from JSON
+    jsonEmpDist.forEach(function(item) {
+        labels.push(item.Month);
+        dist_staff.push(item.Staff);
+        dist_senior.push(item.Senior);
+        dist_manager.push(item.Manager);
+        dist_seniormanager.push(item.SeniorManager);
+        dist_director.push(item.Director);
+        dist_partner.push(item.Partner);
+    });
+
+    var canvas = document.getElementById('line_rankdistribution');
+    var ctx = canvas.getContext('2d');
+    // Data for the multiple line charts                  
+    var data = {
+        labels: labels,
+        datasets: [
+            {
+                label: 'Staff',
+                data: dist_staff,
+                borderColor: 'rgba(0, 102, 153, 0.5)',
+                backgroundColor: 'rgba(0, 102, 153, 0.2)',
+                borderWidth: 7,
+                pointRadius: 7,
+                pointHoverRadius: 8,
+                // fill:true,
+            },
+            {
+                label: 'Senior',
+                data: dist_senior,
+                borderColor: 'rgba(0, 204, 204, 0.5)',
+                backgroundColor: 'rgba(0, 204, 204,0.2)',
+                borderWidth: 7,
+                pointRadius: 7,
+                pointHoverRadius: 8,
+                // fill: true,
+            },
+            // {
+            //     label: 'Manager',
+            //     data: dist_manager,
+            //     borderColor: 'rgba(204, 102, 0, 0.5)',
+            //     // backgroundColor: 'rgba(77, 255, 255,0.2)',
+            //     borderWidth: 5,
+            //     pointRadius: 5,
+            //     // pointHoverRadius: 6,
+            // },
+            {
+                label: 'Senior Manager',
+                data: dist_seniormanager,
+                borderColor: 'rgba(255, 204, 102, 0.5)',
+                backgroundColor: 'rgba(255, 204, 102,0.2)',
+                borderWidth: 7,
+                pointRadius: 7,
+                pointHoverRadius: 8,
+                // fill:true,
+            },
+            {
+                label: 'Total Count',
+                data: [55, 60, 65, 50, 60, 61, 50],
+                backgroundColor: 'rgba(63,103,126,0.8)',
+                borderWidth: 1,
+                barPercentage: 0.6,
+                categoryPercentage: 0.8,
+                type: 'bar'
+            },
+        ]
+    };
+
+    // Create the line chart
+    var lineChart = new Chart(ctx, {
+        type: 'line',
+        data: data,
+        options: {
+            elements: {
+                line: {
+                    tension: 0.4, // Adjust the tension for smooth curves
+                },
+            },
+            scales: {
+                x: {
+                    display: true,
+                    title: {
+                        display: true,
+                    },
+                },
+                y: {
+                    beginAtZero: true, // Begin y-axis at zero
+                    display: true,
+                    title: {
+                        display: true,
+                        text: 'Employee Count',
+                    },
+                },
+            },
+        }
+    });
+});
+
+// Calculate cumulative percentage
+document.addEventListener('DOMContentLoaded', function() {
+    var data = {
+        labels: ['Bengaluru', 'Mumbai', 'Kolkata', 'Hyderabad', 'Gurgaon', 'Noida', 'Pune'],
+        datasets: [
+            {
+                label: 'Employee Count',
+                data: [2541, 1965, 1809, 1592, 1294, 762, 147],
+                backgroundColor: 'rgba(0, 68, 102, 0.7)',
+                yAxisID: 'primary',
+                barPercentage: 0.6,
+            },
+            {
+                label: 'Cumulative Percentage',
+                data: [25,44,62, 78, 91, 99, 100],
+                borderColor: 'rgba(163,103,126,0.7)',
+                backgroundColor: 'transparent',
+                borderWidth: 4,
+                pointRadius: 5,
+                yAxisID: 'secondary',
+                type: 'line',
+            },
+        ],
+    };
+
+    // Get the canvas context
+    var ctx = document.getElementById('pareto_citydistribution').getContext('2d');
+
+    // Create the multi-axis Pareto diagram with options
+    var pareto_citydistribution = new Chart(ctx, {
+        type: 'bar',
+        data: data,
+        options: {
+            scales: {
+                x: {
+                    display: true,
+                    title: {
+                        display: true,
+                        text: 'City',
+                    },
+                },
+                primary: {
+                    type: 'linear',
+                    position: 'left',
+                    beginAtZero: true,
+                    display: true,
+                    title: {
+                        display: true,
+                        text: 'Employee Count',
+                    },
+                },
+                secondary: {
+                    type: 'linear',
+                    position: 'right',
+                    beginAtZero: true,
+                    display: true,
+                    title: {
+                        display: true,
+                        text: 'Cumulative Percentage',
+                    },
+                },
+            },
+            plugins: {
+                legend: {
+                    display: true,
+                },
+            },
+        },
+    });
+});
+
+// chat function which loads user message and gets bot response by invoking the URI
+document.addEventListener("DOMContentLoaded", function () {
+    const msgerForm = document.querySelector(".msger-inputarea");
+    const msgerInput = document.querySelector("#textInput");
+    const msgerChat = document.querySelector(".chat-container");
+  
+    msgerForm.addEventListener("submit", function (event) {
+      event.preventDefault();
+      const msgText = msgerInput.value;
+      if (!msgText) return;
+  
+      appendMessage(msgText);
+      msgerInput.value = "";
+      botResponse(msgText);
+    });
+  
+    function appendMessage(text) {
+      const msgHTML = `
+        <div class="user-message">
+          <div class="user-icon"></div>
+          <div class="user-message-bubble">
+            ${text}
+          </div>
+        </div>
+      `;
+      
+      msgerChat.insertAdjacentHTML("beforeend", msgHTML);
+      msgerChat.scrollTop += 500;
+
+      setTimeout(function() {
+        const loadHTML = `
+           <div id="loading-div">
+                 <div class="spinner-grow spinner-grow-sm text-light" role="status" style="margin-left:40px">
+                     <span class="visually-hidden">Loading...</span>
+                 </div>
+                 <div class="spinner-border spinner-border-sm text-light" role="status">
+                     <span class="visually-hidden">Loading...</span>
+                 </div>
+             </div>
+        `;
+        msgerChat.insertAdjacentHTML("beforeend", loadHTML);
+        msgerChat.scrollTop += 500;
+        }, 1000);
+    }
+  
+    function botResponse(rawText) {
+
+        // const table_selected = document.getElementById("table-dropdown").value;
+        const url = window.location.href;
+        const parts = url.split('/');
+        const lastPart = parts[parts.length - 1];
+        // Remove any query parameters (e.g., '?foo=bar') by splitting at '?'
+        const withoutQuery = lastPart.split('?')[0];
+        // Assign the substring to table_selected
+        table_selected = withoutQuery.replace('validate_', '');;
+  
+      $.get("/get_val_llmresponse", { msg: rawText, table_selected: table_selected }).done(function (data) {
+        // Create a new Date object to represent the current date and time
+        const currentDate = new Date();
+        // Get various components of the current date and time
+        const year = currentDate.getFullYear();        // Get the current year (e.g., 2023)
+        const month = currentDate.getMonth() + 1;      // Get the current month (0-11, add 1 for January to December)
+        const day = currentDate.getDate();             // Get the current day of the month (1-31)
+        const hours = currentDate.getHours();           // Get the current hour (0-23)
+        const minutes = currentDate.getMinutes();       // Get the current minute (0-59)
+        const seconds = currentDate.getSeconds();       // Get the current second (0-59)
+        const milliseconds = currentDate.getMilliseconds(); // Get the current millisecond (0-999)
+        const timestamp_now = `${year}${month}${day}-${hours}${minutes}${seconds}${milliseconds}`
+        const canvasid = "canvasid_" + timestamp_now
+        
+        var responseData = JSON.parse(data);
+
+        var issuccess = responseData.success;
+        var chart_type = responseData.chart_type;
+        var chart_label = responseData.chart_label;
+        var text_to_display = responseData.text_to_display;
+
+        if (chart_type === 'text') {
+            if (text_to_display === null || text_to_display === '') {
+                text_to_display = '<span class="badge badge-danger">Error</span> Please try modifying the prompt with more details.';
+            }
+          }
+        
+        var chart_data = JSON.parse(responseData.chart_json_data);
+
+        // clear the loading icon
+        // Get the element by its ID
+        var element = document.getElementById("loading-div");
+        // Check if the element exists before attempting to remove it
+        if (element) {
+            element.remove(); // Remove the element
+        } else {
+            console.log("Loading div not found");
+        }
+
+        const msgHTML = `
+          <div class="bot-message">
+            <div class="bot-icon"></div>
+            <div class="bot-message-bubble">
+            ${chart_type !=='text' ? `<canvas id="${canvasid}" height="235"></canvas>` : text_to_display}
+            </div>
+          </div>
+        `;
+        msgerChat.insertAdjacentHTML("beforeend", msgHTML);
+        msgerChat.scrollTop += 500;
+
+        // Create the chart for this message
+        new Chart(document.getElementById(canvasid), {
+            // type: 'line',
+            data: chart_data,
+            options: {
+                scales: {
+                    y: {
+                        beginAtZero: true
+                    }
+                },
+                plugins: {
+                    legend: {
+                        display: true,
+                        position: 'bottom',
+                        align: 'center',
+                    },
+                },
+            }
+        });
+      });
+    }
+  });
+  
+
+// Miscellanous | Bar chart count of GUI matching with 
+document.addEventListener('DOMContentLoaded', function() {
+    var tableinconsideration = ['Workforce', 'GTE', 'TR', 'Badges', 'Learning', 'LAT', 'WorkExperience'];
+    var pct_commongui = [100, 98, 73, 89, 75, 70, 78];
+
+    // json_badgecompletion_data.forEach(function(item) {
+    //     month.push(item.Month);
+    //     cnt_gui_badgeinitiated.push(item.cnt_gui_badgeinitiated);
+    //     cnt_gui_badgeawarded.push(item.cnt_gui_badgeawarded);
+    // }); 
+
+    new Chart(document.getElementById('alltable_matchbar'), {
+        type: 'bar',
+        data: {
+            labels: tableinconsideration,
+            datasets: [{
+                label: 'Percentage of GUIs matching with other tables',
+                data: pct_commongui,
+                borderColor: 'rgba(0, 68, 102, 0.8)',
+                backgroundColor: pct_commongui.map(value => value < 75 ? '#EE9322' : 'rgba(0, 68, 102, 0.6)'),
+                barPercentage: 0.65
+            }]
+        },
+        options: {
+            indexAxis: 'y',
+            scales: {
+                x: {
+                    beginAtZero: true
+                }
+            },
+            plugins: {
+                verticalLine: {
+                    x: 75, // The x-coordinate where you want to draw the line
+                    borderColor: '#FF9B50', // Line color
+                    borderWidth: 4, // Line width
+                    label: {
+                        text: '75%', // Label text
+                        position: 'top', // Label position
+                        backgroundColor: 'white', // Label background color
+                    },
+                }
+            },
+        },
+        
+    plugins: [{
+        beforeDraw: (chart) => {
+            const line = chart.options.plugins.verticalLine;
+            if (line) {
+                const ctx = chart.ctx;
+                const x = chart.scales.x.getPixelForValue(line.x);
+                // Set the line to be dotted
+                ctx.setLineDash([10, 5]); // Adjust the numbers to change the dash pattern
+
+                // Draw the vertical line
+                ctx.save();
+                ctx.beginPath();
+                ctx.strokeStyle = line.borderColor;
+                ctx.lineWidth = line.borderWidth;
+                ctx.moveTo(x, chart.chartArea.top);
+                ctx.lineTo(x, chart.chartArea.bottom);
+                ctx.stroke();
+                ctx.restore();
+
+                // Draw the label
+                if (line.label) {
+                    const labelX = x + 5; // Adjust the label's x-coordinate for proper positioning
+                    const labelY = chart.chartArea.top + 145; // Adjust the label's y-coordinate for proper positioning
+
+                    ctx.fillStyle = line.label.backgroundColor;
+                    ctx.fillRect(labelX, labelY, 25, 15);
+
+                    ctx.textAlign = 'left';
+                    ctx.font = '12px Arial';
+                    ctx.fillStyle = 'black';
+                    ctx.fillText(line.label.text, labelX + 5, labelY + 12);
+                }
+            }
+        }
+    }],
+});
+})

templates/chatfile8.html

@@ -0,0 +1,63 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/chat_style.css') }}">
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css">
+    <title>Collapsible Chat Panel</title>
+</head>
+
+<body>
+    <button class="btn btn-primary toggle-button" type="button" id="toggleButton">
+        <i class="fas fa-arrow-left"></i>
+    </button>
+    <div class="container">
+            <div class="sidechatpanel" id="sidechatpanel">
+                <div class="msger-header">
+                    <span class="">LaborEconomicsBot</span>
+                    <a href="javascript:void(0)" class="closebtn" onclick="closeNav()"
+                        style="float: right; margin-right: 15px;">
+                        <i class="fas fa-times"></i>
+                    </a>
+                </div>
+                <div class="chat-container">
+                    <div class="bot-message">
+                        <div class="bot-icon"></div>
+                        <div class="bot-message-bubble">
+                            Ask any question related to labor ecomomics Data!
+                        </div>
+                    </div>
+                </div>
+
+                <form class="msger-inputarea" method="POST">
+                    <div class="msger-input">
+                        <input type="text" id="textInput" autocomplete="off" placeholder="Enter your message...">
+                        <!-- Hidden field to capture any other value from the web page -->
+                        <input type="hidden" name="hiddenField" id="hiddenField">
+                        <button type="submit" class="msger-send-btn"></button>
+                    </div>
+                </form>
+            </div>
+        <!-- </div> -->
+    </div>
+
+    <script src='https://use.fontawesome.com/releases/v5.0.13/js/all.js'></script>
+    <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
+    <script>
+        const toggleButton = document.getElementById('toggleButton');
+        const sidechatpanel = document.getElementById('sidechatpanel');
+
+        toggleButton.addEventListener('click', () => {
+            sidechatpanel.classList.toggle('show');
+        });
+    </script>
+    <script>
+        function closeNav() {
+            document.getElementById("toggleButton").click();
+        }   
+    </script>
+</body>
+
+</html>

templates/data.html

@@ -0,0 +1,114 @@
+{% extends 'lay_side.html' %}
+
+{% block css %}
+    <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/data_style.css') }}">
+    <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.13.5/css/dataTables.bootstrap5.min.css">
+    
+{% endblock %}
+
+{% block content %}
+<div class="container" style="margin-bottom: 75px;">
+    
+    <!-- First Row: Label, Dropdown, and Download Button -->
+    <div class="row" style="background-color:rgb(135, 167, 167); padding-top: 10px; padding-bottom: 10px; margin-left: 0px; margin-right: 0px;">
+      <div class="col-md-2" style="margin-left: 15px;">
+        <!-- Label for the dropdown -->
+        <label for="table-dropdown" style="margin-top: 5px; font-weight: 600;">Table Name : </label>
+      </div>
+      <div class="col-md-6">
+        <!-- Dropdown for table selection -->
+        <select id="table-dropdown" class="form-control" style="margin-left: -85px; width:200px;">
+          {% for tbl_name, _ in table_htmls.items() %}
+            <option value="{{ loop.index }}">{{ tbl_name | upper }}</option>
+          {% endfor %}
+        </select>
+      </div>
+      <div class="col-md-3 d-flex justify-content-end align-items-center" style="margin-left: 70px;">
+        <!-- Download Button -->
+        <button class="btn btn-light" onclick="downloadTableData()">
+          <i class="fas fa-download mr-2"></i> Download All
+        </button>
+      </div>
+    </div>
+
+    <!-- Second Row: Table Contents -->
+    <div class="tab" id="tablestab" style="padding: 10px 10px 10px;">
+      {% for _, table_html in table_htmls.items() %}
+        <div class="tabcontent" id="content{{ loop.index }}">
+          {{ table_html | safe }} <!-- The "safe" filter is used to render the HTML content as-is -->
+        </div>
+      {% endfor %}
+    </div>
+  </div>
+
+    <div class="fixed-button">
+      <!-- <div class="col-md-3 d-flex justify-content-end align-items-center" style="margin-left: 70px;"> -->
+        <div class="container d-flex justify-content-end align-items-center" style="margin-left: 50px;">
+            <button class="btn btn-warning" onclick="redirecttovalidation()">Proceed for validation <i class="fas fa-arrow-right"></i></button>
+        </div>
+    </div>
+    <div id="content">
+      {% include 'chatfile8.html' %}
+    </div>
+</div>
+{% endblock %}
+
+{% block javascript %}
+    <!-- Add any additional JavaScript specific to data.html here -->
+    <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
+    <script src="https://cdn.datatables.net/1.13.5/js/jquery.dataTables.min.js"></script>
+    <script src="https://cdn.datatables.net/1.13.5/js/dataTables.bootstrap5.min.js"></script>
+    <script src="{{ url_for('static', filename='js/data_script.js') }}"></script>
+    
+    <script>
+        $(document).ready(function () {
+          console.log("Print hello")
+            {% for tbl_name, _ in table_htmls.items() %}
+            $('#dtable_{{ tbl_name }}').DataTable({
+                scrollX: true
+            });
+            {% endfor %}
+            // document.getElementById("tab1").click();
+        });
+    </script>
+    <script>
+        // JavaScript code to handle the dropdown change event and show the corresponding content div
+        $(document).ready(function () {
+          // Show the initial selected table content on page load
+          showSelectedTableContent();
+      
+          // Handle dropdown change event
+          $('#table-dropdown').on('change', function () {
+            showSelectedTableContent();
+          });
+      
+          // Function to show the selected table content
+          function showSelectedTableContent() {
+            const selectedTableIndex = $('#table-dropdown').val();
+            $('.tabcontent').hide();
+            $('#content' + selectedTableIndex).show();
+          }
+        });
+      
+        // Function to handle the table data download
+        function downloadTableData() {
+          // Implement your download logic here
+          console.log('Downloading table data...');
+        }
+      </script>
+      <script>
+          var req_tables = {{ req_tables | safe }};
+      </script>
+      <script>
+          document.addEventListener("DOMContentLoaded", function() {
+              console.log("Entered")
+              var dropdown = document.getElementById("table-dropdown");
+              console.log("The value is , ", dropdown.value)
+              var hiddenField = document.getElementById("hiddenField");
+
+              dropdown.addEventListener("change", function() {
+                  hiddenField.value = dropdown.value;
+              });
+          });
+      </script>
+{% endblock %}

templates/home.html

@@ -0,0 +1,72 @@
+{% extends 'lay_side.html' %}
+
+{% block css %}
+    <!-- Add any additional CSS specific to home.html here -->
+    <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/home_style.css') }}">
+    <link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.css" />
+{% endblock %}
+
+{% block content %}
+    <!-- Add the content specific to home.html here -->
+    <div class="container">
+        <div class="row">
+            <div class="col-md-6 left-column" style="background-image: url('{{ url_for('static', filename='images/labormarket.jpeg') }}'); background-size: cover; background-position: center; background-repeat: no-repeat; padding-right: 20px;background-color:rgba(0,0,0,0.2);">
+            </div>
+        <div class="col-md-6 right-column">
+            <h2 style="font-family:'Trebuchet MS', 'Lucida Sans Unicode', 'Lucida Grande', 'Lucida Sans', Arial, sans-serif">User Input Page</h2>
+            <p>Please select below filters. Please note that GPNs extracted from workforce data would be used to get other datasets.</p>
+            <form method="POST">
+              <div class="form-group">
+                <label for="date_range">Date Range</label>
+                <div id="reportrange" class="form-control" name="date_range" style="background: #fff; cursor: pointer; padding: 5px 10px; border: 1px solid #ccc; width: 100%">
+                    <i class="fa fa-calendar"></i>&nbsp;
+                    <input type="hidden" name="calendar_value">
+                    <span></span>
+                    <i class="fa fa-caret-down" style="float:right; margin-right:0; top:50%; position:relative; transform: translateY(-50%);"></i>
+                </div>
+            </div>
+    
+            <div class="form-group">
+                <label for="sl_subsl">SL/SSL</label>
+                <select class="form-control" id = "slssl" name="sl_subsl" required>
+                  <option value="">Select Option</option>
+                  <option value="consulting_dna">Consulting DnA</option>
+                  <option value="consulting_det">Consulting DET</option>
+                  <option value="assurance_dna">Assurance DnA</option>
+                  <option value="assurance_audit">Assurance Audit</option>
+                  <option value="tax_gcr_emeia">TAX GCR EMEIA</option>
+                  <option value="eyk_content_delivery">EYK Content Delivery</option>
+                  <option value="es_rms_iass">ES RMS IASS</option>
+                </select>
+              </div>
+    
+              <div class="form-group">
+                <label for="tbl_selected">Table(s)</label>
+                <select class="form-control" id="multiple-checkboxes" multiple="multiple" name="tbl_selected" required>
+                    <option value="workforce">Workforce</option>
+                    <option value="badges">Badges</option>
+                    <option value="learning">Learning</option>
+                    <option value="lat">LAT</option>
+                    <option value="tr">TR</option>
+                    <option value="mct">MCT</option>
+                    <option value="profqual">Profqual</option>
+                </select>  
+              </div>
+    
+            <button type="submit" class="btn btn-primary">Submit</button>
+            </form>
+          </div>
+        </div>
+      </div>
+{% endblock %}
+
+{% block javascript %}
+    <!-- Add any additional JavaScript specific to home.html here -->
+    <script src="{{url_for('static', filename='js/home_script.js')}}"></script>
+    <!-- Bootstrap Datepicker -->
+    <script type="text/javascript" src="https://cdn.jsdelivr.net/jquery/latest/jquery.min.js"></script>
+    <script type="text/javascript" src="https://cdn.jsdelivr.net/momentjs/latest/moment.min.js"></script>
+    <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-datepicker/1.9.0/js/bootstrap-datepicker.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.min.js"></script>
+{% endblock %}

templates/index.html

@@ -0,0 +1,179 @@
+<!DOCTYPE html>
+<html>
+<head>
+    
+    <title>Laboreconomics_allinone</title>
+    <!-- Add Bootstrap CSS -->
+    <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/style.css') }}">
+    <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.25/css/jquery.dataTables.min.css">
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/5.3.0/css/bootstrap.min.css">
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css" />
+    <link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.css" />
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css">
+</head>
+<body>
+    <nav class="navbar navbar-expand-lg navbar-dark bg-dark">
+        <div class="container">
+            <a class="navbar-brand" href="#">LE_AllinOne</a>
+        </div>
+    </nav>
+    
+    <div class="container mt-4">
+        <form method="POST">
+            <div class="row mb-3">
+                <div class="col">
+                    <select class="form-select" name="Environment" enabled>
+                        <option value="option1">DEV</option>
+                        <option value="option2">UAT</option>
+                        <option value="option3">PROD</option>
+                    </select>
+                </div>
+                <div class="col">
+                    <select class="form-select" name="dbtable" enabled>
+                        <option value="workforce">Workforce</option>
+                        <option value="badges">Badges</option>
+                        <option value="learning">Learning</option>
+                        <option value="lat">LAT</option>
+                        <option value="tr">TR</option>
+                        <option value="mct">MCT</option>
+                        <option value="profqual">Profqual</option>
+                    </select>
+                </div>
+                <div class="col">
+                    <select class="form-select" name="sl_ssl" enabled>
+                        <option value="option7">Consulting DnA</option>
+                        <option value="option8">Consulting DET</option>
+                        <option value="option9">Assurance DnA</option>
+                        <option value="option9">Assurance Audit</option>
+                        <option value="option9">TAX GCR EMEIA</option>
+                        <option value="option9">EYK Content Delivery</option>
+                        <option value="option9">ES RMS IASS</option>
+                    </select>
+                </div>
+
+                <div class="col form-select" id="reportrange">
+                    <i class="fa fa-calendar" name="daterange_report"></i>&nbsp;
+                    <input type="hidden" name="calendar_value">
+                    <span></span>
+                </div>
+            </div>
+
+            <div class="form-check form-switch mb-3">
+                <input class="form-check-input" type="checkbox" id="toggleSwitch" onchange="toggleFormElements()">
+                <label class="form-check-label" for="toggleSwitch">use query</label>
+            </div>
+
+            <div class="row mb-3">
+                <div class="col">
+                    <textarea class="form-control" name="querybox" rows="1" placeholder="Enter query" disabled></textarea>
+                </div>
+            </div>
+
+            <div class="row">
+                <div class="col">
+                    <button type="submit" class="btn btn-primary" enabled>Submit</button>
+                </div>
+            </div>
+        </form>
+
+        <br>
+        {% if show_table %}
+        <div class="tab" id="tablestab">
+            <button class="tablinks" onclick="openTab(event, 'content1')" id="tab1">Workforce</button>
+            <button class="tablinks" onclick="openTab(event, 'content2')">GTE</button>
+            <button class="tablinks" onclick="openTab(event, 'content3')">Badges</button>
+            <button class="tablinks" onclick="downloadTableData()" style="position: absolute; top: 0; right: 20px;">
+                <i class="fas fa-download mr-2"></i> Download All
+            </button>
+          </div>
+        
+          <!-- Content Area -->
+          <div id="content1" class="content" style="margin-bottom:110px">
+<!--            <h2>Content 1</h2>-->
+<!--            <p>This is the content for Tab 1.</p>-->
+            <div class="container" >
+                <div class="row">
+                    <div class="col-9" style="overflow-x: auto; overflow-y:auto; height:342px; white-space: nowrap;border-top: 3px red">
+                        <div style="border-top: 4px red;">
+                        {{ form_data | safe}}
+                        <!-- Your table content here -->
+                        </div>
+                    </div>
+                    <div class="col-3">
+                        <div class="card bg-light">
+                            <div class="card-body" style="white-space: nowrap;background-color:#c5e0e0;">
+                                <h5 class="card-title">Stats</h5>
+                                <p class="card-text">Table Name: <strong>Table 1</strong></p>
+                                <p class="card-text">Number of Records: <strong>100</strong></p>
+                                <p class="card-text">No of Columns: <strong>24</strong></p>
+                                <p class="card-text">DataTypes: <strong>String</strong></p>
+                                <p class="card-text">Table values: <strong>99</strong></p>
+                                <p class="card-text">Possible: <strong>Yes</strong></p>
+                                <p class="card-text">DataVolume: <strong>Yes</strong></p>
+                            </div>
+                        </div>
+                    </div>
+
+                </div>
+            </div>
+        </div>
+        
+          <div id="content2" class="content">
+            <div class="container" style="overflow-x: auto; white-space: nowrap; margin-top: 20px; ">
+                {{ form_data | safe }}
+            </div>
+          </div>
+        
+          <div id="content3" class="content">
+            <div class="container" style="overflow-x: auto; white-space: nowrap; margin-top: 20px; ">
+                {{ form_data | safe }}
+            </div>
+          </div>
+
+        <div class="fixed-button">
+            <div class="container d-flex justify-content-end">
+                <button class="btn btn-info btn-lg" onclick="redirecttovalidation()">Proceed for validation ></button>
+            </div>
+        </div>
+        {% endif %}
+
+<!--        {% if plot_div %}-->
+<!--        <div class="container mt-4">-->
+<!--            <h4>Bar Chart:</h4>-->
+<!--            {{ plot_div | safe }}-->
+<!--        </div>-->
+<!--        {% endif %}-->
+
+    </div>
+
+    <!-- Add Bootstrap JS -->
+    <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
+    <script src="https://cdn.datatables.net/1.10.25/js/jquery.dataTables.min.js"></script>
+<!--    <script>-->
+<!--      $(document).ready(function() {-->
+<!--        $('#dtable').DataTable();-->
+<!--      });-->
+<!--      </script>-->
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.3.0/js/bootstrap.bundle.min.js"></script>
+    <!-- jQuery -->
+    <script src="https://code.jquery.com/jquery-3.5.1.min.js"></script>
+    <!-- Bootstrap JS -->
+    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js"></script>
+    <!-- Bootstrap Datepicker JS -->
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-datepicker/1.9.0/js/bootstrap-datepicker.min.js"></script>
+    <script type="text/javascript" src="https://cdn.jsdelivr.net/jquery/latest/jquery.min.js"></script>
+    <script type="text/javascript" src="https://cdn.jsdelivr.net/momentjs/latest/moment.min.js"></script>
+    <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.min.js"></script>
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js"></script>
+
+    <!-- DataTables JavaScript -->
+    <script src="https://cdn.datatables.net/1.11.3/js/jquery.dataTables.min.js"></script>
+    <script src="https://cdn.datatables.net/1.11.3/js/dataTables.bootstrap5.min.js"></script>
+
+    <script src="{{url_for('static', filename='js/mytest.js')}}"></script>
+    <!-- <script>
+        document.getElementById("tab1").click();
+    </script> -->
+</body>
+</html>

templates/lay_side.html

@@ -0,0 +1,130 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>{% block title %}LE_Web{% endblock %}</title>
+  <!-- Bootstrap CSS -->
+  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
+  <!-- Font Awesome CSS -->
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css">
+  <!-- Custom CSS -->
+  <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/layout_style.css') }}">
+  <!-- Add your additional CSS files here -->
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/5.3.0/css/bootstrap.min.css">
+  <!-- <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.8.1/font/bootstrap-icons.css">
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css" /> -->
+  {% block css %}
+  {% endblock %}
+</head>
+<body>
+  <header>
+    <!-- Your common header code here -->
+    <nav class="navbar navbar-expand-lg navbar-dark" style="background-color: rgb(17, 67, 86);z-index: 2;position: fixed; top: 0; width: 100%;">
+      <div class="container-fluid">
+        <a class="navbar-brand" href="index.html" style="padding-left: 70px;">Labor Economics</a>
+        <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarNav"
+          aria-controls="navbarNav" aria-expanded="false" aria-label="Toggle navigation">
+          <span class="navbar-toggler-icon"></span>
+        </button>
+        <div class="collapse navbar-collapse justify-content-end" id="navbarNav">
+          <ul class="navbar-nav">
+            <li class="nav-item">
+              <a class="nav-link" href="home.html"><i class="bi bi-house-fill"></i> Home</a>
+            </li>
+            <li class="nav-item">
+              <a class="nav-link" href="data.html"><i class="bi bi-graph-up"></i> Data</a>
+            </li>
+            <li class="nav-item">
+              <a class="nav-link" href="stats.html"><i class="bi bi-bar-chart-fill"></i> Stats & Validation</a>
+            </li>
+            <li class="nav-item">
+              <a class="nav-link" href="dashboard.html"><i class="bi bi-speedometer2"></i> Performance Dashboard</a>
+            </li>
+          </ul>
+        </div>
+      </div>
+    </nav>
+  </header>
+
+  <div class="container-fluid" style="margin-top: 55px;">
+    <div class="row">
+      {% if show_sidebar %}
+      <!-- Sidebar content goes here -->
+      <div class="col-md-1">
+        <div class="left_tblsidebar">
+          <!-- Sidebar content here -->
+          <!-- Sidebar content for dashboard page goes here -->
+          <ul class="nav flex-column" style="z-index: 0;">
+            <!-- Icon and tooltip for Workforce -->
+            <li class="nav-item {% if 'workforce' not in req_tables|map('lower') %} disabled {% endif %}">
+              <a class="val_leftsidebarcls" href="/validate_workforce" data-section="workforce">
+                <i class="fas fa-users"></i>
+                <span>Workforce</span>
+              </a>
+            </li>
+            <!-- Icon and tooltip for Badges -->
+            <li class="nav-item {% if 'badges' not in req_tables|map('lower') %} disabled {% endif %}">
+              <a class="val_leftsidebarcls" href="/validate_badges" data-section="badges">
+                <i class="fas fa-ribbon"></i>
+                <span style="text-align: center;">Badges</span>
+              </a>
+            </li>
+
+            <!-- Icon and tooltip for Learning -->
+            <li class="nav-item {% if 'learning' not in req_tables|map('lower') %} disabled {% endif %}">
+              <a class="val_leftsidebarcls" href="/validate_learning" data-section="learning">
+                <i class="fas fa-graduation-cap"></i>
+                <span>Learning</span>
+              </a>
+            </li>
+
+            <!-- Icon and tooltip for Timecard -->
+            <li class="nav-item {% if 'gte' not in req_tables|map('lower') %} disabled {% endif %}">
+              <a class="val_leftsidebarcls" href="#" data-section="timecard">
+                <i class="fas fa-clock"></i>
+                <span>Timecard</span>
+              </a>
+            </li>
+
+            <!-- Icon and tooltip for Qualification Details -->
+            <li class="nav-item {% if 'lat' not in req_tables|map('lower') %} disabled {% endif %}">
+              <a class="val_leftsidebarcls" href="#" data-toggle="tooltip" data-placement="right" title="Qualification Details">
+                <i class="fas fa-user-graduate"></i>
+                <span>Qualification</span>
+              </a>
+            </li>
+
+            <!-- Icon and tooltip for Misc Details -->
+            <li class="nav-item">
+              <a class="val_leftsidebarcls" href="/validate_miscellaneous" data-toggle="tooltip" data-placement="right" title="Misc Details">
+                <i class="fa-solid fa-layer-group"></i>
+                <span>Misc</span>
+              </a>
+            </li>
+          </ul>
+        </div>
+      </div>
+      {% endif %}
+
+      <!-- Main content goes here -->
+      <div class="{% if show_sidebar %}col-md-10{% else %}col-md-12{% endif %}">
+        {% block content %}
+        <!-- Your main content goes here -->
+        {% endblock %}
+      </div>
+    </div>
+    
+  </div>
+
+  <!-- Bootstrap JS (Make sure to include this before your custom JS, if any) -->
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/js/all.min.js"></script>
+  <script src="https://code.jquery.com/jquery-3.6.0.slim.min.js"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/2.10.2/umd/popper.min.js"></script>
+  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js"></script>
+
+  <!-- Add your additional JavaScript files here -->
+  {% block javascript %}
+  {% endblock %}
+</body>
+</html>

templates/survey_display.html

@@ -0,0 +1,93 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+  <!-- Font Awesome Library -->
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/js/all.min.js"></script>
+</head>
+<body>
+  <div class="container">
+    <canvas id="paretoChart"></canvas>
+  </div>
+</body>
+<script>
+  // Sample data for the Pareto diagram
+  var data = {
+      labels: ['Category A', 'Category B', 'Category C', 'Category D', 'Category E'],
+      datasets: [
+          {
+              label: 'Frequency',
+              data: [15, 10, 25, 5, 30],
+              backgroundColor: 'rgba(75, 192, 192, 0.7)',
+              yAxisID: 'primary',
+          },
+      ],
+  };
+
+  // Calculate cumulative percentage
+  var cumulativePercentage = [0];
+  var total = data.datasets[0].data.reduce((acc, value) => acc + value, 0);
+  data.datasets[0].data.forEach(value => {
+      cumulativePercentage.push((cumulativePercentage[cumulativePercentage.length - 1] + value) / total * 100);
+  });
+
+  // Create the secondary dataset for the cumulative percentage line
+  data.datasets.push({
+      label: 'Cumulative Percentage',
+      data: cumulativePercentage,
+      borderColor: 'rgba(255, 99, 132, 1)',
+      backgroundColor: 'rgba(255, 99, 132, 0.2)',
+      yAxisID: 'secondary',
+      type: 'line',
+  });
+
+  // Get the canvas context
+  var ctx = document.getElementById('paretoChart').getContext('2d');
+
+  // Create the multi-axis Pareto diagram with options
+  var paretoChart = new Chart(ctx, {
+      type: 'bar',
+      data: data,
+      options: {
+          scales: {
+              x: {
+                  display: true,
+                  title: {
+                      display: true,
+                      text: 'Categories',
+                  },
+              },
+              primary: {
+                  type: 'linear',
+                  position: 'left',
+                  beginAtZero: true,
+                  display: true,
+                  title: {
+                      display: true,
+                      text: 'Frequency',
+                  },
+              },
+              secondary: {
+                  type: 'linear',
+                  position: 'right',
+                  beginAtZero: true,
+                  display: true,
+                  title: {
+                      display: true,
+                      text: 'Cumulative Percentage (%)',
+                  },
+                  grid: {
+                      drawOnChartArea: false, // Hide grid lines for the secondary axis
+                  },
+              },
+          },
+          plugins: {
+              legend: {
+                  display: true,
+              },
+          },
+      },
+  });
+</script>
+
+</html>

templates/validate_badges.html

@@ -0,0 +1,156 @@
+{% extends 'lay_side.html' %}
+
+{% block css %}
+<!-- style="pointer-events: none;color: #ccc;cursor: not-allowed;" -->
+<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/validate_style.css') }}">
+{% endblock %}
+
+
+{% block content %}
+<br>
+<!-- Dashboard Content -->
+<div class="dashboard-content">
+  <div class="container">
+    <div class="row">
+      <div class="col-md-3">
+        <div class="card headstats">
+          <div class="card-body">
+            <h5 class="card-title">Employee Count</h5>
+            <p class="card-text">{{ table_info['unique_gui'] }}</p>
+          </div>
+        </div>
+      </div>
+      <div class="col-md-3">
+        <div class="card headstats">
+          <div class="card-body">
+            <h5 class="card-title">Record Count</h5>
+            <p class="card-text">{{ table_info['n_rows'] }}</p>
+          </div>
+        </div>
+      </div>
+
+      <div class="col-md-3">
+        <div class="card headstats">
+          <div class="card-body">
+            <h5 class="card-title"> # Features</h5>
+            <p class="card-text">{{ table_info['n_cols'] }}</p>
+          </div>
+        </div>
+      </div>
+
+      <div class="col-md-3">
+        <div class="card headstats">
+          <div class="card-body">
+            <h5 class="card-title">Placeholder</h5>
+            <p class="card-text">Yes</p>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+  
+
+  <br>
+  <div class="container">
+    <div class="row">
+      <div class="col-md-12">
+        <h3 class="h3" style="font-weight: bolder;">Data Validation for badges</h3>
+        <p>All entries have undergone thorough validation and have been meticulously categorized into specific groups.
+          The Y-axis of the input data table lists the columns against which validation checks were conducted. These
+          validations have culminated in a visually informative bar chart, offering a clear representation of the
+          percentage distribution across each validated category.
+        </p>
+        <canvas id="validationbar"></canvas>
+      </div>
+    </div>
+  </div>
+  <br>
+  <br>
+  <div class="container mt-4">
+    <div class="row">
+        <div class="col-md-8">
+            <div class="card" style="box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);">
+                <div class="card-body">
+                    <h5 class="card-title"><span class="badge badge-primary">Badge Earned month over month</span></h5>
+                    <p>The data shows a monthly trend in badge initiation and awarding, 
+                      with the highest initiation count in February (56) and the highest award count in January (38). 
+                      Overall, badge activity remains consistent, with variations in each month.</p>
+                    <canvas id="barchart_badgecompletionpermonth"></canvas>
+                    <!-- <p class="card-text">The organization's workforce composition is characterized by a <i class="fa-solid fa-arrow-trend-up"></i> <span style="font-size: 24px; font-weight: bolder;color: #0d3460;">13</span>% higher representation of men compared to women, resulting in a diverse and varied cultural mix within the company.</p> -->
+                </div>
+            </div>
+        </div>
+        <div class="col-md-4">
+            <div class="card" style="box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);">
+                <div class="card-body">
+                    <h5 class="card-title"><span class="badge badge-info">Pillar distribution across badges</span></h5>
+                    <p>The chart below shows the distribution of pillars across badges</p>
+                    <canvas id="doughnutChart2"></canvas>
+                </div>
+            </div>
+        </div>
+      </div>
+  </div>
+  <br>
+  <div class="container mt-4">
+    <div class="row">
+        <div class="col-md-4">
+            <div class="card" style="box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);">
+                <div class="card-body">
+                    <h5 class="card-title"><span class="badge badge-primary">Structured vs Unstructured</span></h5>
+                    <!-- <p>The following chart shows the distribution of Badge Type</p> -->
+                    <canvas id="doughnutchart_strvsunstr"></canvas>
+                    <!-- <p class="card-text">The organization's workforce composition is characterized by a <i class="fa-solid fa-arrow-trend-up"></i> <span style="font-size: 24px; font-weight: bolder;color: #0d3460;">13</span>% higher representation of men compared to women, resulting in a diverse and varied cultural mix within the company.</p> -->
+                </div>
+            </div>
+        </div>
+        <div class="col-md-8">
+            <div class="card" style="box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);">
+                <div class="card-body">
+                    <h5 class="card-title"><span class="badge badge-info">Badge Title Distribution</span></h5>
+                    <p class="card-text">
+                        {% for item in lst_topfive_badgetitle %}
+                        <span class="badge badge-secondary"> {{ item }} </span>
+                        {% endfor %}
+                    </p>
+                    <p>The most interest lies in Agile Learning and Data Integration, with a strong focus on skill
+                      develop ment. Data Visualization and Cloud Learning are also popular, reflecting a growing interest 
+                      in data presentation and cloud technologies. Robotic Process Automation badges signify a commitment
+                       to automation skills, while Data Science Learning badges show an emerging interest in data science.  
+                       These badges offer valuable opportunities for personal and professional growth.</p>
+                </div>
+            </div>
+        </div>
+      </div>
+  </div>
+  <br>
+
+  <div id="content">
+    {% include 'chatfile8.html' %}
+  </div>
+  
+  {% endblock %}
+
+  {% block javascript %}
+  
+  <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+  <script src="{{ url_for('static', filename='js/validate_script.js') }}"></script>
+  <!-- Add any additional JavaScript specific to data.html here -->
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/plotly.js/1.5.1/plotly.min.js"></script>
+  <!-- Bootstrap JS (Make sure to include this before your custom JS, if any) -->
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/js/all.min.js"></script>
+  <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/2.10.2/umd/popper.min.js"></script>
+  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js"></script>
+
+  <script>
+    var jsonData = {{ json_data | safe }};
+    var json_pillar_data = {{ json_pillar_data | safe }};
+    var json_badgecompletion_data = {{ json_badgecompletion_data | safe }};
+  </script>
+  {% endblock %}
+
+  <!-- var labels = {{ labels | tojson }};
+  var pattern_issue = {{ pattern_issue | tojson }};
+  var null_issue = {{ null_issue | tojson }};
+  var good_data = {{ good_data | tojson }}; -->