Upload 4 files

Information.py

@@ -0,0 +1,61 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+import io
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import LabelEncoder
+import seaborn as sns
+import base64
+
+
+
+
+def show_general_data_statistics():
+    if "data" in st.session_state:
+        data = st.session_state["data"]
+        num_var = len(data.columns)
+        num_rows = len(data)
+        missing_cells = data.isnull().sum().sum()
+        missing_cells_percent = (missing_cells / (data.size)) * 100
+        duplicate_rows = data.duplicated().sum()
+        duplicate_rows_percent = (duplicate_rows / num_rows) * 100
+        var_types = data.dtypes.value_counts()
+
+        st.write("### General Data Statistics:")
+        st.write(f"- **Number of Variables:**   {num_var}")
+        st.write(f"- **Number of Rows:**    {num_rows}")
+        st.write(f"- **Missing Cells:**     {missing_cells}")
+        st.write(f"- **Missing Cells (%):**     {missing_cells_percent:.2f}%")
+        st.write(f"- **Duplicate Rows:**    {duplicate_rows}")
+        st.write(f"- **Duplicate Rows (%):**    {duplicate_rows_percent:.2f}%")
+        st.write("#### Variable Types:")
+        st.write(var_types)
+    else:
+        st.warning("Please upload a dataset first.")
+
+
+
+
+def describe_data():
+    st.title("Describe Data")
+
+    if "data" in st.session_state:
+        data = st.session_state["data"]
+        st.write("Dataset Description:")
+        st.write(data.describe())
+    else:
+        st.warning("Please upload a dataset first.")
+
+
+def info_data():
+    st.title("Dataset Info")
+
+    if "data" in st.session_state:
+        data = st.session_state["data"]
+        buffer = io.StringIO()
+        data.info(buf=buffer)
+        info = buffer.getvalue()
+        st.text(info)
+    else:
+        st.warning("Please upload a dataset first.")
+

Preprocessing1.py

@@ -0,0 +1,132 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+import io
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import LabelEncoder
+import seaborn as sns
+import base64
+
+
+def preview_data():
+    if "data" in st.session_state:
+        data = st.session_state["data"]
+
+        st.write("### Dataset Preview Options:")
+
+        preview_option = st.radio(
+            "Select how to preview the dataset:",
+            options=["Head", "Tail", "Custom Number of Rows"],
+            index=0
+        )
+
+        if preview_option == "Head":
+            st.write("### First 5 Rows of the Dataset:")
+            st.dataframe(data.head())
+        elif preview_option == "Tail":
+            st.write("### Last 5 Rows of the Dataset:")
+            st.dataframe(data.tail())
+        elif preview_option == "Custom Number of Rows":
+            number = st.slider(
+                "Select Number of Rows to Display:", 1, len(data))
+            st.write(f"### First {number} Rows of the Dataset:")
+            st.dataframe(data.head(number))
+
+        # Show entire data
+        if st.checkbox("Show all data"):
+            st.write(data)
+
+        # Show column names
+        if st.checkbox("Show Column Names"):
+            st.write(data.columns)
+
+        # Show dataset dimensions (rows and columns)
+        if st.checkbox("Show Dimensions"):
+            st.write(data.shape)
+
+    else:
+        st.warning("Please upload a dataset to view options.")
+
+
+def data_cleaning():
+    if "data" in st.session_state:
+        data = st.session_state["data"]
+
+        st.subheader("Data Cleaning")
+
+        col_option = st.selectbox("Choose your option", [
+                                  "Check all numeric features are numeric?", "Show unique values of categorical features"])
+
+        # Check and convert numeric columns
+        if col_option == "Check all numeric features are numeric?":
+            st.write("Converting all numeric columns to numeric types...")
+            numeric_columns = list(
+                data.select_dtypes(include=np.number).columns)
+            for col in numeric_columns:
+                data[col] = pd.to_numeric(data[col], errors='coerce')
+            st.write("Done!")
+
+        # Show unique values for categorical features
+        elif col_option == "Show unique values of categorical features":
+            st.write("Unique values for categorical features:")
+            for column in data.columns:
+                # check for categorical features (strings)
+                if data[column].dtype == object:
+                    st.write(f"{column}: {data[column].unique()}")
+            st.write("====================================")
+
+    else:
+        st.warning("Please upload a dataset to perform data cleaning.")
+
+
+def modify_column_names():
+    st.title("Modify Column Names")
+
+    # Ensure data exists in the session
+    if "data" in st.session_state:
+        df = st.session_state["data"]
+
+        st.write('### *Current Column Names*')
+        st.table(df.columns)
+
+        st.write('### *Modify Column Names*')
+        with st.expander("Modify Column Names", expanded=True):
+            before_col = st.session_state.get(
+                "modified_columns", list(df.columns))
+            before_col_df = pd.DataFrame(before_col, columns=['Column Name'])
+            st.table(before_col_df)
+
+            col3, col4, col5, col6 = st.columns(4)
+            if st.button('Convert to Uppercase'):
+                st.session_state.modified_columns = [
+                    col.upper() for col in before_col]
+            if st.button('Convert to Lowercase'):
+                st.session_state.modified_columns = [
+                    col.lower() for col in before_col]
+            if st.button('Replace Spaces with Underscore'):
+                st.session_state.modified_columns = [
+                    col.replace(" ", "_") for col in before_col]
+            if st.button('Capitalize First Letters'):
+                st.session_state.modified_columns = [
+                    col.title() for col in before_col]
+
+            df.columns = st.session_state.modified_columns
+
+            st.success("Changes applied successfully.")
+            st.table(pd.DataFrame(df.columns, columns=['Modified Columns']))
+
+        st.write("### *Modify a Specific Column Name*")
+        column_select = st.selectbox(
+            'Select column to modify', options=st.session_state.modified_columns)
+        new_column_name = st.text_input('Enter new column name')
+        if st.button('Update Column Name'):
+            if column_select and new_column_name:
+                st.session_state.modified_columns = [
+                    new_column_name if col == column_select else col for col in st.session_state.modified_columns]
+                df.columns = st.session_state.modified_columns
+                st.success("Column name updated.")
+                st.table(pd.DataFrame(
+                    df.columns, columns=['Modified Columns']))
+
+    else:
+        st.warning("Please upload a dataset first.")

Preprocessing2.py

@@ -0,0 +1,222 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+import io
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import LabelEncoder
+import seaborn as sns
+import base64
+
+
+def handle_categorical_values():
+    if "data" in st.session_state:
+        data = st.session_state["data"]
+
+        st.subheader("Handle Categorical Values")
+
+        categorical_cols_features = list(
+            data.select_dtypes(include="object").columns)
+
+        # One-Hot Encoding for nominal categorical features
+        one_hot_enc = st.multiselect(
+            "Select nominal categorical columns", categorical_cols_features)
+
+        # Apply one-hot encoding to selected columns
+        if one_hot_enc:
+            for column in one_hot_enc:
+                if data[column].dtype == 'object':  # Only apply to categorical/string columns
+                    data = pd.get_dummies(data, columns=[column])
+            st.write("### Data after One-Hot Encoding:")
+            st.write(data.head())
+
+        # Label Encoding for ordinal categorical features
+        label_encoder = LabelEncoder()
+        label_enc = st.multiselect(
+            "Select ordinal categorical columns", categorical_cols_features)
+
+        # Apply label encoding to selected columns
+        if label_enc:
+            for column in label_enc:
+                if data[column].dtype == 'object':  # Only apply to categorical/string columns
+                    data[column] = label_encoder.fit_transform(data[column])
+            st.write("### Data after Label Encoding:")
+            st.write(data.head())
+
+    else:
+        st.warning("Please upload a dataset to handle categorical values.")
+
+
+def missing_values():
+    st.title("Handle Missing Values")
+
+    if "data" in st.session_state:
+        data = st.session_state["data"].copy()
+
+        action = st.selectbox(
+            "Select Action", ["Drop", "Dropna", "Fill missing val"])
+
+        column = st.selectbox("Select Column", data.columns)
+
+        # Before Visualization
+        st.write("### Before:")
+        st.dataframe(data)
+
+        # Placeholder for After Visualization
+        after_placeholder = st.empty()
+
+        if st.button("OK"):
+            modified_data = data.copy()
+
+            if action == "Drop":
+                modified_data.drop(columns=[column], inplace=True)
+            elif action == "Dropna":
+                modified_data.dropna(subset=[column], inplace=True)
+            elif action == "Fill missing val":
+
+                fill_method = st.selectbox(
+                    "Select fill method", ["Mean", "Mode", "Median"])
+
+                if fill_method == "Mean":
+                    fill_value = data[column].mean()
+                elif fill_method == "Mode":
+                    fill_value = data[column].mode()[0]
+                elif fill_method == "Median":
+                    fill_value = data[column].median()
+
+                modified_data[column].fillna(fill_value, inplace=True)
+
+            # After Visualization
+            after_placeholder.write("### After:")
+            after_placeholder.dataframe(modified_data)
+
+            st.session_state["data"] = modified_data
+    else:
+        st.warning("Please upload a dataset first.")
+
+
+def handle_duplicates():
+    st.title("Handle Duplicates")
+
+    if "data" in st.session_state:
+        data = st.session_state["data"].copy()
+
+        action = st.selectbox(
+            "Select Action", ["Drop Duplicates", "Drop Duplicates in Column", "Keep First", "Keep Last"])
+
+        if action in ["Drop Duplicates in Column", "Keep First", "Keep Last"]:
+            column = st.selectbox("Select Column", data.columns)
+        else:
+            column = None
+
+        # Before Visualization
+        st.write("### Before:")
+        st.dataframe(data)
+
+        # Placeholder for After Visualization
+        after_placeholder = st.empty()
+
+        if st.button("OK"):
+            modified_data = data.copy()
+
+            if action == "Drop Duplicates":
+                modified_data.drop_duplicates(inplace=True)
+
+            elif action == "Drop Duplicates in Column":
+                modified_data.drop_duplicates(subset=[column], inplace=True)
+
+            elif action == "Keep First":
+                # Keep the first occurrence of duplicates and drop others
+                modified_data.drop_duplicates(
+                    subset=[column], keep="first", inplace=True)
+
+            elif action == "Keep Last":
+                # Keep the last occurrence of duplicates and drop others
+                modified_data.drop_duplicates(
+                    subset=[column], keep="last", inplace=True)
+
+            # After Visualization
+            after_placeholder.write("### After:")
+            after_placeholder.dataframe(modified_data)
+
+            st.session_state["data"] = modified_data
+    else:
+        st.warning("Please upload a dataset first.")
+
+
+def handle_outliers():
+    st.title("Handle Outliers")
+
+    if "data" in st.session_state:
+        data = st.session_state["data"].copy()
+
+        column = st.selectbox("Select Column", data.select_dtypes(
+            include=[np.number]).columns)
+
+        action = st.selectbox(
+            "Select Action",
+            ["Remove Outliers (IQR)", "Set Bounds Manually",
+             "Replace Outliers"]
+        )
+
+        st.write("### Before:")
+        st.dataframe(data)
+
+        after_placeholder = st.empty()
+
+        if st.button("OK"):
+            modified_data = data.copy()
+
+            if action == "Remove Outliers (IQR)":
+                Q1 = data[column].quantile(0.25)
+                Q3 = data[column].quantile(0.75)
+                IQR = Q3 - Q1
+                lower_bound = Q1 - 1.5 * IQR
+                upper_bound = Q3 + 1.5 * IQR
+
+                # Remove outliers
+                modified_data = modified_data[
+                    (modified_data[column] >= lower_bound) & (
+                        modified_data[column] <= upper_bound)
+                ]
+
+            elif action == "Set Bounds Manually":
+                # User inputs for bounds
+                lower_bound = st.number_input(
+                    f"Set lower bound for {column}", value=float(data[column].min()))
+                upper_bound = st.number_input(
+                    f"Set upper bound for {column}", value=float(data[column].max()))
+
+                # Remove rows outside the bounds
+                modified_data = modified_data[
+                    (modified_data[column] >= lower_bound) & (
+                        modified_data[column] <= upper_bound)
+                ]
+
+            elif action == "Replace Outliers":
+
+                Q1 = data[column].quantile(0.25)
+                Q3 = data[column].quantile(0.75)
+                IQR = Q3 - Q1
+                lower_bound = Q1 - 1.5 * IQR
+                upper_bound = Q3 + 1.5 * IQR
+
+                replace_method = st.radio(
+                    "Select Replacement Method",
+                    ["Mean", "Median"]
+                )
+                if replace_method == "Mean":
+                    replacement_value = data[column].mean()
+                else:
+                    replacement_value = data[column].median()
+
+                # Replace outliers
+                modified_data[column] = modified_data[column].apply(
+                    lambda x: replacement_value if x < lower_bound or x > upper_bound else x
+                )
+
+            after_placeholder.write("### After:")
+            after_placeholder.dataframe(modified_data)
+
+            st.session_state["data"] = modified_data
+    else:
+        st.warning("Please upload a dataset first.")

RAG.py

@@ -0,0 +1,72 @@
+import pandas as pd
+import json
+from langchain.docstore.document import Document
+from langchain.vectorstores import Chroma
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.llms import HuggingFaceHub
+from langchain.chains import RetrievalQA
+
+
+# file_path = "thyroidDF.csv"
+# df = pd.read_csv(file_path)
+
+def create_doucment(df):
+    documents = [
+        Document(
+            metadata={"id": str(i)},
+            # Serialize the dictionary to a JSON string
+            page_content=json.dumps(row.to_dict())
+        )
+        for i, row in df.iterrows()
+    ]
+    return documents
+
+
+def load_models_embedding():
+    embeddings = HuggingFaceEmbeddings(
+        model_name="sentence-transformers/all-MiniLM-L6-v2")
+    return embeddings
+
+
+def load_models_llm():
+    llm = HuggingFaceHub(
+        repo_id="Qwen/Qwen2.5-72B-Instruct",
+        # Replace with your token
+        api="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22"
+        huggingfacehub_api_token=api[:-2],
+        model_kwargs={"temperature": 0.5,
+                      "max_length": 100}  # Faster inference
+    )
+    return llm
+
+
+def create_database(embedding, documents):
+    vector_store = Chroma.from_documents(documents, embedding=embedding)
+    return vector_store
+
+# retriever = create_database().as_retriever()
+
+
+def ask_me(question, retriever, llm):
+
+    qa_chain = RetrievalQA.from_chain_type(
+        retriever=retriever,
+        chain_type="stuff",
+        llm=load_models_llm(),
+        return_source_documents=True)
+
+    response = qa_chain.invoke({"query": question})
+    print("Answer:", response["result"])
+
+
+# qa_chain = RetrievalQA.from_chain_type(
+#     retriever=retriever,
+#     chain_type="stuff",
+#     llm=llm,
+#     return_source_documents=True
+# )
+
+# question = "Can you provide the TSH, T3, and FTI values for patients aged 55?"
+# # question = "What columns are in the dataset?"
+# response = qa_chain.invoke({"query": question})
+# print("Answer:", response["result"])