Upload 6 files

app.py

@@ -0,0 +1,41 @@
+
+import streamlit as st
+import requests
+from fastapi import FastAPI
+from main import app as fastapi_app
+
+import uvicorn
+import threading
+
+FASTAPI_URL = "http://127.0.0.1:8000/preprocess_data/"
+
+# Start FastAPI server in background thread
+def run_fastapi():
+    uvicorn.run(fastapi_app, host="127.0.0.1", port=8000)
+
+threading.Thread(target=run_fastapi, daemon=True).start()
+
+st.title("📊 Data Preprocessing App")
+
+uploaded_file = st.file_uploader("Upload CSV File", type=["csv"])
+
+if uploaded_file is not None:
+    st.write("✅ File uploaded successfully!")
+
+if st.button("🚀 Process Data"):
+    if uploaded_file is None:
+        st.warning("⚠️ Please upload a CSV file first!")
+    else:
+        with st.spinner("Processing... ⏳"):
+            files = {"upload_file": (uploaded_file.name, uploaded_file, "text/csv")}
+            response = requests.post(FASTAPI_URL, files=files)
+
+            if response.status_code == 200:
+                st.success("✅ Data processed successfully!")
+                cleaned_data_path = "cleaned_dataset.csv"
+                with open(cleaned_data_path, "wb") as f:
+                    f.write(response.content)
+                with open(cleaned_data_path, "rb") as f:
+                    st.download_button("📥 Download Processed CSV", f, "cleaned_dataset.csv", "text/csv")
+            else:
+                st.error(f"❌ Error: {response.json()['detail']}")

app/main.py

@@ -0,0 +1,40 @@
+
+from fastapi import FastAPI, HTTPException, File, UploadFile
+from fastapi.responses import FileResponse
+from app.services.preprocessing import data_quality, standardize_data_types, handle_missing_data, handle_outliers, generate_final_report, save_cleaned_data
+import pandas as pd
+import io
+import os
+
+app = FastAPI(title="Data Preprocessing")
+os.makedirs("output", exist_ok=True)
+
+@app.get("/")
+async def root():
+    return {"message": "Welcome to the Data Preprocessing API!"}
+
+@app.post("/preprocess_data/")
+async def upload_csv(upload_file: UploadFile = File(...)):
+    try:
+        if not upload_file.filename.endswith('.csv'):
+            raise HTTPException(status_code=400, detail="File must be in CSV format!")
+        content = await upload_file.read()
+        df = pd.read_csv(io.BytesIO(content), encoding_errors="replace")
+        if df.empty:
+            raise HTTPException(status_code=400, detail="File is empty, upload the correct file")
+
+        data_quality(df)
+        df = standardize_data_types(df)
+        df = handle_missing_data(df)
+        df = handle_outliers(df)
+
+        REPORT_PATH = "output/preprocessing_report.txt"
+        generate_final_report(df, REPORT_PATH)
+
+        CLEANED_DATA_PATH = "output/cleaned_dataset.csv"
+        save_cleaned_data(df, CLEANED_DATA_PATH)
+
+        return FileResponse(CLEANED_DATA_PATH, media_type="text/csv", filename="cleaned_dataset.csv")
+
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"Error processing file: {str(e)}")

app/services/preprocessing.py

@@ -0,0 +1,68 @@
+
+from sklearn.impute import SimpleImputer
+import pandas as pd
+import numpy as np
+import json
+
+def data_quality(df: pd.DataFrame):
+    df.drop_duplicates(inplace=True)
+    return df
+
+def standardize_data_types(df: pd.DataFrame) -> pd.DataFrame:
+    for col in df.columns:
+        if df[col].isin([True, False]).all():
+            continue
+        if df[col].dtype == 'object' and df[col].str.replace('.', '', 1).str.isnumeric().all():
+            df[col] = pd.to_numeric(df[col], errors='ignore')
+        try:
+            df[col] = pd.to_datetime(df[col], errors='coerce')
+            if df[col].notna().sum() == 0:
+                df[col] = df[col].astype(str)
+        except Exception:
+            pass
+        try:
+            if df[col].apply(lambda x: isinstance(x, str) and x.startswith("[") and x.endswith("]")).all():
+                df[col] = df[col].apply(json.loads)
+        except Exception:
+            pass
+        if df[col].dtype == 'object' and df[col].dropna().isin(["TRUE", "FALSE"]).all():
+            df[col] = df[col].map({"TRUE": True, "FALSE": False})
+        if df[col].dtype == 'object':
+            df[col] = df[col].astype(str)
+    df.fillna("", inplace=True)
+    return df
+
+def handle_missing_data(df: pd.DataFrame) -> pd.DataFrame:
+    numeric_col = df.select_dtypes(include=['number']).columns
+    if not numeric_col.empty:
+        df[numeric_col] = SimpleImputer(strategy='median').fit_transform(df[numeric_col])
+    categorical_col = df.select_dtypes(include=['object']).columns
+    if not categorical_col.empty:
+        df[categorical_col] = SimpleImputer(strategy='most_frequent').fit_transform(df[categorical_col])
+    return df
+
+def handle_outliers(df: pd.DataFrame) -> pd.DataFrame:
+    numeric_col = df.select_dtypes(include=['number','int64', 'float64']).columns
+    if not numeric_col.empty:
+        for col in numeric_col:
+            Q1 = df[col].quantile(0.25)
+            Q3 = df[col].quantile(0.75) 
+            IQR = Q3 - Q1
+            lower = Q1 - 1.5 * IQR
+            upper = Q3 + 1.5 * IQR
+            df[col] = df[col].apply(lambda x: lower if x < lower else upper if x > upper else x)
+    return df
+
+def generate_final_report(df: pd.DataFrame, file_path: str):
+    with open(file_path, "w") as file:
+        file.write("FINAL DATA PREPROCESSING REPORT\n")
+        file.write("=" * 50 + "\n\n")
+        missing = df.isnull().sum()
+        for col, count in missing.items():
+            file.write(f"{col}: {count} missing values\n")
+        file.write(f"Total Duplicate Rows: {df.duplicated().sum()}\n")
+        file.write("Preprocessing Completed Successfully!\n")
+
+def save_cleaned_data(df: pd.DataFrame, file_path: str):
+    df.to_csv(file_path, index=False)
+    return file_path

main.py

@@ -0,0 +1,40 @@
+
+from fastapi import FastAPI, HTTPException, File, UploadFile
+from fastapi.responses import FileResponse
+from preprocessing import data_quality, standardize_data_types, handle_missing_data, handle_outliers, generate_final_report, save_cleaned_data
+import pandas as pd
+import io
+
+app = FastAPI(title="Data Preprocessing")
+
+@app.get("/")
+async def root():
+    return {"message": "Welcome to the Data Preprocessing API!"}
+
+@app.post("/preprocess_data/")
+async def upload_csv(upload_file: UploadFile = File(...)):
+    try:
+        if not upload_file.filename.endswith('.csv'):
+            raise HTTPException(status_code=400, detail="File must be in CSV format!")
+
+        content = await upload_file.read()
+        df = pd.read_csv(io.BytesIO(content), encoding_errors="replace")
+
+        if df.empty:
+            raise HTTPException(status_code=400, detail="File is empty, upload the correct file")
+
+        data_quality(df)
+        df = standardize_data_types(df)
+        df = handle_missing_data(df)
+        df = handle_outliers(df)
+
+        REPORT_PATH = "output/preprocessing_report.txt"
+        generate_final_report(df, REPORT_PATH)
+
+        CLEANED_DATA_PATH = "output/cleaned_dataset.csv"
+        save_cleaned_data(df, CLEANED_DATA_PATH)
+
+        return FileResponse(CLEANED_DATA_PATH, media_type="text/csv", filename="cleaned_dataset.csv")
+
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"Error processing file: {str(e)}")

preprocessing.py

@@ -0,0 +1,78 @@
+
+from sklearn.impute import SimpleImputer
+import pandas as pd
+import numpy as np
+import json
+
+def data_quality(df: pd.DataFrame):
+    print("Missing values before handling:")
+    print(df.isnull().sum())
+    print("Duplicate rows before handling:")
+    print(int(df.duplicated().sum()))
+    df.drop_duplicates(inplace=True)
+    print("Duplicate rows after handling:")
+    print(int(df.duplicated().sum()))
+    return df
+
+def standardize_data_types(df: pd.DataFrame) -> pd.DataFrame:
+    for col in df.columns:
+        if df[col].isin([True, False]).all():
+            continue
+        if df[col].dtype == 'object' and df[col].str.replace('.', '', 1).str.isnumeric().all():
+            df[col] = pd.to_numeric(df[col], errors='ignore')
+        try:
+            df[col] = pd.to_datetime(df[col], errors='coerce')
+            if df[col].notna().sum() == 0:
+                df[col] = df[col].astype(str)
+        except Exception:
+            pass
+        try:
+            if df[col].apply(lambda x: isinstance(x, str) and x.startswith("[") and x.endswith("]")).all():
+                df[col] = df[col].apply(json.loads)
+        except Exception:
+            pass
+        if df[col].dtype == 'object' and df[col].dropna().isin(["TRUE", "FALSE"]).all():
+            df[col] = df[col].map({"TRUE": True, "FALSE": False})
+        if df[col].dtype == 'object':
+            df[col] = df[col].astype(str)
+    df.fillna("", inplace=True)
+    return df
+
+def handle_missing_data(df: pd.DataFrame) -> pd.DataFrame:
+    numeric_col = df.select_dtypes(include=['number']).columns
+    if not numeric_col.empty:
+        num_imputer = SimpleImputer(strategy='median')
+        df[numeric_col] = num_imputer.fit_transform(df[numeric_col])
+    categorical_col = df.select_dtypes(include=['object']).columns
+    if not categorical_col.empty:
+        cat_imputer = SimpleImputer(strategy='most_frequent')
+        df[categorical_col] = cat_imputer.fit_transform(df[categorical_col])
+    return df
+
+def handle_outliers(df: pd.DataFrame) -> pd.DataFrame:
+    numeric_col = df.select_dtypes(include=['number','int64', 'float64']).columns
+    if not numeric_col.empty:
+        for col in numeric_col:
+            Q1 = df[col].quantile(0.25)
+            Q3 = df[col].quantile(0.75)
+            IQR = Q3 - Q1
+            lower_bound = Q1 - 1.5 * IQR
+            upper_bound = Q3 + 1.5 * IQR
+            df[col] = df[col].apply(lambda x: lower_bound if x < lower_bound else upper_bound if x > upper_bound else x)
+    return df
+
+def generate_final_report(df: pd.DataFrame, file_path: str):
+    with open(file_path, "w") as file:
+        file.write("FINAL DATA PREPROCESSING REPORT\n")
+        file.write("=" * 50 + "\n\n")
+        file.write("Missing Values (After Preprocessing):\n")
+        missing_values = df.isnull().sum()
+        for col, count in missing_values.items():
+            file.write(f"{col}: {count} missing values\n")
+        file.write("\nDuplicate Rows (After Preprocessing):\n")
+        file.write(f"Total Duplicate Rows: {df.duplicated().sum()}\n\n")
+        file.write("Preprocessing Completed Successfully!\n")
+
+def save_cleaned_data(df: pd.DataFrame, file_path: str):
+    df.to_csv(file_path, index=False)
+    return file_path

requirements.txt

@@ -0,0 +1,7 @@
+
+streamlit
+fastapi
+pandas
+scikit-learn
+uvicorn
+requests