Spaces:
Sleeping
Sleeping
Upload 7 files
Browse files- src/.env +1 -0
- src/cleaner.py +25 -0
- src/excel_reader.py +11 -0
- src/gemini_agent.py +28 -0
- src/outlier.py +29 -0
- src/profiler.py +17 -0
- src/reporter.py +24 -0
src/.env
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
GEMINI_API_KEY="AQ.Ab8RN6KA0xNgoncudDRxlsjRUe0RW15JwCoqCDJP4x38Vw8lCw"
|
src/cleaner.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import pandas as pd
|
| 2 |
+
|
| 3 |
+
def clean_dataframe(df):
|
| 4 |
+
|
| 5 |
+
df = df.drop_duplicates()
|
| 6 |
+
|
| 7 |
+
for col in df.columns:
|
| 8 |
+
|
| 9 |
+
if df[col].dtype == "object":
|
| 10 |
+
|
| 11 |
+
df[col] = (
|
| 12 |
+
df[col]
|
| 13 |
+
.astype(str)
|
| 14 |
+
.str.strip()
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
if pd.api.types.is_numeric_dtype(
|
| 18 |
+
df[col]
|
| 19 |
+
):
|
| 20 |
+
|
| 21 |
+
df[col] = df[col].fillna(
|
| 22 |
+
df[col].median()
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
return df
|
src/excel_reader.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import pandas as pd
|
| 2 |
+
|
| 3 |
+
def read_file(file):
|
| 4 |
+
|
| 5 |
+
if file.name.endswith(".csv"):
|
| 6 |
+
return {"Sheet1": pd.read_csv(file)}
|
| 7 |
+
|
| 8 |
+
return pd.read_excel(
|
| 9 |
+
file,
|
| 10 |
+
sheet_name=None
|
| 11 |
+
)
|
src/gemini_agent.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import importlib
|
| 2 |
+
import importlib.util
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
package_name = "google.generativeai"
|
| 6 |
+
if importlib.util.find_spec(package_name) is None:
|
| 7 |
+
raise ImportError(
|
| 8 |
+
"Package 'google.generativeai' not found. Install it with 'pip install google-generative-ai'"
|
| 9 |
+
)
|
| 10 |
+
genai = importlib.import_module(package_name)
|
| 11 |
+
|
| 12 |
+
genai.configure(
|
| 13 |
+
api_key=os.getenv(
|
| 14 |
+
"GEMINI_API_KEY"
|
| 15 |
+
)
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
model = genai.GenerativeModel(
|
| 19 |
+
"gemini-2.5-flash"
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
def get_suggestions(summary):
|
| 23 |
+
|
| 24 |
+
response = model.generate_content(
|
| 25 |
+
summary
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
return response.text
|
src/outlier.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
def detect_outliers(df):
|
| 2 |
+
|
| 3 |
+
result = {}
|
| 4 |
+
|
| 5 |
+
for col in df.select_dtypes(
|
| 6 |
+
include="number"
|
| 7 |
+
).columns:
|
| 8 |
+
|
| 9 |
+
q1 = df[col].quantile(0.25)
|
| 10 |
+
|
| 11 |
+
q3 = df[col].quantile(0.75)
|
| 12 |
+
|
| 13 |
+
iqr = q3 - q1
|
| 14 |
+
|
| 15 |
+
lower = q1 - 1.5 * iqr
|
| 16 |
+
|
| 17 |
+
upper = q3 + 1.5 * iqr
|
| 18 |
+
|
| 19 |
+
count = len(
|
| 20 |
+
df[
|
| 21 |
+
(df[col] < lower)
|
| 22 |
+
|
|
| 23 |
+
(df[col] > upper)
|
| 24 |
+
]
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
result[col] = count
|
| 28 |
+
|
| 29 |
+
return result
|
src/profiler.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
def profile_dataframe(df):
|
| 2 |
+
|
| 3 |
+
profile = {}
|
| 4 |
+
|
| 5 |
+
profile["rows"] = len(df)
|
| 6 |
+
|
| 7 |
+
profile["columns"] = len(df.columns)
|
| 8 |
+
|
| 9 |
+
profile["missing"] = (
|
| 10 |
+
df.isnull().sum().sum()
|
| 11 |
+
)
|
| 12 |
+
|
| 13 |
+
profile["duplicates"] = (
|
| 14 |
+
df.duplicated().sum()
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
return profile
|
src/reporter.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
def create_report(
|
| 2 |
+
profile,
|
| 3 |
+
outliers
|
| 4 |
+
):
|
| 5 |
+
|
| 6 |
+
report = f"""
|
| 7 |
+
|
| 8 |
+
Rows:
|
| 9 |
+
{profile['rows']}
|
| 10 |
+
|
| 11 |
+
Columns:
|
| 12 |
+
{profile['columns']}
|
| 13 |
+
|
| 14 |
+
Missing:
|
| 15 |
+
{profile['missing']}
|
| 16 |
+
|
| 17 |
+
Duplicates:
|
| 18 |
+
{profile['duplicates']}
|
| 19 |
+
|
| 20 |
+
Outliers:
|
| 21 |
+
{outliers}
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
return report
|