Upload 2 files

app.py

@@ -0,0 +1,109 @@
+# app.py
+
+import streamlit as st
+import pandas as pd
+from io import StringIO, BytesIO
+import os
+from data_clean_final import clean_data, display_llm_report
+import tempfile
+
+# Set page config
+st.set_page_config(page_title="LLM Data Cleaner", page_icon=":bar_chart:", layout="wide")
+
+st.title("LLM-Powered Data Cleaner")
+st.markdown(
+    """
+    Upload a CSV, TSV or Excel file, and we'll clean it up for you using AI-augmented techniques.
+    The system will automatically:
+    - Fix formatting issues
+    - Handle missing values
+    - Standardize data entries
+    - Provide AI-powered suggestions for data improvements
+    
+    Then, you can download the cleaned data for your analysis.
+    """
+)
+
+# File uploader
+uploaded_file = st.file_uploader("Choose a file", type=["csv", "tsv", "xlsx"])
+
+if uploaded_file:
+    file_bytes = uploaded_file.read() # Convert to BytesIO for pandas
+    file_name = uploaded_file.name
+    file_type = os.path.splitext(file_name)[1].lower()
+
+    # Clean data and get LLM suggestions
+    with st.spinner("Cleaning your data..."):
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix=file_type) as temp_file:
+            temp_file.write(file_bytes)
+            temp_file_path = temp_file.name
+
+        try:
+            # Temporary file to clean_data function
+            cleaned_df, suggestions = clean_data(temp_file_path)
+
+            st.success("Data cleaned successfully!")
+
+        finally:
+            # Clean up the temporary file
+            if os.path.exists(temp_file_path):
+                os.unlink(temp_file_path)
+
+    # Show original data
+    st.subheader("Original Data")
+    try:
+        if file_type == ".tsv":
+            original_df = pd.read_csv(BytesIO(file_bytes), sep='\t')
+        elif file_type == ".xlsx":
+            original_df = pd.read_excel(BytesIO(file_bytes))
+        else:
+            original_df = pd.read_csv(BytesIO(file_bytes))
+
+        st.dataframe(original_df.head(10), use_container_width=True)
+    
+    except Exception as e:
+        st.error(f"Error loading original data: {str(e)}")
+
+    # Show cleaned data
+    st.subheader("Cleaned Data Preview")
+    st.dataframe(cleaned_df.head(10), use_container_width=True)
+
+    # Data statistics
+    col1, col2, col3 = st.columns(3)
+    with col1:
+        st.metric("Total Rows", len(cleaned_df))
+    with col2:
+        st.metric("Total Columns", len(cleaned_df.columns))
+    with col3:
+        null_percentage = round((cleaned_df.isnull().sum().sum() / (cleaned_df.shape[0] * cleaned_df.shape[1])) * 100, 2)
+        st.metric("Null Values (%)", f"{null_percentage}%")
+        
+    # Show suggestions from LLM
+    display_llm_report(suggestions)
+    
+
+    # Convert to downloadable format
+    if file_type == ".csv":
+        cleaned_file = cleaned_df.to_csv(index=False).encode("utf-8")
+        download_name = "cleaned_data.csv"
+        mime_type = "text/csv"
+    elif file_type == ".tsv":
+        cleaned_file = cleaned_df.to_csv(index=False, sep="\t").encode("utf-8")
+        download_name = "cleaned_data.tsv"
+        mime_type = "text/tsv"
+    elif file_type == ".xlsx":
+        output = BytesIO()
+        with pd.ExcelWriter(output, engine='xlsxwriter') as writer:
+            cleaned_df.to_excel(writer, index=False)
+        cleaned_file = output.getvalue()
+        download_name = "cleaned_data.xlsx"
+        mime_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+
+    # Download button
+    st.download_button(
+        label="📁 Download Cleaned Data",
+        data=cleaned_file,
+        file_name=download_name,
+        mime=mime_type
+    )

data_clean_final.py

@@ -0,0 +1,130 @@
+# data_clean_final.py
+
+import pandas as pd
+import os
+import streamlit as st
+from dotenv import load_dotenv
+import tempfile
+from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+import torch
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Load local TinyLlama model
+model_name = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32)
+generator = pipeline("text-generation", model=model, tokenizer=tokenizer, device=0 if torch.cuda.is_available() else -1)
+
+
+# Function to get data cleaning suggestions from LLM
+def suggest_llm_fixes_and_fill(column_name, examples):
+
+    examples_text = "\n".join([f"- {ex}" for ex in examples if ex])
+    
+    prompt = (
+        f"""You are a data cleaning assistant. Some entries in the '{column_name}' column are missing or inconsistent.\n
+
+        Examine these sample values.
+
+        {examples_text}
+
+        Return ONLY a valid Python list of tuples, like:
+        [("original_value1", "replacement1", "reason"), ("original_value2", "replacement2", "reason"), ...]
+        No explanation or extra text — just the list.
+        """
+    )
+
+    try:
+        response = generator(prompt, max_new_tokens=200, do_sample=True, temperature=0.7)
+        return response[0]['generated_text'].split(prompt)[-1].strip()
+    
+    except Exception as e:
+        error_message = f"LLM for error column {column_name}: {str(e)}"
+        st.error(error_message)
+        return error_message
+
+def clean_data(file_path):
+    # Support CSV and TSV files
+    # Load data and drop duplicates
+    if file_path.endswith('.tsv'):
+        df = pd.read_csv(file_path, sep='\t').drop_duplicates().copy()
+    else:
+        df = pd.read_csv(file_path).drop_duplicates().copy()
+
+    suggestions_log = []
+
+    # Convert column types
+    for col in df.columns:
+        if df[col].dtype == 'object':
+            df[col] = df[col].str.strip().str.lower()  # Normalize text
+
+            # Escape newline characters
+            df[col] = df[col].str.replace('\n', ' ', regex=False).replace('\r', ' ', regex=False)
+            
+            if any(keyword in col.lower() for keyword in ['date', 'year', 'time', 'timestamp']):
+                df[col] = df[col].str.replace(r'[^\d]', '', regex=True)
+                # Normalize 4-digit year ranges (e.g., 2000-2001, 2000--2001, 20002001)
+                df[col] = df[col].replace(
+                    r'(?<!\d)(\d{4})\s*[-–—./]?\s*(\d{4})(?!\d)', r'\1-\2', regex=True
+                )
+            # Remove currency symbols and commas
+            if df[col].astype(str).str.contains(r'[$,]', na=False, regex=True).any():
+                df[col] = df[col].str.replace(r'[$,]', '', regex=True)
+
+        elif df[col].dtype in ['int64', 'float64'] or pd.api.types.is_numeric_dtype(df[col]):  # Convert numeric columns to proper type
+            df[col] = pd.to_numeric(df[col], errors='coerce')
+
+        # LLM assistance or missing or weird values
+        null_count = df[col].isnull().sum()
+        empty_str_count = (df[col] == '').sum() if df[col].dtype == 'object' else 0
+        pattern_matches = df[col].astype(str).str.contains(r'none|null|n/a|na|\?+missing|unknown',
+                                                           na=False, case=False, regex=True).sum()
+        
+        if null_count > 0 or empty_str_count > 0 or pattern_matches > 0:
+            # Get examples for LLM analysis (both good and bad examples)
+            # Get non-null, non-empty examples
+            good_examples = df[col][df[col].notnull() & (df[col] != '')].drop_duplicates().sample(n=min(5, len(df)), random_state=1)
+
+            # Get bad examples
+            bad_examples = df[col][df[col].isna() | (df[col] == '') | df[col].astype(str).str.contains(r'none|null|n/a|na|\?+missing|unknown', 
+                                                                                                       na=False, case=False, regex=True)].sample(
+                                                                                                           min(5, df[col].isna().sum()), random_state=1)
+            # Combine good and bad examples
+            examples = good_examples + bad_examples
+
+            if examples is not None:
+                llm_suggestion = suggest_llm_fixes_and_fill(col, examples)
+                suggestions_log.append({
+                    'col': col,
+                    'suggestion': llm_suggestion
+                })
+
+                 # Automatically apply replacements from LLM if in expected format
+                # if suggestions_log:
+                #     try:
+                #         parsed = ast.literal_eval(llm_suggestion)
+                #         if isinstance(parsed, list) and all(isinstance(t, tuple) and len(t) == 3 for t in parsed):
+                #             for original, replacement, _ in parsed:
+                #                 df[col] = df[col].replace(original, replacement)
+                #         else:
+                #             raise ValueError("Parsed suggestion is not a list of 3-item tuples.")
+                #     except Exception as e:
+                #         print(f"Failed to apply replacements for column {col}: {e}")
+                #         st.warning(f"❌ Could not parse suggestion for column `{col}`. Make sure the LLM returned a valid Python list of tuples.")
+                #         st.code(llm_suggestion, language="python")
+
+    df = df.reset_index(drop=True)
+
+    return df, suggestions_log
+
+def display_llm_report(suggestions_log):
+    if suggestions_log:
+        st.subheader("🤖 LLM Cleaning Suggestions")
+        for col, suggestion in suggestions_log:
+            st.markdown(f"**Column:** `{col}`")
+            if suggestion:
+                    st.code(suggestion, language="python")
+            else:
+                st.write("No suggestions or LLM response error.")