Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| import pandas as pd | |
| import numpy as np | |
| from sklearn.cluster import KMeans | |
| from sklearn.preprocessing import StandardScaler | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.metrics.pairwise import cosine_similarity | |
| import google.generativeai as genai | |
| import io | |
| import re | |
| from datetime import datetime | |
| import warnings | |
| warnings.filterwarnings('ignore') | |
| class VendorDuplicateAnalyzer: | |
| def __init__(self): | |
| self.scaler = StandardScaler() | |
| self.vectorizer = TfidfVectorizer(max_features=100, stop_words='english') | |
| self.gemini_api_key = None | |
| def configure_gemini(self, api_key): | |
| """Configure Gemini API""" | |
| try: | |
| genai.configure(api_key=api_key) | |
| self.gemini_api_key = api_key | |
| return "✅ Gemini API configured successfully" | |
| except Exception as e: | |
| return f"❌ Error configuring Gemini API: {str(e)}" | |
| def load_sample_data(self): | |
| """Generate sample payment data""" | |
| sample_data = { | |
| 'Payment_ID': ['PAY001', 'PAY002', 'PAY003', 'PAY004', 'PAY005', 'PAY006', 'PAY007', 'PAY008'], | |
| 'Vendor_Name': ['ABC Corp', 'ABC Corporation', 'XYZ Ltd', 'XYZ Limited', 'Tech Solutions', 'TechSolutions Inc', 'ABC Corp', 'Global Services'], | |
| 'Amount': [1500.00, 1500.00, 2300.50, 2300.50, 890.25, 890.00, 1500.00, 1200.75], | |
| 'Date': ['2024-01-15', '2024-01-15', '2024-02-10', '2024-02-10', '2024-03-05', '2024-03-05', '2024-01-16', '2024-04-20'], | |
| 'Invoice_Number': ['INV-001', 'INV-001', 'INV-456', 'INV-456', 'INV-789', 'INV-790', 'INV-002', 'INV-333'], | |
| 'Description': ['Office supplies', 'Office supplies purchase', 'Software license', 'Software licensing', 'Consulting services', 'Consulting work', 'Office supplies', 'Maintenance service'] | |
| } | |
| return pd.DataFrame(sample_data) | |
| def preprocess_data(self, df): | |
| """Preprocess the payment data for analysis""" | |
| df = df.copy() | |
| # Convert date to datetime | |
| if 'Date' in df.columns: | |
| df['Date'] = pd.to_datetime(df['Date'], errors='coerce') | |
| df['Date_Numeric'] = df['Date'].astype('int64') // 10**9 # Convert to timestamp | |
| # Clean vendor names | |
| if 'Vendor_Name' in df.columns: | |
| df['Vendor_Clean'] = df['Vendor_Name'].str.lower().str.strip() | |
| df['Vendor_Clean'] = df['Vendor_Clean'].str.replace(r'[^\w\s]', '', regex=True) | |
| # Clean amounts | |
| if 'Amount' in df.columns: | |
| df['Amount'] = pd.to_numeric(df['Amount'], errors='coerce') | |
| return df | |
| def extract_features(self, df): | |
| """Extract features for clustering""" | |
| features = [] | |
| feature_names = [] | |
| # Amount feature | |
| if 'Amount' in df.columns: | |
| amounts = df['Amount'].fillna(0).values.reshape(-1, 1) | |
| features.append(amounts) | |
| feature_names.append('Amount') | |
| # Date feature | |
| if 'Date_Numeric' in df.columns: | |
| dates = df['Date_Numeric'].fillna(0).values.reshape(-1, 1) | |
| features.append(dates) | |
| feature_names.append('Date') | |
| # Vendor name similarity | |
| if 'Vendor_Clean' in df.columns: | |
| vendor_tfidf = self.vectorizer.fit_transform(df['Vendor_Clean'].fillna('')) | |
| features.append(vendor_tfidf.toarray()) | |
| feature_names.extend([f'Vendor_Feature_{i}' for i in range(vendor_tfidf.shape[1])]) | |
| # Description similarity | |
| if 'Description' in df.columns: | |
| desc_vectorizer = TfidfVectorizer(max_features=50, stop_words='english') | |
| desc_tfidf = desc_vectorizer.fit_transform(df['Description'].fillna('')) | |
| features.append(desc_tfidf.toarray()) | |
| feature_names.extend([f'Desc_Feature_{i}' for i in range(desc_tfidf.shape[1])]) | |
| # Combine all features | |
| if features: | |
| combined_features = np.hstack(features) | |
| return combined_features, feature_names | |
| else: | |
| return np.array([]), [] | |
| def find_duplicates_kmeans(self, df, n_clusters=5): | |
| """Find potential duplicates using K-means clustering""" | |
| if len(df) < 2: | |
| return df, "Not enough data for analysis" | |
| # Preprocess data | |
| df_processed = self.preprocess_data(df) | |
| # Extract features | |
| features, feature_names = self.extract_features(df_processed) | |
| if features.size == 0: | |
| return df, "No suitable features found for analysis" | |
| # Normalize features | |
| features_scaled = self.scaler.fit_transform(features) | |
| # Apply K-means clustering | |
| n_clusters = min(n_clusters, len(df)) | |
| kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10) | |
| clusters = kmeans.fit_predict(features_scaled) | |
| # Add cluster information to dataframe | |
| df_result = df.copy() | |
| df_result['Cluster'] = clusters | |
| df_result['Potential_Duplicate'] = 'No' | |
| # Identify potential duplicates (clusters with multiple entries) | |
| cluster_counts = pd.Series(clusters).value_counts() | |
| duplicate_clusters = cluster_counts[cluster_counts > 1].index | |
| for cluster_id in duplicate_clusters: | |
| cluster_mask = df_result['Cluster'] == cluster_id | |
| df_result.loc[cluster_mask, 'Potential_Duplicate'] = 'Yes' | |
| return df_result, f"Analysis complete. Found {len(duplicate_clusters)} clusters with potential duplicates." | |
| async def analyze_with_gemini(self, df_duplicates, api_key): | |
| """Analyze duplicates using Gemini AI""" | |
| if not api_key: | |
| return "Please provide Gemini API key" | |
| try: | |
| genai.configure(api_key=api_key) | |
| model = genai.GenerativeModel('gemini-pro') | |
| # Filter only potential duplicates | |
| duplicates = df_duplicates[df_duplicates['Potential_Duplicate'] == 'Yes'] | |
| if len(duplicates) == 0: | |
| return "No potential duplicates found to analyze" | |
| # Prepare data for Gemini analysis | |
| analysis_text = "Analyze these potential duplicate payments:\n\n" | |
| for cluster_id in duplicates['Cluster'].unique(): | |
| cluster_data = duplicates[duplicates['Cluster'] == cluster_id] | |
| analysis_text += f"Cluster {cluster_id}:\n" | |
| for _, row in cluster_data.iterrows(): | |
| analysis_text += f"- ID: {row.get('Payment_ID', 'N/A')}, Vendor: {row.get('Vendor_Name', 'N/A')}, Amount: {row.get('Amount', 'N/A')}, Date: {row.get('Date', 'N/A')}\n" | |
| analysis_text += "\n" | |
| analysis_text += "\nPlease analyze these clusters and provide:\n1. Confidence level for each duplicate pair\n2. Reasoning for duplicate classification\n3. Recommendations for action" | |
| response = model.generate_content(analysis_text) | |
| return response.text | |
| except Exception as e: | |
| return f"Error with Gemini analysis: {str(e)}" | |
| # Initialize the analyzer | |
| analyzer = VendorDuplicateAnalyzer() | |
| def process_file(file, n_clusters, gemini_key): | |
| """Process uploaded file and analyze duplicates""" | |
| if file is None: | |
| return None, "Please upload a file", "" | |
| try: | |
| # Read file | |
| if file.name.endswith('.csv'): | |
| df = pd.read_csv(file) | |
| elif file.name.endswith(('.xlsx', '.xls')): | |
| df = pd.read_excel(file) | |
| else: | |
| return None, "Please upload a CSV or Excel file", "" | |
| # Analyze duplicates | |
| result_df, status_msg = analyzer.find_duplicates_kmeans(df, n_clusters) | |
| # Gemini analysis | |
| gemini_analysis = "" | |
| if gemini_key: | |
| import asyncio | |
| try: | |
| gemini_analysis = asyncio.run(analyzer.analyze_with_gemini(result_df, gemini_key)) | |
| except: | |
| gemini_analysis = "Gemini analysis not available" | |
| return result_df, status_msg, gemini_analysis | |
| except Exception as e: | |
| return None, f"Error processing file: {str(e)}", "" | |
| def load_sample(): | |
| """Load sample data""" | |
| sample_df = analyzer.load_sample_data() | |
| return sample_df, "Sample data loaded successfully", "" | |
| def analyze_sample(n_clusters, gemini_key): | |
| """Analyze sample data""" | |
| sample_df = analyzer.load_sample_data() | |
| result_df, status_msg = analyzer.find_duplicates_kmeans(sample_df, n_clusters) | |
| # Gemini analysis | |
| gemini_analysis = "" | |
| if gemini_key: | |
| import asyncio | |
| try: | |
| gemini_analysis = asyncio.run(analyzer.analyze_with_gemini(result_df, gemini_key)) | |
| except: | |
| gemini_analysis = "Gemini analysis not available" | |
| return result_df, status_msg, gemini_analysis | |
| # Create Gradio interface | |
| with gr.Blocks(theme=gr.themes.Soft(), title="Vendor Duplicate Analyzer") as app: | |
| gr.HTML(""" | |
| <div style="text-align: center; padding: 20px; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); border-radius: 10px; margin-bottom: 20px;"> | |
| <h1 style="color: white; margin: 0;">🔍 Vendor Duplicate Analyzer</h1> | |
| <p style="color: white; margin: 5px 0;">Using K-means Clustering & Gemini AI for Duplicate Detection</p> | |
| </div> | |
| """) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| gr.HTML("<h3>📤 Upload Data</h3>") | |
| file_input = gr.File( | |
| label="Upload Payment CSV/Excel", | |
| file_types=[".csv", ".xlsx", ".xls"] | |
| ) | |
| with gr.Row(): | |
| load_sample_btn = gr.Button("📊 Load Sample Data", variant="secondary") | |
| analyze_btn = gr.Button("🔍 Analyze with K-means", variant="primary") | |
| gr.HTML("<h3>⚙️ Parameters</h3>") | |
| n_clusters = gr.Slider( | |
| minimum=2, | |
| maximum=10, | |
| value=5, | |
| step=1, | |
| label="Number of Clusters", | |
| info="K-means will group similar payments into this many clusters" | |
| ) | |
| gemini_key = gr.Textbox( | |
| label="Gemini API Key", | |
| placeholder="Enter your Gemini API key for AI analysis", | |
| type="password" | |
| ) | |
| with gr.Column(scale=2): | |
| gr.HTML("<h3>📊 Results</h3>") | |
| status_output = gr.Textbox( | |
| label="Analysis Status", | |
| placeholder="Upload data and click 'Analyze' to begin...", | |
| interactive=False | |
| ) | |
| results_table = gr.Dataframe( | |
| label="Potential Duplicate Pairs Found by K-means", | |
| interactive=False, | |
| wrap=True | |
| ) | |
| gr.HTML("<h3>🤖 AI Analysis</h3>") | |
| gemini_output = gr.Textbox( | |
| label="Gemini AI Analysis", | |
| placeholder="AI analysis will appear here...", | |
| lines=10, | |
| interactive=False | |
| ) | |
| # Event handlers | |
| file_input.upload( | |
| fn=process_file, | |
| inputs=[file_input, n_clusters, gemini_key], | |
| outputs=[results_table, status_output, gemini_output] | |
| ) | |
| load_sample_btn.click( | |
| fn=load_sample, | |
| outputs=[results_table, status_output, gemini_output] | |
| ) | |
| analyze_btn.click( | |
| fn=analyze_sample, | |
| inputs=[n_clusters, gemini_key], | |
| outputs=[results_table, status_output, gemini_output] | |
| ) | |
| gr.HTML(""" | |
| <div style="margin-top: 20px; padding: 15px; background-color: #f0f0f0; border-radius: 5px;"> | |
| <h4>📋 Instructions:</h4> | |
| <ol> | |
| <li>Upload your payment CSV/Excel file or load sample data</li> | |
| <li>Adjust the number of clusters for K-means analysis</li> | |
| <li>Optionally add your Gemini API key for AI-powered analysis</li> | |
| <li>Click 'Analyze' to detect potential duplicates</li> | |
| </ol> | |
| <p><strong>Expected columns:</strong> Payment_ID, Vendor_Name, Amount, Date, Invoice_Number, Description</p> | |
| </div> | |
| """) | |
| if __name__ == "__main__": | |
| app.launch(share=True, debug=True) |