import gradio as gr import pandas as pd import numpy as np from sklearn.cluster import KMeans from sklearn.preprocessing import StandardScaler from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity import google.generativeai as genai import os import re from datetime import datetime, timedelta import io import warnings warnings.filterwarnings('ignore') # Configure Gemini API def setup_gemini(): """Setup Gemini API with API key from environment or Hugging Face secrets""" try: # Try to get from Hugging Face secrets first from huggingface_hub import HfApi api_key = os.getenv('GEMINI_API_KEY') if not api_key: # If running on HF Spaces, try to get from secrets try: import spaces api_key = spaces.config.GEMINI_API_KEY except: pass if api_key: genai.configure(api_key=api_key) return True else: print("Warning: GEMINI_API_KEY not found. LLM analysis will be limited.") return False except Exception as e: print(f"Error setting up Gemini: {e}") return False # Sample data generator def generate_sample_data(): """Generate sample payment data with potential duplicates""" vendors = [ "ABC Corp", "ABC Corporation", "XYZ Ltd", "XYZ Limited", "Tech Solutions Inc", "Tech Sol Inc", "Global Services", "International Trade Co", "Int'l Trade Company", "Quick Fix LLC", "QuickFix Limited", "Alpha Systems", "Beta Enterprises", "Gamma Holdings", "Delta Corp", "Epsilon Ltd" ] amounts = [1000, 1500, 2000, 2500, 3000, 5000, 7500, 10000] data = [] for i in range(50): # Create some intentional duplicates if i % 10 == 0 and i > 0: # Every 10th record, create a potential duplicate prev_record = data[i-5] vendor = prev_record['vendor_name'] amount = prev_record['amount'] + np.random.uniform(-50, 50) # Slight variation date = pd.to_datetime(prev_record['payment_date']) + timedelta(days=np.random.randint(1, 5)) else: vendor = np.random.choice(vendors) amount = np.random.choice(amounts) + np.random.uniform(-100, 100) date = datetime.now() - timedelta(days=np.random.randint(1, 365)) data.append({ 'payment_id': f"PAY_{i+1:04d}", 'vendor_name': vendor, 'amount': round(amount, 2), 'payment_date': date.strftime('%Y-%m-%d'), 'invoice_number': f"INV_{np.random.randint(1000, 9999)}", 'description': f"Payment for services - {np.random.choice(['Consulting', 'Software', 'Hardware', 'Maintenance'])}", 'payment_method': np.random.choice(['Wire Transfer', 'Check', 'ACH', 'Credit Card']) }) return pd.DataFrame(data) class VendorDuplicateAnalyzer: def __init__(self): self.scaler = StandardScaler() self.vectorizer = TfidfVectorizer(stop_words='english', max_features=100) self.gemini_available = setup_gemini() def preprocess_data(self, df): """Preprocess the data for analysis""" # Clean vendor names df['vendor_clean'] = df['vendor_name'].str.lower().str.strip() df['vendor_clean'] = df['vendor_clean'].str.replace(r'[^\w\s]', '', regex=True) df['vendor_clean'] = df['vendor_clean'].str.replace(r'\s+', ' ', regex=True) # Convert amount to numeric df['amount'] = pd.to_numeric(df['amount'], errors='coerce') # Convert date df['payment_date'] = pd.to_datetime(df['payment_date'], errors='coerce') return df def extract_features(self, df): """Extract features for clustering""" features = [] # Vendor name similarity features using TF-IDF vendor_tfidf = self.vectorizer.fit_transform(df['vendor_clean'].fillna('')) # Amount features (normalized) amount_features = self.scaler.fit_transform(df[['amount']].fillna(0)) # Date features (days since earliest date) min_date = df['payment_date'].min() date_features = (df['payment_date'] - min_date).dt.days.fillna(0).values.reshape(-1, 1) date_features = self.scaler.fit_transform(date_features) # Combine features features = np.hstack([ vendor_tfidf.toarray(), amount_features, date_features ]) return features def find_duplicates_kmeans(self, df, n_clusters=5): """Find potential duplicates using K-means clustering""" df_clean = self.preprocess_data(df.copy()) features = self.extract_features(df_clean) # Apply K-means clustering kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10) clusters = kmeans.fit_predict(features) df_clean['cluster'] = clusters # Find potential duplicates within clusters duplicate_pairs = [] for cluster_id in range(n_clusters): cluster_data = df_clean[df_clean['cluster'] == cluster_id] if len(cluster_data) > 1: # Calculate pairwise similarities within cluster cluster_indices = cluster_data.index.tolist() for i in range(len(cluster_indices)): for j in range(i + 1, len(cluster_indices)): idx1, idx2 = cluster_indices[i], cluster_indices[j] # Calculate similarity score similarity_score = self.calculate_similarity( df_clean.loc[idx1], df_clean.loc[idx2] ) if similarity_score > 0.6: # Threshold for potential duplicates duplicate_pairs.append({ 'index1': idx1, 'index2': idx2, 'similarity_score': similarity_score, 'cluster': cluster_id }) return duplicate_pairs, df_clean def calculate_similarity(self, row1, row2): """Calculate similarity between two payment records""" # Vendor name similarity (Jaccard similarity) vendor1_words = set(row1['vendor_clean'].split()) vendor2_words = set(row2['vendor_clean'].split()) if len(vendor1_words) == 0 and len(vendor2_words) == 0: vendor_sim = 1.0 elif len(vendor1_words) == 0 or len(vendor2_words) == 0: vendor_sim = 0.0 else: vendor_sim = len(vendor1_words & vendor2_words) / len(vendor1_words | vendor2_words) # Amount similarity (using relative difference) amount_diff = abs(row1['amount'] - row2['amount']) / max(row1['amount'], row2['amount'], 1e-6) amount_sim = max(0, 1 - amount_diff) # Date similarity (within 30 days gets high similarity) date_diff = abs((row1['payment_date'] - row2['payment_date']).days) date_sim = max(0, 1 - date_diff / 30) # Combined similarity (weighted average) total_sim = (vendor_sim * 0.5 + amount_sim * 0.3 + date_sim * 0.2) return total_sim def analyze_with_llm(self, duplicate_pairs, df): """Analyze duplicates using Gemini LLM""" if not self.gemini_available: return "LLM analysis not available. Please set GEMINI_API_KEY." try: model = genai.GenerativeModel('gemini-pro') analysis_text = "Analyze these potential duplicate payments:\n\n" for i, pair in enumerate(duplicate_pairs[:5]): # Limit to first 5 pairs row1 = df.iloc[pair['index1']] row2 = df.iloc[pair['index2']] analysis_text += f"Potential Duplicate Pair {i+1}:\n" analysis_text += f"Payment 1: {row1['vendor_name']} - ${row1['amount']} on {row1['payment_date']}\n" analysis_text += f"Payment 2: {row2['vendor_name']} - ${row2['amount']} on {row2['payment_date']}\n" analysis_text += f"Similarity Score: {pair['similarity_score']:.3f}\n\n" prompt = f""" {analysis_text} Please analyze these potential duplicate payments and provide: 1. Assessment of whether each pair is likely a true duplicate 2. Risk level (High/Medium/Low) for each pair 3. Recommended actions for investigation 4. Overall summary of findings Focus on practical business insights for payment processing teams. """ response = model.generate_content(prompt) return response.text except Exception as e: return f"Error in LLM analysis: {str(e)}" # Initialize analyzer analyzer = VendorDuplicateAnalyzer() def analyze_file(file, n_clusters): """Main analysis function""" try: if file is None: return "Please upload a file first.", None, "" # Read file if file.name.endswith('.csv'): df = pd.read_csv(file.name) elif file.name.endswith(('.xlsx', '.xls')): df = pd.read_excel(file.name) else: return "Unsupported file format. Please upload CSV or Excel file.", None, "" # Validate required columns required_columns = ['vendor_name', 'amount', 'payment_date'] missing_columns = [col for col in required_columns if col not in df.columns] if missing_columns: return f"Missing required columns: {missing_columns}. Required: {required_columns}", None, "" # Find duplicates duplicate_pairs, df_processed = analyzer.find_duplicates_kmeans(df, n_clusters) if not duplicate_pairs: return "No potential duplicates found.", None, "No duplicates detected in the uploaded data." # Create results DataFrame results = [] for pair in duplicate_pairs: row1 = df.iloc[pair['index1']] row2 = df.iloc[pair['index2']] results.append({ 'Pair_ID': len(results) + 1, 'Vendor_1': row1['vendor_name'], 'Amount_1': row1['amount'], 'Date_1': row1['payment_date'], 'Vendor_2': row2['vendor_name'], 'Amount_2': row2['amount'], 'Date_2': row2['payment_date'], 'Similarity_Score': f"{pair['similarity_score']:.3f}", 'Cluster': pair['cluster'] }) results_df = pd.DataFrame(results) # LLM Analysis llm_analysis = analyzer.analyze_with_llm(duplicate_pairs, df) status_message = f"Analysis complete! Found {len(duplicate_pairs)} potential duplicate pairs." return status_message, results_df, llm_analysis except Exception as e: return f"Error analyzing file: {str(e)}", None, "" def load_sample_data(): """Load sample data""" sample_df = generate_sample_data() return sample_df.to_csv(index=False), "Sample data loaded successfully!" # Create Gradio interface def create_interface(): with gr.Blocks(title="Vendor Duplicate Payment Analyzer", theme=gr.themes.Soft()) as demo: gr.HTML("""

🔍 Vendor Duplicate Payment Analyzer

Using K-means Clustering and AI for Duplicate Detection

""") with gr.Row(): with gr.Column(scale=1): gr.HTML("

📁 Upload Data

") file_input = gr.File( label="Upload Payment CSV/Excel", file_types=[".csv", ".xlsx", ".xls"], type="filepath" ) with gr.Row(): sample_btn = gr.Button("📊 Load Sample Data", variant="secondary") sample_output = gr.File(label="Sample Data", visible=False) gr.HTML("

⚙️ Parameters

") n_clusters = gr.Slider( minimum=2, maximum=10, value=5, step=1, label="Number of Clusters", info="K-means will group similar payments into this many clusters" ) analyze_btn = gr.Button("🔍 Analyze with K-means", variant="primary", size="lg") with gr.Column(scale=2): gr.HTML("

📊 Results

") status_output = gr.Textbox( label="Analysis Status", value="Upload data and click 'Analyze' to begin...", interactive=False ) results_output = gr.Dataframe( label="Potential Duplicate Pairs Found by K-means", headers=["Pair_ID", "Vendor_1", "Amount_1", "Date_1", "Vendor_2", "Amount_2", "Date_2", "Similarity_Score", "Cluster"], interactive=False ) gr.HTML("

🤖 AI Analysis

") llm_output = gr.Textbox( label="Gemini AI Analysis", lines=10, placeholder="AI analysis will appear here after processing...", interactive=False ) # Event handlers sample_btn.click( fn=load_sample_data, outputs=[sample_output, status_output] ) analyze_btn.click( fn=analyze_file, inputs=[file_input, n_clusters], outputs=[status_output, results_output, llm_output] ) # Instructions gr.HTML("""

📋 Instructions:

  1. Upload Data: CSV/Excel file with columns: vendor_name, amount, payment_date
  2. Set Parameters: Adjust number of clusters (more clusters = more granular grouping)
  3. Analyze: Click analyze to run K-means clustering and AI analysis
  4. Review Results: Check potential duplicates and AI recommendations

🔧 Setup for Hugging Face Spaces:

Set your GEMINI_API_KEY in the Hugging Face Spaces secrets for AI analysis.

""") return demo # Launch the app if __name__ == "__main__": demo = create_interface() demo.launch(share=True)