prernajeet14 commited on
Commit
5412dfc
·
verified ·
1 Parent(s): 38349d7

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +605 -0
app.py ADDED
@@ -0,0 +1,605 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import pandas as pd
3
+ import numpy as np
4
+ import matplotlib.pyplot as plt
5
+ import seaborn as sns
6
+ import gradio as gr
7
+ import plotly.express as px
8
+ import plotly.graph_objects as go
9
+ from sklearn.ensemble import IsolationForest
10
+ from sklearn.preprocessing import StandardScaler
11
+ import google.generativeai as genai
12
+ from datetime import datetime, timedelta
13
+ import json
14
+ import tempfile
15
+
16
+ # Set Google Generative AI API key from Hugging Face Spaces secrets
17
+ genai.configure(api_key=os.environ.get("GEMINI_API_KEY"))
18
+
19
+ def analyze_dataset_structure(df):
20
+ """Use Google Gemini to analyze the dataset structure and identify relevant columns"""
21
+ gemini_api_key = os.environ.get("GEMINI_API_KEY")
22
+ if not gemini_api_key:
23
+ return None, "Gemini API key not found. Please add it to the Hugging Face Spaces secrets."
24
+
25
+ try:
26
+ # Get basic dataset info
27
+ sample_data = df.head(3).copy()
28
+
29
+ # Convert any non-serializable data types to strings
30
+ for col in sample_data.columns:
31
+ if pd.api.types.is_datetime64_any_dtype(sample_data[col]):
32
+ sample_data[col] = sample_data[col].astype(str)
33
+ elif isinstance(sample_data[col].iloc[0], (np.int64, np.float64)):
34
+ sample_data[col] = sample_data[col].astype(float)
35
+
36
+ # Now convert to dict
37
+ sample_data_dict = sample_data.to_dict(orient='records')
38
+
39
+ column_info = []
40
+
41
+ for col in df.columns:
42
+ dtype = str(df[col].dtype)
43
+ unique_values = len(df[col].unique())
44
+ null_percentage = round((df[col].isna().sum() / len(df)) * 100, 2)
45
+
46
+ # Handle sample values more carefully
47
+ try:
48
+ sample_values = df[col].dropna().sample(min(3, len(df[col].dropna()))).tolist()
49
+ # Convert numpy types to native Python types
50
+ if isinstance(sample_values, list):
51
+ sample_values = [item.item() if hasattr(item, 'item') else str(item) for item in sample_values]
52
+ sample_values_str = str(sample_values)[:100] # Limit sample length
53
+ except:
54
+ sample_values_str = "Error getting sample values"
55
+
56
+ column_info.append({
57
+ "column_name": col,
58
+ "data_type": dtype,
59
+ "unique_values_count": unique_values,
60
+ "null_percentage": null_percentage,
61
+ "sample_values": sample_values_str
62
+ })
63
+
64
+ # Create prompt for Gemini
65
+ prompt = f"""
66
+ Analyze this transaction dataset structure to identify the purpose of each column.
67
+
68
+ Dataset Information:
69
+ - Number of rows: {len(df)}
70
+ - Number of columns: {len(df.columns)}
71
+
72
+ Column Information:
73
+ {json.dumps(column_info, indent=2)}
74
+
75
+ Sample Data:
76
+ {json.dumps(sample_data_dict, indent=2)}
77
+
78
+ For each column in the dataset, identify its likely purpose in a transaction dataset.
79
+ Specifically identify:
80
+
81
+ 1. Which column is likely the transaction ID or reference number
82
+ 2. Which column represents the transaction amount or value
83
+ 3. Which column represents the timestamp or date of the transaction
84
+ 4. Which column represents the user ID, account ID, or customer identifier
85
+ 5. Which column might represent location information
86
+ 6. Which columns might be useful for fraud detection (e.g., IP address, device info, transaction status)
87
+
88
+ Return your analysis as a JSON object with this structure:
89
+ {{
90
+ "id_column": "column_name",
91
+ "amount_column": "column_name",
92
+ "timestamp_column": "column_name",
93
+ "user_column": "column_name",
94
+ "location_column": "column_name",
95
+ "fraud_indicator_columns": ["column1", "column2"],
96
+ "column_descriptions": {{
97
+ "column_name": "description of purpose"
98
+ }}
99
+ }}
100
+
101
+ Include only columns that you're reasonably confident about, and use null for any category where you can't identify a matching column.
102
+ """
103
+
104
+ # Create Gemini model
105
+ model = genai.GenerativeModel('gemini-pro')
106
+
107
+ # Call Gemini API
108
+ response = model.generate_content(prompt)
109
+
110
+ # Extract JSON from response text
111
+ response_text = response.text
112
+ # Find JSON content within the response
113
+ json_start = response_text.find('{')
114
+ json_end = response_text.rfind('}') + 1
115
+ if json_start != -1 and json_end != -1:
116
+ json_content = response_text[json_start:json_end]
117
+ structure_analysis = json.loads(json_content)
118
+ else:
119
+ raise ValueError("No valid JSON found in response")
120
+
121
+ # Also get a natural language explanation
122
+ explanation_prompt = f"""
123
+ Based on your analysis of the dataset structure, provide a brief natural language explanation of:
124
+ 1. What kind of transactions this dataset appears to contain
125
+ 2. What the key columns are and what they represent
126
+ 3. What approach would be best for detecting anomalies or fraud in this specific dataset
127
+
128
+ Keep your explanation concise and focused on the unique characteristics of this dataset.
129
+
130
+ Previous analysis: {json.dumps(structure_analysis)}
131
+ """
132
+
133
+ explanation_response = model.generate_content(explanation_prompt)
134
+ explanation = explanation_response.text
135
+
136
+ return structure_analysis, explanation
137
+
138
+ except Exception as e:
139
+ import traceback
140
+ error_trace = traceback.format_exc()
141
+ return None, f"Error analyzing dataset structure: {str(e)}\n\nTrace: {error_trace}"
142
+
143
+ def load_and_preprocess_data(file):
144
+ """Load and preprocess transaction data from CSV or Excel file"""
145
+ if file is None:
146
+ return None, None, None # Return three values instead of two
147
+
148
+ # Get file extension
149
+ file_extension = os.path.splitext(file.name)[1].lower()
150
+
151
+ # Read file based on extension
152
+ if file_extension == '.csv':
153
+ df = pd.read_csv(file.name)
154
+ elif file_extension in ['.xlsx', '.xls']:
155
+ df = pd.read_excel(file.name)
156
+ else:
157
+ raise ValueError("Unsupported file format. Please upload a CSV or Excel file.")
158
+
159
+ # Check if the DataFrame is empty
160
+ if df.empty:
161
+ raise ValueError("The uploaded file is empty.")
162
+
163
+ # Analyze dataset structure with LLM
164
+ column_mapping, dataset_explanation = analyze_dataset_structure(df)
165
+
166
+ # If LLM analysis failed, perform basic preprocessing
167
+ if column_mapping is None:
168
+ return df, dataset_explanation, None # Return three values with column_mapping as None
169
+
170
+ # Process the data based on identified columns
171
+ processed_df = df.copy()
172
+
173
+ # Convert timestamp to datetime if identified
174
+ timestamp_col = column_mapping.get("timestamp_column")
175
+ if timestamp_col and timestamp_col in df.columns:
176
+ try:
177
+ processed_df[timestamp_col] = pd.to_datetime(df[timestamp_col])
178
+ except:
179
+ print(f"Warning: Could not convert {timestamp_col} to datetime format.")
180
+
181
+ # Ensure amount column is numeric if identified
182
+ amount_col = column_mapping.get("amount_column")
183
+ if amount_col and amount_col in df.columns:
184
+ try:
185
+ processed_df[amount_col] = pd.to_numeric(df[amount_col])
186
+ except:
187
+ print(f"Warning: Could not convert {amount_col} to numeric values.")
188
+
189
+ return processed_df, dataset_explanation, column_mapping
190
+
191
+ def detect_fraud_and_anomalies(df, column_mapping):
192
+ """Detect fraud and anomalies in transaction data based on LLM-identified columns"""
193
+ # Create feature set for anomaly detection
194
+ features = pd.DataFrame()
195
+
196
+ # Add amount feature if available
197
+ amount_col = column_mapping.get("amount_column")
198
+ if amount_col and amount_col in df.columns:
199
+ features['amount'] = df[amount_col]
200
+
201
+ # Add time-based features if available
202
+ timestamp_col = column_mapping.get("timestamp_column")
203
+ if timestamp_col and timestamp_col in df.columns and pd.api.types.is_datetime64_any_dtype(df[timestamp_col]):
204
+ # Extract hour and day of week
205
+ features['hour_of_day'] = pd.to_numeric(df[timestamp_col].dt.hour)
206
+ features['day_of_week'] = pd.to_numeric(df[timestamp_col].dt.dayofweek)
207
+
208
+ # Add location feature if available
209
+ location_col = column_mapping.get("location_column")
210
+ if location_col and location_col in df.columns:
211
+ # One-hot encode location
212
+ location_dummies = pd.get_dummies(df[location_col], prefix='location')
213
+ features = pd.concat([features, location_dummies], axis=1)
214
+
215
+ # Add fraud indicator columns if identified
216
+ fraud_indicators = column_mapping.get("fraud_indicator_columns", [])
217
+ for col in fraud_indicators:
218
+ if col in df.columns:
219
+ if pd.api.types.is_numeric_dtype(df[col]):
220
+ features[col] = df[col]
221
+ else:
222
+ # One-hot encode categorical indicators
223
+ indicator_dummies = pd.get_dummies(df[col], prefix=col)
224
+ features = pd.concat([features, indicator_dummies], axis=1)
225
+
226
+ # If still no features available, use all numeric columns
227
+ if features.empty or features.shape[1] < 2:
228
+ numeric_cols = df.select_dtypes(include=['number']).columns.tolist()
229
+ if numeric_cols:
230
+ for col in numeric_cols:
231
+ if col not in features.columns:
232
+ features[col] = df[col]
233
+
234
+ # If still not enough features, add dummy feature
235
+ if features.empty or features.shape[1] < 2:
236
+ features['dummy1'] = np.random.random(len(df))
237
+ features['dummy2'] = np.random.random(len(df))
238
+
239
+ # Standardize features
240
+ scaler = StandardScaler()
241
+ scaled_features = scaler.fit_transform(features)
242
+
243
+ # Apply Isolation Forest for anomaly detection
244
+ clf = IsolationForest(contamination=0.05, random_state=42)
245
+ anomaly_scores = clf.fit_predict(scaled_features)
246
+
247
+ # Create a result DataFrame with original data and anomaly scores
248
+ result_df = df.copy()
249
+
250
+ # Add anomaly flags
251
+ result_df['anomaly_score'] = anomaly_scores
252
+ result_df['is_anomaly'] = result_df['anomaly_score'] == -1
253
+
254
+ # Initialize fraud indicators
255
+ result_df['high_amount'] = False
256
+ result_df['unusual_hour'] = False
257
+ result_df['high_frequency'] = False
258
+ result_df['rapid_succession'] = False
259
+
260
+ # 1. Unusually large transactions (if amount column is available)
261
+ if amount_col and amount_col in df.columns:
262
+ amount_threshold = df[amount_col].quantile(0.95)
263
+ result_df['high_amount'] = df[amount_col] > amount_threshold
264
+
265
+ # 2. Transactions occurring at unusual hours (if timestamp available)
266
+ if timestamp_col and timestamp_col in df.columns and pd.api.types.is_datetime64_any_dtype(df[timestamp_col]):
267
+ hours = np.array(df[timestamp_col].dt.hour)
268
+ result_df['unusual_hour'] = np.isin(hours, [0, 1, 2, 3, 4])
269
+
270
+ # 3. Calculate transaction frequency by user or account (if available)
271
+ user_col = column_mapping.get("user_column")
272
+ if user_col and user_col in df.columns:
273
+ transaction_counts = df.groupby(user_col).size().reset_index(name='transaction_count')
274
+ result_df = result_df.merge(transaction_counts, on=user_col, how='left')
275
+ result_df['high_frequency'] = result_df['transaction_count'] > result_df['transaction_count'].quantile(0.9)
276
+
277
+ # 4. Velocity check: multiple transactions in short time period
278
+ if timestamp_col and user_col and timestamp_col in df.columns and user_col in df.columns:
279
+ if pd.api.types.is_datetime64_any_dtype(df[timestamp_col]):
280
+ velocity_df = df[[timestamp_col, user_col]].copy().sort_values([user_col, timestamp_col])
281
+ velocity_df['time_diff'] = velocity_df.groupby(user_col)[timestamp_col].diff()
282
+
283
+ # Handle potential NaT values
284
+ velocity_df['time_diff_seconds'] = velocity_df['time_diff'].dt.total_seconds().fillna(0)
285
+ velocity_df['rapid_succession'] = velocity_df['time_diff_seconds'] < 300 # Less than 5 minutes
286
+
287
+ # Map back to the original DataFrame
288
+ result_df = result_df.merge(
289
+ velocity_df[['rapid_succession']],
290
+ left_index=True,
291
+ right_index=True,
292
+ how='left'
293
+ )
294
+ result_df['rapid_succession'] = result_df['rapid_succession'].fillna(False)
295
+
296
+ # Combine all fraud indicators with adaptive weighting
297
+ weights = {
298
+ 'is_anomaly': 3, # Base weight for anomaly detection
299
+ 'high_amount': 2,
300
+ 'unusual_hour': 1,
301
+ 'high_frequency': 1,
302
+ 'rapid_succession': 1
303
+ }
304
+
305
+ # Calculate fraud score based on available indicators
306
+ result_df['fraud_score'] = 0
307
+ for indicator, weight in weights.items():
308
+ if indicator in result_df.columns:
309
+ result_df['fraud_score'] += result_df[indicator].astype(int) * weight
310
+
311
+ # Flag as suspicious if fraud score is above threshold (adapt based on available indicators)
312
+ available_weights = sum([weight for indicator, weight in weights.items() if indicator in result_df.columns])
313
+ threshold = max(3, available_weights * 0.3) # At least 3 or 30% of max possible score
314
+ result_df['is_suspicious'] = result_df['fraud_score'] >= threshold
315
+
316
+ return result_df
317
+
318
+ def create_visualizations(df, column_mapping):
319
+ """Create visualizations for transaction data and anomalies based on LLM-identified columns"""
320
+ visualizations = {}
321
+
322
+ try:
323
+ # Prepare a copy for plotting
324
+ plot_df = df.copy()
325
+
326
+ # Get important columns
327
+ timestamp_col = column_mapping.get("timestamp_column")
328
+ amount_col = column_mapping.get("amount_column")
329
+ user_col = column_mapping.get("user_column")
330
+
331
+ # Convert timestamp to string for plotly if it exists
332
+ if timestamp_col and timestamp_col in plot_df.columns:
333
+ if pd.api.types.is_datetime64_any_dtype(plot_df[timestamp_col]):
334
+ plot_df['timestamp_str'] = plot_df[timestamp_col].dt.strftime('%Y-%m-%d %H:%M:%S')
335
+
336
+ # 1. Distribution of transaction amounts with anomalies highlighted (if amount column exists)
337
+ if amount_col and amount_col in plot_df.columns:
338
+ fig1 = px.histogram(
339
+ plot_df, x=amount_col, color='is_suspicious',
340
+ color_discrete_map={True: 'red', False: 'blue'},
341
+ title='Distribution of Transaction Amounts',
342
+ labels={amount_col: 'Transaction Amount', 'is_suspicious': 'Suspicious'}
343
+ )
344
+ fig1.update_layout(height=500, width=700)
345
+ visualizations['amount_distribution'] = fig1
346
+
347
+ # 2. Time series of transaction amounts (if both timestamp and amount columns exist)
348
+ if timestamp_col and amount_col and 'timestamp_str' in plot_df.columns:
349
+ fig2 = px.scatter(
350
+ plot_df, x='timestamp_str', y=amount_col, color='is_suspicious',
351
+ color_discrete_map={True: 'red', False: 'blue'},
352
+ title='Transaction Amounts Over Time',
353
+ labels={amount_col: 'Transaction Amount', 'timestamp_str': 'Time', 'is_suspicious': 'Suspicious'}
354
+ )
355
+ fig2.update_layout(height=500, width=700)
356
+ visualizations['time_series'] = fig2
357
+
358
+ # 3. Fraud score distribution
359
+ fig3 = px.histogram(
360
+ plot_df, x='fraud_score',
361
+ title='Distribution of Fraud Scores',
362
+ labels={'fraud_score': 'Fraud Score'}
363
+ )
364
+ fig3.update_layout(height=500, width=700)
365
+ visualizations['fraud_score_dist'] = fig3
366
+
367
+ # 4. User transaction frequency (if user column exists)
368
+ if user_col and user_col in plot_df.columns:
369
+ user_counts = plot_df.groupby([user_col, 'is_suspicious']).size().reset_index(name='count')
370
+ # Limit to top 20 users by transaction count
371
+ top_users = plot_df.groupby(user_col).size().sort_values(ascending=False).head(20).index
372
+ user_counts_filtered = user_counts[user_counts[user_col].isin(top_users)]
373
+
374
+ fig4 = px.bar(
375
+ user_counts_filtered, x=user_col, y='count', color='is_suspicious',
376
+ color_discrete_map={True: 'red', False: 'blue'},
377
+ title='Transaction Frequency by User (Top 20)',
378
+ labels={user_col: 'User', 'count': 'Number of Transactions', 'is_suspicious': 'Suspicious'}
379
+ )
380
+ fig4.update_layout(height=500, width=700)
381
+ visualizations['user_frequency'] = fig4
382
+
383
+ # 5. Hourly transaction pattern (if timestamp available)
384
+ if timestamp_col and timestamp_col in plot_df.columns:
385
+ if pd.api.types.is_datetime64_any_dtype(plot_df[timestamp_col]):
386
+ # Get hourly data
387
+ hourly_counts = plot_df.groupby([plot_df[timestamp_col].dt.hour, 'is_suspicious']).size()
388
+ hourly_df = hourly_counts.reset_index()
389
+ hourly_df.columns = ['hour', 'is_suspicious', 'count']
390
+
391
+ fig5 = px.line(
392
+ hourly_df, x='hour', y='count', color='is_suspicious',
393
+ color_discrete_map={True: 'red', False: 'blue'},
394
+ title='Hourly Transaction Pattern',
395
+ labels={'hour': 'Hour of Day', 'count': 'Number of Transactions', 'is_suspicious': 'Suspicious'}
396
+ )
397
+ fig5.update_layout(height=500, width=700)
398
+ visualizations['hourly_pattern'] = fig5
399
+
400
+ except Exception as e:
401
+ print(f"Error in visualization creation: {str(e)}")
402
+
403
+ return visualizations
404
+
405
+ def analyze_transaction_with_ai(transaction_data, suspicious_transactions, column_mapping):
406
+ """Use Google Gemini to analyze suspicious transactions and provide insights"""
407
+ gemini_api_key = os.environ.get("GEMINI_API_KEY")
408
+ if not gemini_api_key:
409
+ return "Gemini API key not found. Please add it to the Hugging Face Spaces secrets."
410
+
411
+ try:
412
+ # Prepare information for Gemini, converting to a JSON-serializable format
413
+ suspicious_sample = suspicious_transactions.head(5).copy()
414
+
415
+ # Convert any datetime columns to string format to make it JSON serializable
416
+ for col in suspicious_sample.columns:
417
+ if pd.api.types.is_datetime64_any_dtype(suspicious_sample[col]):
418
+ suspicious_sample[col] = suspicious_sample[col].astype(str)
419
+ # Convert NumPy types to Python native types
420
+ elif suspicious_sample[col].dtype in (np.int64, np.float64):
421
+ suspicious_sample[col] = suspicious_sample[col].astype(float)
422
+ # Handle boolean columns
423
+ elif suspicious_sample[col].dtype == bool:
424
+ suspicious_sample[col] = suspicious_sample[col].astype(str)
425
+
426
+ # Convert to dictionary
427
+ suspicious_dict = suspicious_sample.to_dict(orient='records')
428
+
429
+ # Get summary statistics
430
+ summary_stats = {
431
+ "total_transactions": int(len(transaction_data)),
432
+ "flagged_transactions": int(len(suspicious_transactions)),
433
+ "flagged_percentage": float(round(len(suspicious_transactions) / len(transaction_data) * 100, 2)),
434
+ }
435
+
436
+ # Add amount-related statistics if available
437
+ amount_col = column_mapping.get("amount_column")
438
+ if amount_col and amount_col in transaction_data.columns:
439
+ summary_stats.update({
440
+ "avg_transaction_amount": float(round(transaction_data[amount_col].mean(), 2)),
441
+ "suspicious_avg_amount": float(round(suspicious_transactions[amount_col].mean(), 2))
442
+ })
443
+
444
+ # Create prompt for Gemini
445
+ prompt = f"""
446
+ Analyze these potentially fraudulent transactions and identify patterns or anomalies:
447
+
448
+ Transaction Data Summary:
449
+ {json.dumps(summary_stats)}
450
+
451
+ Column Mapping:
452
+ {json.dumps(column_mapping)}
453
+
454
+ Sample of Suspicious Transactions:
455
+ {json.dumps(suspicious_dict)}
456
+
457
+ Provide a concise fraud analysis report with:
458
+ 1. Key patterns and red flags in these transactions
459
+ 2. Possible fraud scenarios explaining the anomalies
460
+ 3. Recommended next steps for investigation
461
+ """
462
+
463
+ # Create Gemini model
464
+ model = genai.GenerativeModel('gemini-pro')
465
+
466
+ # Call Gemini API
467
+ response = model.generate_content(prompt)
468
+
469
+ # Return the AI analysis
470
+ return response.text
471
+
472
+ except Exception as e:
473
+ import traceback
474
+ error_trace = traceback.format_exc()
475
+ return f"Error in AI analysis: {str(e)}\n\nTrace: {error_trace}"
476
+
477
+ def process_transactions(file):
478
+ """Main function to process transaction data and detect fraud"""
479
+ try:
480
+ # Load and preprocess data with LLM-based analysis
481
+ processed_df, dataset_explanation, column_mapping = load_and_preprocess_data(file)
482
+
483
+ if processed_df is None:
484
+ return "No file uploaded or error in processing", None, None, None, None, None
485
+
486
+ # If column_mapping is None, only dataset_explanation was returned (containing error message)
487
+ if column_mapping is None:
488
+ return f"Error analyzing dataset: {dataset_explanation}", None, None, None, None, None
489
+
490
+ # Detect fraud and anomalies using the LLM-identified column mapping
491
+ df_with_anomalies = detect_fraud_and_anomalies(processed_df, column_mapping)
492
+
493
+ # Get suspicious transactions
494
+ suspicious_transactions = df_with_anomalies[df_with_anomalies['is_suspicious']]
495
+
496
+ # Create visualizations using the identified columns
497
+ visualizations = create_visualizations(df_with_anomalies, column_mapping)
498
+
499
+ # Basic statistics
500
+ total_transactions = len(df_with_anomalies)
501
+ suspicious_count = len(suspicious_transactions)
502
+ suspicious_percentage = round((suspicious_count / total_transactions) * 100, 2)
503
+
504
+ # Format statistics for display
505
+ stats_summary = f"""
506
+ ## Transaction Analysis Summary
507
+
508
+ - **Total Transactions**: {total_transactions}
509
+ - **Suspicious Transactions**: {suspicious_count} ({suspicious_percentage}%)
510
+ """
511
+
512
+ # Add amount-related statistics if available
513
+ amount_col = column_mapping.get("amount_column")
514
+ if amount_col and amount_col in df_with_anomalies.columns:
515
+ stats_summary += f"""
516
+ - **Total Transaction Value**: ${df_with_anomalies[amount_col].sum():,.2f}
517
+ - **Suspicious Transaction Value**: ${suspicious_transactions[amount_col].sum():,.2f}
518
+ - **Average Transaction Amount**: ${df_with_anomalies[amount_col].mean():,.2f}
519
+ - **Average Suspicious Amount**: ${suspicious_transactions[amount_col].mean():,.2f}
520
+ """
521
+
522
+ # Add dataset explanation from LLM
523
+ stats_summary += f"""
524
+ ## Dataset Analysis
525
+
526
+ {dataset_explanation}
527
+
528
+ ## Detected Columns
529
+ """
530
+ for purpose, col_name in column_mapping.items():
531
+ if col_name and purpose not in ["column_descriptions", "fraud_indicator_columns"]:
532
+ stats_summary += f"- **{purpose.replace('_column', '')}**: {col_name}\n"
533
+
534
+ if column_mapping.get("fraud_indicator_columns"):
535
+ stats_summary += "\n**Potential Fraud Indicator Columns**:\n"
536
+ for col in column_mapping.get("fraud_indicator_columns", []):
537
+ stats_summary += f"- {col}\n"
538
+
539
+ # Get AI analysis of suspicious transactions
540
+ ai_analysis = analyze_transaction_with_ai(df_with_anomalies, suspicious_transactions, column_mapping)
541
+
542
+ # Save suspicious transactions to a temporary file
543
+ temp_csv = tempfile.NamedTemporaryFile(delete=False, suffix='.csv')
544
+ suspicious_transactions.to_csv(temp_csv.name, index=False)
545
+ temp_csv.close()
546
+
547
+ # Return results and visualizations
548
+ return (
549
+ stats_summary,
550
+ ai_analysis,
551
+ temp_csv.name, # Return the path to the temporary file
552
+ visualizations.get('amount_distribution', None),
553
+ visualizations.get('time_series', None),
554
+ visualizations.get('fraud_score_dist', None)
555
+ )
556
+
557
+ except Exception as e:
558
+ import traceback
559
+ error_trace = traceback.format_exc()
560
+ return f"Error: {str(e)}\n\nTrace: {error_trace}", None, None, None, None, None
561
+
562
+ def create_gradio_interface():
563
+ """Create Gradio interface for the application"""
564
+ with gr.Blocks(title="AI Fraud Detection System") as app:
565
+ gr.Markdown("# AI Transaction Fraud & Anomaly Detection System")
566
+ gr.Markdown("Upload your transaction data (CSV or Excel) to detect potential fraud and anomalies. The system will use AI to analyze your dataset structure and identify relevant columns.")
567
+
568
+ with gr.Row():
569
+ file_input = gr.File(label="Upload Transaction Data", file_types=[".csv", ".xlsx", ".xls"])
570
+
571
+ with gr.Row():
572
+ submit_btn = gr.Button("Analyze Transactions", variant="primary")
573
+
574
+ with gr.Tabs():
575
+ with gr.TabItem("Summary"):
576
+ stats_output = gr.Markdown(label="Statistics Summary")
577
+ ai_analysis_output = gr.Markdown(label="AI Analysis")
578
+
579
+ with gr.TabItem("Visualizations"):
580
+ with gr.Row():
581
+ amount_dist_plot = gr.Plot(label="Transaction Amount Distribution")
582
+
583
+ with gr.Row():
584
+ time_series_plot = gr.Plot(label="Transactions Over Time")
585
+ fraud_score_plot = gr.Plot(label="Fraud Score Distribution")
586
+
587
+ with gr.TabItem("Suspicious Transactions"):
588
+ suspicious_csv = gr.File(label="Download Suspicious Transactions (CSV)")
589
+
590
+ submit_btn.click(
591
+ process_transactions,
592
+ inputs=[file_input],
593
+ outputs=[stats_output, ai_analysis_output, suspicious_csv,
594
+ amount_dist_plot, time_series_plot, fraud_score_plot]
595
+ )
596
+
597
+ return app
598
+
599
+ if __name__ == "__main__":
600
+ # Enable debug mode to get detailed error messages
601
+ import logging
602
+ logging.basicConfig(level=logging.DEBUG)
603
+
604
+ app = create_gradio_interface()
605
+ app.launch(share=True)