Update app.py

app.py

@@ -1,5 +1,6 @@
 # pip install seaborn mlxtend
 # app.py
+# app.py
 import os
 import io
 import tempfile
@@ -36,8 +37,6 @@ from sklearn.exceptions import NotFittedError
 import scipy.stats as stats
 import matplotlib
 matplotlib.use("Agg")  # for headless plotting
-from mlxtend.frequent_patterns import apriori, association_rules, fpgrowth
-from mlxtend.preprocessing import TransactionEncoder
 import matplotlib.pyplot as plt
 
 # --- lightweight password hashing (stdlib) ---
@@ -992,32 +991,229 @@ def run_clustering(clustering_method: str, X, n_clusters=3, eps=0.5, min_samples
 # -----------------------------
 # Rule-Based (Apriori) Functions
 # -----------------------------
-def prepare_transaction_data(df, transaction_col=None):
-    if transaction_col and transaction_col in df.columns:
-        # If transaction column is specified
+def prepare_transaction_data(df, transaction_col=None, threshold=0.5):
+    """
+    Prepare data for association rule mining.
+    For categorical columns, create binary indicators.
+    For numeric columns, bin them into categories.
+    """
+    df_prep = df.copy()
+    
+    # Convert numeric columns to categorical by binning
+    for col in df_prep.select_dtypes(include=[np.number]).columns:
+        if len(df_prep[col].unique()) > 10:  # Only bin if many unique values
+            # Create 5 equal-width bins
+            df_prep[col] = pd.qcut(df_prep[col], q=5, duplicates='drop')
+            df_prep[col] = df_prep[col].astype(str)
+    
+    # Convert all to string for consistency
+    for col in df_prep.columns:
+        df_prep[col] = df_prep[col].astype(str)
+    
+    # If transaction column is specified, use it
+    if transaction_col and transaction_col in df_prep.columns:
         transactions = []
-        for items in df[transaction_col]:
-            if isinstance(items, str):
-                transactions.append([item.strip() for item in items.split(',')])
+        for items in df_prep[transaction_col]:
+            if pd.isna(items):
+                transactions.append([])
             else:
-                transactions.append([str(items)])
+                # Split by common delimiters
+                if isinstance(items, str):
+                    split_items = [item.strip() for item in str(items).replace(';', ',').split(',')]
+                    transactions.append([f"{transaction_col}={item}" for item in split_items if item])
+                else:
+                    transactions.append([f"{transaction_col}={items}"])
     else:
-        # Convert entire dataframe to transactions (binary/one-hot expected)
+        # Create transactions from entire dataframe (one-hot like)
         transactions = []
-        for _, row in df.iterrows():
+        for _, row in df_prep.iterrows():
             transaction = []
-            for col in df.columns:
-                if row[col] == 1 or row[col] == True:
-                    transaction.append(col)
+            for col in df_prep.columns:
+                if col == transaction_col:
+                    continue
+                value = str(row[col])
+                if value and value.lower() not in ['nan', 'null', 'none', '']:
+                    transaction.append(f"{col}={value}")
             if transaction:
                 transactions.append(transaction)
     
-    # Encode transactions
-    te = TransactionEncoder()
-    te_ary = te.fit(transactions).transform(transactions)
-    df_encoded = pd.DataFrame(te_ary, columns=te.columns_)
+    return transactions
+
+
+def simple_apriori(transactions, min_support=0.1):
+    """
+    A simplified Apriori algorithm implementation.
+    Returns frequent itemsets and their support.
+    """
+    from collections import defaultdict
+    import itertools
+    
+    # Count item frequencies
+    item_counts = defaultdict(int)
+    for transaction in transactions:
+        for item in set(transaction):  # Use set to avoid duplicate items in same transaction
+            item_counts[item] += 1
+    
+    total_transactions = len(transactions)
+    
+    # Get frequent 1-itemsets
+    frequent_1_itemsets = {}
+    for item, count in item_counts.items():
+        support = count / total_transactions
+        if support >= min_support:
+            frequent_1_itemsets[frozenset([item])] = support
+    
+    frequent_itemsets = {1: frequent_1_itemsets}
+    
+    # Generate larger itemsets (up to 3-itemsets for simplicity)
+    k = 2
+    while True:
+        candidate_itemsets = set()
+        prev_itemsets = list(frequent_itemsets[k-1].keys())
+        
+        # Generate candidates
+        for i in range(len(prev_itemsets)):
+            for j in range(i+1, len(prev_itemsets)):
+                itemset1 = prev_itemsets[i]
+                itemset2 = prev_itemsets[j]
+                union_set = itemset1.union(itemset2)
+                if len(union_set) == k:
+                    candidate_itemsets.add(union_set)
+        
+        if not candidate_itemsets:
+            break
+        
+        # Count support for candidates
+        candidate_counts = defaultdict(int)
+        for transaction in transactions:
+            trans_set = set(transaction)
+            for candidate in candidate_itemsets:
+                if candidate.issubset(trans_set):
+                    candidate_counts[candidate] += 1
+        
+        # Filter by minimum support
+        frequent_k_itemsets = {}
+        for itemset, count in candidate_counts.items():
+            support = count / total_transactions
+            if support >= min_support:
+                frequent_k_itemsets[itemset] = support
+        
+        if not frequent_k_itemsets:
+            break
+        
+        frequent_itemsets[k] = frequent_k_itemsets
+        k += 1
+        if k > 3:  # Limit to 3-itemsets for performance
+            break
+    
+    return frequent_itemsets
+
+def generate_association_rules(frequent_itemsets, min_confidence=0.5, min_lift=1.0):
+    """Generate association rules from frequent itemsets."""
+    rules = []
+    
+    for k, itemsets in frequent_itemsets.items():
+        if k < 2:
+            continue
+        
+        for itemset, support_itemset in itemsets.items():
+            itemset_list = list(itemset)
+            
+            # Generate all non-empty proper subsets
+            for i in range(1, k):
+                for antecedent in itertools.combinations(itemset_list, i):
+                    antecedent_set = frozenset(antecedent)
+                    consequent_set = itemset - antecedent_set
+                    
+                    if not consequent_set:
+                        continue
+                    
+                    # Find support of antecedent
+                    antecedent_support = None
+                    for size in range(1, k):
+                        if size in frequent_itemsets and antecedent_set in frequent_itemsets[size]:
+                            antecedent_support = frequent_itemsets[size][antecedent_set]
+                            break
+                    
+                    if antecedent_support is None or antecedent_support == 0:
+                        continue
+                    
+                    # Calculate confidence and lift
+                    confidence = support_itemset / antecedent_support
+                    consequent_support = None
+                    
+                    # Find support of consequent
+                    for size in range(1, k):
+                        if size in frequent_itemsets and consequent_set in frequent_itemsets[size]:
+                            consequent_support = frequent_itemsets[size][consequent_set]
+                            break
+                    
+                    if consequent_support is None or consequent_support == 0:
+                        continue
+                    
+                    lift = confidence / consequent_support
+                    
+                    if confidence >= min_confidence and lift >= min_lift:
+                        rules.append({
+                            'antecedents': set(antecedent),
+                            'consequents': set(consequent_set),
+                            'support': support_itemset,
+                            'confidence': confidence,
+                            'lift': lift
+                        })
+    
+    # Sort by confidence, then lift
+    rules.sort(key=lambda x: (x['confidence'], x['lift']), reverse=True)
+    return rules
+
+def run_association_mining(transactions, min_support=0.1, min_confidence=0.5, min_lift=1.0):
+    """Run association rule mining using our simple implementation."""
+    frequent_itemsets = simple_apriori(transactions, min_support)
+    rules = generate_association_rules(frequent_itemsets, min_confidence, min_lift)
+    return frequent_itemsets, rules
+
+def run_frequency_based_rules(df, categorical_cols=None, min_frequency=0.1, min_cooccurrence=0.5):
+    """
+    An alternative rule discovery method based on frequency and co-occurrence.
+    Simpler but effective for many use cases.
+    """
+    if categorical_cols is None:
+        categorical_cols = df.select_dtypes(include=['object', 'category']).columns.tolist()
+    
+    rules = []
+    
+    for col1 in categorical_cols:
+        for col2 in categorical_cols:
+            if col1 == col2:
+                continue
+            
+            # Calculate contingency table
+            contingency = pd.crosstab(df[col1], df[col2], normalize='all')
+            
+            # Find strong associations
+            for val1 in contingency.index:
+                for val2 in contingency.columns:
+                    p_val1_val2 = contingency.loc[val1, val2]
+                    p_val1 = df[col1].value_counts(normalize=True).get(val1, 0)
+                    p_val2 = df[col2].value_counts(normalize=True).get(val2, 0)
+                    
+                    if p_val1 > 0 and p_val2 > 0 and p_val1_val2 > 0:
+                        confidence = p_val1_val2 / p_val1
+                        lift = p_val1_val2 / (p_val1 * p_val2)
+                        
+                        if p_val1_val2 >= min_frequency and confidence >= min_cooccurrence:
+                            rules.append({
+                                'rule': f"If {col1} = {val1} then {col2} = {val2}",
+                                'support': p_val1_val2,
+                                'confidence': confidence,
+                                'lift': lift,
+                                'antecedent': f"{col1}={val1}",
+                                'consequent': f"{col2}={val2}"
+                            })
     
-    return df_encoded, transactions
+    # Sort by confidence and lift
+    rules.sort(key=lambda x: (x['confidence'], x['lift']), reverse=True)
+    return rules
 
 def run_apriori(df_encoded, min_support=0.1, min_confidence=0.5, min_lift=1.0, max_length=4):
     # Find frequent itemsets
@@ -1469,7 +1665,7 @@ with gr.Blocks(css=css, title="DataSynth — Analytics Hub") as demo:
                     with gr.Column(scale=1):
                         gr.Markdown("### 🎯 Select Task Type")
                         task_select = gr.Radio(
-                            choices=["Regression", "Classification", "Clustering", "Rule-Based"],
+                            choices=["Regression", "Classification", "Clustering", "Rule-Based (Association)"],
                             value="Regression",
                             label="ML Task Type"
                         )
@@ -1546,14 +1742,14 @@ with gr.Blocks(css=css, title="DataSynth — Analytics Hub") as demo:
                         with gr.Column(visible=False) as rule_based_params:
                             gr.Markdown("### 🔗 Rule-Based Mining Settings")
                             rule_method = gr.Radio(
-                                choices=["Apriori", "FP-Growth"],
-                                value="Apriori",
-                                label="Association Rule Algorithm"
+                                choices=["Apriori (Simple)", "Frequency-Based Rules"],
+                                value="Apriori (Simple)",
+                                label="Rule Discovery Method"
                             )
                             
                             transaction_col = gr.Dropdown(
                                 choices=[],
-                                label="Transaction Column (optional)",
+                                label="Transaction Column (optional - for Apriori)",
                                 interactive=True
                             )
                             
@@ -1581,13 +1777,6 @@ with gr.Blocks(css=css, title="DataSynth — Analytics Hub") as demo:
                                     step=0.1,
                                     label="Minimum Lift"
                                 )
-                                max_rule_length = gr.Slider(
-                                    minimum=2,
-                                    maximum=10,
-                                    value=4,
-                                    step=1,
-                                    label="Maximum Rule Length"
-                                )
                     
                     with gr.Column(scale=2):
                         gr.Markdown("### 🚀 Model Training & Evaluation")
@@ -1999,64 +2188,121 @@ with gr.Blocks(css=css, title="DataSynth — Analytics Hub") as demo:
                 detailed_df = cluster_stats
                 
             else:  # Rule-Based
-                # Prepare transaction data
-                df_encoded, transactions = prepare_transaction_data(df, trans_col if trans_col else None)
-                
-                if rule_algo == "Apriori":
-                    frequent_itemsets, rules = run_apriori(
-                        df_encoded, 
-                        min_support=min_sup,
-                        min_confidence=min_conf,
-                        min_lift=min_lft,
-                        max_length=max_len
-                    )
-                else:  # FP-Growth
-                    frequent_itemsets, rules = run_fp_growth(
-                        df_encoded,
-                        min_support=min_sup,
-                        min_confidence=min_conf,
-                        min_lift=min_lft,
-                        max_length=max_len
-                    )
-                
-                if rules is None or rules.empty:
-                    return "No association rules found with the given parameters.", None, None, None
-                
-                results_summary = f"## 🔗 {rule_algo} Association Rules\n\n"
-                results_summary += f"- **Total rules found:** {len(rules)}\n"
-                results_summary += f"- **Minimum support:** {min_sup}\n"
-                results_summary += f"- **Minimum confidence:** {min_conf}\n"
-                results_summary += f"- **Minimum lift:** {min_lft}\n\n"
-                
-                # Top 10 rules by confidence
-                top_rules = rules.head(10)
-                results_summary += "### 🏆 Top 10 Rules by Confidence\n\n"
-                for idx, rule in top_rules.iterrows():
-                    antecedents = ', '.join(list(rule['antecedents']))
-                    consequents = ', '.join(list(rule['consequents']))
-                    results_summary += f"{idx+1}. **IF** {antecedents} **THEN** {consequents}  \n"
-                    results_summary += f"   Support: {rule['support']:.3f}, Confidence: {rule['confidence']:.3f}, Lift: {rule['lift']:.3f}\n\n"
-                
-                # Rules visualization
-                if not rules.empty:
-                    plot_rules = rules.head(20).copy()
-                    plot_rules['rule'] = plot_rules.apply(
-                        lambda x: f"{', '.join(list(x['antecedents']))} → {', '.join(list(x['consequents']))}", axis=1)
-                    
-                    fig = px.scatter(plot_rules, x='support', y='confidence',
-                                   size='lift', color='lift',
-                                   hover_name='rule',
-                                   title=f'{rule_algo} Association Rules',
-                                   labels={'support': 'Support', 'confidence': 'Confidence'})
-                    plot_fig = fig
-                    
-                    detailed_df = rules[['antecedents', 'consequents', 'support', 'confidence', 'lift']].head(20)
-            
-            return results_summary, plot_fig, detailed_df
-            
-        except Exception as e:
-            traceback.print_exc()
-            return f"Error in model training: {str(e)}", None, None
+                    try:
+                        if rule_algo == "Apriori (Simple)":
+                            # Prepare transaction data
+                            transactions = prepare_transaction_data(df, trans_col if trans_col else None)
+                            
+                            if not transactions:
+                                return "No valid transactions found in the data.", None, None
+                            
+                            # Run Apriori
+                            frequent_itemsets, rules = run_association_mining(
+                                transactions, 
+                                min_support=min_sup,
+                                min_confidence=min_conf,
+                                min_lift=min_lft
+                            )
+                            
+                            results_summary = f"## 🔗 Simple Apriori Association Rules\n\n"
+                            
+                        else:  # Frequency-Based Rules
+                            # Identify categorical columns
+                            cat_cols = df.select_dtypes(include=['object', 'category']).columns.tolist()
+                            if not cat_cols:
+                                # Try to use all columns as categorical
+                                cat_cols = df.columns.tolist()
+                            
+                            rules_list = run_frequency_based_rules(
+                                df, 
+                                categorical_cols=cat_cols,
+                                min_frequency=min_sup,
+                                min_cooccurrence=min_conf
+                            )
+                            
+                            # Convert to rules format similar to Apriori
+                            rules = []
+                            for rule in rules_list:
+                                rules.append({
+                                    'antecedents': {rule['antecedent']},
+                                    'consequents': {rule['consequent']},
+                                    'support': rule['support'],
+                                    'confidence': rule['confidence'],
+                                    'lift': rule['lift'],
+                                    'rule_str': rule['rule']
+                                })
+                            
+                            results_summary = f"## 🔗 Frequency-Based Association Rules\n\n"
+                        
+                        if not rules:
+                            return "No association rules found with the given parameters.", None, None
+                        
+                        results_summary += f"- **Total rules found:** {len(rules)}\n"
+                        results_summary += f"- **Minimum support:** {min_sup}\n"
+                        results_summary += f"- **Minimum confidence:** {min_conf}\n"
+                        results_summary += f"- **Minimum lift:** {min_lft}\n\n"
+                        
+                        # Top 10 rules by confidence
+                        top_rules = rules[:10]
+                        results_summary += "### 🏆 Top 10 Rules by Confidence\n\n"
+                        for idx, rule in enumerate(top_rules):
+                            if 'rule_str' in rule:
+                                rule_text = rule['rule_str']
+                            else:
+                                antecedents = ', '.join(list(rule['antecedents']))
+                                consequents = ', '.join(list(rule['consequents']))
+                                rule_text = f"IF {antecedents} THEN {consequents}"
+                            
+                            results_summary += f"{idx+1}. **{rule_text}**  \n"
+                            results_summary += f"   Support: {rule['support']:.3f}, Confidence: {rule['confidence']:.3f}, Lift: {rule['lift']:.3f}\n\n"
+                        
+                        # Prepare rules for visualization
+                        plot_data = []
+                        for rule in rules[:20]:  # Limit to 20 for plotting
+                            if 'rule_str' in rule:
+                                rule_name = rule['rule_str'][:50] + "..." if len(rule['rule_str']) > 50 else rule['rule_str']
+                            else:
+                                antecedents = ', '.join(list(rule['antecedents']))[:20]
+                                consequents = ', '.join(list(rule['consequents']))[:20]
+                                rule_name = f"{antecedents}→{consequents}"
+                            
+                            plot_data.append({
+                                'rule': rule_name,
+                                'support': rule['support'],
+                                'confidence': rule['confidence'],
+                                'lift': rule['lift']
+                            })
+                        
+                        if plot_data:
+                            plot_df = pd.DataFrame(plot_data)
+                            fig = px.scatter(plot_df, x='support', y='confidence',
+                                           size='lift', color='lift',
+                                           hover_name='rule',
+                                           title=f'{rule_algo} Association Rules',
+                                           labels={'support': 'Support', 'confidence': 'Confidence'})
+                            plot_fig = fig
+                        
+                        # Prepare detailed results
+                        detailed_data = []
+                        for rule in rules[:20]:
+                            if 'rule_str' in rule:
+                                rule_str = rule['rule_str']
+                            else:
+                                antecedents = ', '.join(list(rule['antecedents']))
+                                consequents = ', '.join(list(rule['consequents']))
+                                rule_str = f"{antecedents} → {consequents}"
+                            
+                            detailed_data.append({
+                                'Rule': rule_str,
+                                'Support': f"{rule['support']:.3f}",
+                                'Confidence': f"{rule['confidence']:.3f}",
+                                'Lift': f"{rule['lift']:.3f}"
+                            })
+                        
+                        detailed_df = pd.DataFrame(detailed_data)
+                        
+                    except Exception as e:
+                        return f"Error in rule-based mining: {str(e)}", None, None
 
     train_btn.click(
         fn=train_model_wrapper,