Source code

pestle.py

@@ -0,0 +1,437 @@
+"""
+PESTLE MODEL - COMPLETE USAGE GUIDE WITH MODEL PERSISTENCE
+==========================================================
+
+This script demonstrates:
+1. Training and saving the model
+2. Loading a saved model
+3. Making predictions with prompts
+4. Batch predictions
+"""
+
+import pandas as pd
+import numpy as np
+import pickle
+import json
+from pathlib import Path
+from datetime import datetime
+from scipy.sparse import hstack, csr_matrix
+import warnings
+warnings.filterwarnings('ignore')
+
+from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import LabelEncoder
+from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.naive_bayes import MultinomialNB
+from sklearn.metrics import accuracy_score, classification_report
+
+
+class PESTLEModel:
+    """Production-ready PESTLE classifier with save/load functionality"""
+    
+    def __init__(self):
+        self.model = None
+        self.vectorizers = {}
+        self.label_encoder = LabelEncoder()
+        self.best_model_name = None
+        self.pestle_keywords = {
+            'Political': ['government', 'election', 'policy', 'congress', 'senate', 
+                         'president', 'legislation', 'vote', 'parliament', 'diplomacy'],
+            'Economic': ['economy', 'market', 'stock', 'trade', 'gdp', 'inflation',
+                        'interest rate', 'unemployment', 'fed', 'revenue', 'profit'],
+            'Social': ['healthcare', 'education', 'social', 'community', 'demographic',
+                      'population', 'immigration', 'diversity', 'equality', 'housing'],
+            'Technological': ['technology', 'ai', 'artificial intelligence', 'innovation',
+                            'digital', 'cyber', 'data', 'software', 'internet', 'automation'],
+            'Legal': ['law', 'court', 'legal', 'lawsuit', 'judge', 'attorney',
+                     'regulation', 'compliance', 'contract', 'patent', 'trial'],
+            'Environmental': ['climate', 'environment', 'carbon', 'emission', 'pollution',
+                            'renewable', 'energy', 'sustainability', 'green', 'conservation']
+        }
+        self.metadata = {}
+    
+    def train(self, csv_path='pestle_news_samples_6000_rows.csv'):
+        """Train the model from scratch"""
+        print("="*80)
+        print("TRAINING PESTLE MODEL".center(80))
+        print("="*80)
+        
+        # Load data
+        print("\n1. Loading data...")
+        df = pd.read_csv(csv_path)
+        print(f"   ✅ Loaded {len(df)} records")
+        
+        # Prepare text features
+        print("\n2. Preparing features...")
+        df['text_features'] = (
+            df['Headline'].fillna('') + ' ' +
+            df['Description'].fillna('') + ' ' +
+            df['Topic_Tags'].fillna('').str.replace(',', ' ')
+        ).str.lower().str.replace(r'[^\w\s]', '', regex=True)
+        
+        # Create keyword features
+        keyword_features = []
+        for _, row in df.iterrows():
+            text = row['text_features']
+            features = []
+            for category, keywords in self.pestle_keywords.items():
+                score = sum(1 for kw in keywords if kw in text) / len(keywords)
+                features.append(score)
+            keyword_features.append(features)
+        
+        # TF-IDF vectorization
+        tfidf = TfidfVectorizer(
+            max_features=3000,
+            ngram_range=(1, 3),
+            stop_words='english',
+            min_df=2,
+            max_df=0.95
+        )
+        X_tfidf = tfidf.fit_transform(df['text_features'])
+        self.vectorizers['tfidf'] = tfidf
+        print(f"   ✅ TF-IDF features: {X_tfidf.shape}")
+        
+        # Combine features
+        X_combined = hstack([X_tfidf, csr_matrix(keyword_features)])
+        
+        # Encode labels
+        y = self.label_encoder.fit_transform(df['PESTLE_Category'])
+        
+        # Train-test split
+        print("\n3. Training models...")
+        X_train, X_test, y_train, y_test = train_test_split(
+            X_combined, y, test_size=0.2, random_state=42, stratify=y
+        )
+        
+        # Train multiple models
+        models = {
+            'Random Forest': RandomForestClassifier(
+                n_estimators=200, max_depth=30, random_state=42, n_jobs=-1
+            ),
+            'Gradient Boosting': GradientBoostingClassifier(
+                n_estimators=150, learning_rate=0.1, random_state=42
+            ),
+            'Logistic Regression': LogisticRegression(
+                max_iter=1000, C=1.0, class_weight='balanced', random_state=42
+            )
+        }
+        
+        best_score = 0
+        best_model = None
+        best_name = None
+        
+        for name, model in models.items():
+            model.fit(X_train, y_train)
+            y_pred = model.predict(X_test)
+            accuracy = accuracy_score(y_test, y_pred)
+            print(f"   {name}: {accuracy:.4f}")
+            
+            if accuracy > best_score:
+                best_score = accuracy
+                best_model = model
+                best_name = name
+        
+        self.model = best_model
+        self.best_model_name = best_name
+        
+        # Store metadata
+        self.metadata = {
+            'model_type': best_name,
+            'accuracy': best_score,
+            'trained_date': datetime.now().isoformat(),
+            'n_samples': len(df),
+            'categories': self.label_encoder.classes_.tolist()
+        }
+        
+        print(f"\n🏆 Best Model: {best_name} (Accuracy: {best_score:.4f})")
+        print("\n   Category Performance:")
+        report = classification_report(y_test, self.model.predict(X_test),
+                                      target_names=self.label_encoder.classes_,
+                                      output_dict=True)
+        for cat in self.label_encoder.classes_:
+            f1 = report[cat]['f1-score']
+            print(f"   - {cat}: F1={f1:.3f}")
+        
+        return True
+    
+    def save(self, model_name="pestle_model"):
+        """Save model to disk"""
+        print(f"\n{'='*80}")
+        print(f"SAVING MODEL: {model_name}".center(80))
+        print("="*80)
+        
+        model_dir = Path("pestle_models") / model_name
+        model_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Save model
+        with open(model_dir / "model.pkl", 'wb') as f:
+            pickle.dump(self.model, f)
+        print(f"✅ Model saved")
+        
+        # Save vectorizers
+        with open(model_dir / "vectorizers.pkl", 'wb') as f:
+            pickle.dump(self.vectorizers, f)
+        print(f"✅ Vectorizers saved")
+        
+        # Save label encoder
+        with open(model_dir / "label_encoder.pkl", 'wb') as f:
+            pickle.dump(self.label_encoder, f)
+        print(f"✅ Label encoder saved")
+        
+        # Save keywords
+        with open(model_dir / "keywords.pkl", 'wb') as f:
+            pickle.dump(self.pestle_keywords, f)
+        print(f"✅ Keywords saved")
+        
+        # Save metadata
+        with open(model_dir / "metadata.json", 'w') as f:
+            json.dump(self.metadata, f, indent=2)
+        print(f"✅ Metadata saved")
+        
+        print(f"\n📁 Model saved to: {model_dir.absolute()}")
+        return str(model_dir)
+    
+    def load(self, model_name="pestle_model"):
+        """Load model from disk"""
+        print(f"\n{'='*80}")
+        print(f"LOADING MODEL: {model_name}".center(80))
+        print("="*80)
+        
+        model_dir = Path("pestle_models") / model_name
+        
+        if not model_dir.exists():
+            raise FileNotFoundError(f"Model directory not found: {model_dir}")
+        
+        # Load components
+        with open(model_dir / "model.pkl", 'rb') as f:
+            self.model = pickle.load(f)
+        print("✅ Model loaded")
+        
+        with open(model_dir / "vectorizers.pkl", 'rb') as f:
+            self.vectorizers = pickle.load(f)
+        print("✅ Vectorizers loaded")
+        
+        with open(model_dir / "label_encoder.pkl", 'rb') as f:
+            self.label_encoder = pickle.load(f)
+        print("✅ Label encoder loaded")
+        
+        with open(model_dir / "keywords.pkl", 'rb') as f:
+            self.pestle_keywords = pickle.load(f)
+        print("✅ Keywords loaded")
+        
+        with open(model_dir / "metadata.json", 'r') as f:
+            self.metadata = json.load(f)
+        print("✅ Metadata loaded")
+        
+        print(f"\n📊 Model Info:")
+        print(f"   Type: {self.metadata.get('model_type', 'Unknown')}")
+        print(f"   Accuracy: {self.metadata.get('accuracy', 0):.4f}")
+        print(f"   Trained: {self.metadata.get('trained_date', 'Unknown')}")
+        print(f"   Categories: {', '.join(self.metadata.get('categories', []))}")
+        
+        return True
+    
+    def predict(self, text, show_probabilities=True):
+        """Predict PESTLE category for text"""
+        if self.model is None:
+            raise ValueError("Model not loaded. Call train() or load() first.")
+        
+        # Preprocess text
+        text_processed = text.lower()
+        text_processed = ''.join(c for c in text_processed if c.isalnum() or c.isspace())
+        
+        # Extract TF-IDF features
+        X_tfidf = self.vectorizers['tfidf'].transform([text_processed])
+        
+        # Extract keyword features
+        keyword_features = []
+        for category, keywords in self.pestle_keywords.items():
+            score = sum(1 for kw in keywords if kw in text_processed) / len(keywords)
+            keyword_features.append(score)
+        
+        # Combine features
+        X_combined = hstack([X_tfidf, csr_matrix([keyword_features])])
+        
+        # Predict
+        prediction = self.model.predict(X_combined)[0]
+        predicted_category = self.label_encoder.inverse_transform([prediction])[0]
+        
+        result = {'category': predicted_category}
+        
+        if show_probabilities and hasattr(self.model, 'predict_proba'):
+            probabilities = self.model.predict_proba(X_combined)[0]
+            prob_dict = {
+                cat: float(prob)
+                for cat, prob in zip(self.label_encoder.classes_, probabilities)
+            }
+            result['probabilities'] = prob_dict
+            result['confidence'] = float(max(probabilities))
+        
+        return result
+    
+    def predict_batch(self, texts):
+        """Predict categories for multiple texts"""
+        results = []
+        for text in texts:
+            results.append(self.predict(text, show_probabilities=True))
+        return results
+
+
+# =============================================================================
+# USAGE EXAMPLES
+# =============================================================================
+
+def example_1_train_and_save():
+    """Example 1: Train a new model and save it"""
+    print("\n" + "="*80)
+    print("EXAMPLE 1: TRAIN AND SAVE MODEL".center(80))
+    print("="*80)
+    
+    model = PESTLEModel()
+    model.train('pestle_news_samples_6000_rows.csv')
+    model.save("pestle_model")
+    
+    print("\n✅ Model trained and saved successfully!")
+
+
+def example_2_load_and_predict():
+    """Example 2: Load saved model and make predictions"""
+    print("\n" + "="*80)
+    print("EXAMPLE 2: LOAD MODEL AND PREDICT".center(80))
+    print("="*80)
+    
+    # Load model
+    model = PESTLEModel()
+    model.load("pestle_model")
+    
+    # Test prompts
+    test_prompts = [
+        "Congress passes new healthcare reform bill",
+        "Stock market reaches all-time high amid economic growth",
+        "New AI technology revolutionizes manufacturing",
+        "Supreme Court ruling on environmental regulations",
+        "Rising sea levels threaten coastal communities",
+        "Social media platforms face data privacy concerns"
+    ]
+    
+    print("\n" + "="*80)
+    print("PREDICTIONS".center(80))
+    print("="*80)
+    
+    for i, prompt in enumerate(test_prompts, 1):
+        result = model.predict(prompt)
+        print(f"\n{i}. Text: {prompt}")
+        print(f"   Category: {result['category']}")
+        print(f"   Confidence: {result['confidence']:.2%}")
+        print(f"   Top 3 Probabilities:")
+        sorted_probs = sorted(result['probabilities'].items(), 
+                            key=lambda x: x[1], reverse=True)[:3]
+        for cat, prob in sorted_probs:
+            print(f"      - {cat}: {prob:.2%}")
+
+
+def example_3_interactive_mode():
+    """Example 3: Interactive prediction mode"""
+    print("\n" + "="*80)
+    print("EXAMPLE 3: INTERACTIVE MODE".center(80))
+    print("="*80)
+    
+    model = PESTLEModel()
+    
+    # Try to load existing model, otherwise train new one
+    try:
+        model.load("pestle_model")
+    except FileNotFoundError:
+        print("\n⚠️  No saved model found. Training new model...")
+        model.train('pestle_news_samples_6000_rows.csv')
+        model.save("pestle_model")
+    
+    print("\n" + "="*80)
+    print("Enter text to classify (or 'quit' to exit)".center(80))
+    print("="*80)
+    
+    while True:
+        text = input("\n📝 Enter text: ").strip()
+        
+        if text.lower() in ['quit', 'exit', 'q']:
+            print("\n👋 Goodbye!")
+            break
+        
+        if not text:
+            print("⚠️  Please enter some text")
+            continue
+        
+        result = model.predict(text)
+        print(f"\n🎯 Predicted Category: {result['category']}")
+        print(f"📊 Confidence: {result['confidence']:.2%}")
+
+
+def example_4_batch_prediction():
+    """Example 4: Batch prediction with export"""
+    print("\n" + "="*80)
+    print("EXAMPLE 4: BATCH PREDICTION".center(80))
+    print("="*80)
+    
+    model = PESTLEModel()
+    model.load("pestle_model")
+    
+    # Sample batch data
+    batch_texts = [
+        "Federal Reserve raises interest rates",
+        "Climate change summit reaches agreement",
+        "Tech giant faces antitrust lawsuit",
+        "New immigration policy announced",
+        "Breakthrough in quantum computing",
+        "Healthcare costs continue to rise"
+    ]
+    
+    print(f"\nProcessing {len(batch_texts)} texts...")
+    results = model.predict_batch(batch_texts)
+    
+    # Create DataFrame
+    df_results = pd.DataFrame({
+        'Text': batch_texts,
+        'Category': [r['category'] for r in results],
+        'Confidence': [r['confidence'] for r in results]
+    })
+    
+    print("\n" + "="*80)
+    print("BATCH RESULTS".center(80))
+    print("="*80)
+    print(df_results.to_string(index=False))
+    
+    # Save to CSV
+    output_file = "batch_predictions.csv"
+    df_results.to_csv(output_file, index=False)
+    print(f"\n✅ Results saved to: {output_file}")
+
+
+# =============================================================================
+# MAIN EXECUTION
+# =============================================================================
+
+if __name__ == "__main__":
+    print("\n" + "="*80)
+    print("PESTLE MODEL - USAGE GUIDE".center(80))
+    print("="*80)
+    print("\nChoose an example to run:")
+    print("1. Train and save a new model")
+    print("2. Load model and make predictions")
+    print("3. Interactive prediction mode")
+    print("4. Batch prediction with export")
+    
+    choice = input("\nEnter choice (1-4): ").strip()
+    
+    if choice == '1':
+        example_1_train_and_save()
+    elif choice == '2':
+        example_2_load_and_predict()
+    elif choice == '3':
+        example_3_interactive_mode()
+    elif choice == '4':
+        example_4_batch_prediction()
+    else:
+        print("Invalid choice. Running example 1...")
+        example_1_train_and_save()