Update app.py

app.py

@@ -1,7 +1,886 @@
-import gradio as gr
+   import gradio as gr
+   import pandas as pd
+   import os
+   import subprocess
+   import sys
+   
+   # Install spaCy model
+   os.system("python -m spacy download en_core_web_sm")
+   
+   def process_tweets(files, reset_processing=False):
+       # Save uploaded files
+       file_paths = []
+       for file in files:
+           if file.name.endswith('.csv'):
+               # Ensure directory exists
+               os.makedirs("projects_twitter_post", exist_ok=True)
+               
+               # Save file to the directory
+               dest_path = f"projects_twitter_post/{os.path.basename(file.name)}"
+               os.system(f"cp {file.name} {dest_path}")
+               file_paths.append(dest_path)
+       
+       if not file_paths:
+           return "No CSV files uploaded. Please upload CSV files containing tweet data."
+       
+       # Run the processing script
+       reset_flag = "--reset" if reset_processing else ""
+       result = subprocess.run(
+           f"python process_tweet_huggingface.py {reset_flag}",
+           shell=True,
+           capture_output=True,
+           text=True
+       )
+       
+       # Check if output files were created
+       output_files = []
+       for file_path in file_paths:
+           base_name = os.path.basename(file_path).replace('.csv', '')
+           processed_path = f"projects_twitter_post/{base_name}_processed.csv"
+           analysis_path = f"projects_twitter_post/{base_name}_analysis.csv"
+           
+           if os.path.exists(processed_path):
+               output_files.append(processed_path)
+           if os.path.exists(analysis_path):
+               output_files.append(analysis_path)
+       
+       return_files = [f for f in output_files if os.path.exists(f)]
+       
+       log_output = result.stdout + "\n" + result.stderr
+       
+       return log_output, return_files
+   
+   with gr.Blocks() as demo:
+       gr.Markdown("# Crypto Tweet Processor")
+       gr.Markdown("Upload CSV files containing tweet data to process")
+       
+       with gr.Row():
+           files_input = gr.File(file_count="multiple", label="Upload CSV Files")
+           reset_checkbox = gr.Checkbox(label="Reset Processing", value=False)
+       
+       process_btn = gr.Button("Process Tweets")
+       
+       output_text = gr.Textbox(label="Processing Log")
+       output_files = gr.File(label="Processed Files", file_count="multiple")
+       
+       process_btn.click(
+           process_tweets,
+           inputs=[files_input, reset_checkbox],
+           outputs=[output_text, output_files]
+       )
+   
+   # Add the modified processing script code here
+   with open("process_tweet_huggingface.py", "w") as f:
+       f.write(#!/usr/bin/env python3
+"""
+Tweet Processing Script for Google Colab - Enhanced with NLP and Sentiment Analysis
+This version is optimized for Google Colab with GPU acceleration and Google Drive integration.
+"""
 
-def greet(name):
-    return "Hello " + name + "!!"
+import os
+import re
+import json
+import pandas as pd
+import numpy as np
+import torch
+import math
+import gc
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForMaskedLM, pipeline
+import spacy
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()
+# ==============================================
+# COLAB SETUP - Run these cells first in Colab
+# ==============================================
+
+# Uncomment and run this cell to mount your Google Drive
+"""
+from google.colab import drive
+drive.mount('/content/drive')
+"""
+
+# Uncomment and run this cell to install required packages
+"""
+!pip install pandas tqdm transformers spacy
+!python -m spacy download en_core_web_sm
+"""
+
+# Uncomment and run this cell to verify GPU availability
+"""
+import torch
+print(f"GPU available: {torch.cuda.is_available()}")
+print(f"GPU device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'None'}")
+"""
+
+# ==============================================
+# Constants - Update these paths for your setup
+# ==============================================
+
+# Update this to your Google Drive path
+DRIVE_PATH = "./projects_twitter_post"
+OUTPUT_FOLDER = f"{DRIVE_PATH}"
+CHECKPOINT_FILE = f"{OUTPUT_FOLDER}/processing_checkpoint.json"
+BATCH_SIZE = 500  # Reduced batch size for GPU memory management
+
+# Create output folder if it doesn't exist
+if not os.path.exists(OUTPUT_FOLDER):
+    os.makedirs(OUTPUT_FOLDER)
+
+# ==============================================
+# Model Initialization with GPU Acceleration
+# ==============================================
+
+print("Loading RoBERTa model...")
+model_name = "roberta-base"
+device = "cuda" if torch.cuda.is_available() else "cpu"
+print(f"Using device: {device}")
+
+# Initialize with GPU acceleration
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForMaskedLM.from_pretrained(model_name).to(device)
+nlp_pipeline = pipeline("fill-mask", model=model_name, device=0 if torch.cuda.is_available() else -1)
+
+# Initialize sentiment analysis pipeline
+print("Loading sentiment analysis model...")
+try:
+    # Using a Twitter-specific sentiment model for better results on social media text
+    sentiment_model = "cardiffnlp/twitter-roberta-base-sentiment"
+    sentiment_pipeline = pipeline("sentiment-analysis", model=sentiment_model, device=0 if torch.cuda.is_available() else -1)
+    SENTIMENT_AVAILABLE = True
+except Exception as e:
+    print(f"Error loading sentiment model: {e}")
+    # Fallback to a simpler sentiment model if the Twitter-specific one fails
+    try:
+        sentiment_pipeline = pipeline("sentiment-analysis", device=0 if torch.cuda.is_available() else -1)
+        SENTIMENT_AVAILABLE = True
+    except:
+        print("Sentiment analysis not available. Continuing without sentiment analysis.")
+        SENTIMENT_AVAILABLE = False
+
+# Try to load spaCy for basic text preprocessing
+try:
+    import spacy
+    spacy_nlp = spacy.load("en_core_web_sm")
+    SPACY_AVAILABLE = True
+except:
+    SPACY_AVAILABLE = False
+    print("SpaCy not available. Using basic text processing instead.")
+
+# Crypto-specific keywords with hierarchical categories
+CRYPTO_TAXONOMY = {
+    "COIN": {
+        "MAJOR": [
+            "bitcoin", "ethereum", "btc", "eth", "bnb", "xrp", "sol", "doge",
+            "cardano", "polkadot", "dot", "avalanche", "avax", "solana", "polygon", "matic"
+        ],
+        "STABLECOIN": [
+            "tether", "usdt", "usdc", "busd", "dai", "frax", "tusd", "usdd", "lusd", "gusd", "husd"
+        ],
+        "ALTCOIN": [
+            "litecoin", "ltc", "chainlink", "link", "stellar", "xlm", "dogecoin", "shib",
+            "tron", "trx", "cosmos", "atom", "near", "algo", "fantom", "ftm", "monero", "xmr"
+        ],
+        "DEFI": [
+            "uniswap", "uni", "aave", "sushi", "cake", "comp", "maker", "mkr", "curve", "crv",
+            "yearn", "yfi", "compound", "balancer", "bal", "synthetix", "snx"
+        ],
+        "UTILITY": [
+            "filecoin", "fil", "the graph", "grt", "arweave", "ar", "chainlink", "link", 
+            "helium", "hnt", "theta", "icp"
+        ],
+        "NFT": [
+            "enjin", "enj", "decentraland", "mana", "sandbox", "sand", "axie", "axs",
+            "gala", "apecoin", "ape", "flow", "ens", "stepn", "gmt"
+        ]
+    },
+    
+    "TECH": {
+        "CONCEPTS": [
+            "blockchain", "defi", "nft", "dao", "smart contract", "web3", "dapp", "protocol",
+            "consensus", "tokenomics", "tokenization"
+        ],
+        "CHAIN_TYPES": [
+            "layer1", "layer2", "rollup", "sidechain", "mainnet", "testnet", "devnet",
+            "pow", "pos", "poh", "pbft", "dpos"
+        ],
+        "PRIVACY": [
+            "zk", "zk-rollups", "zero-knowledge", "zkp", "zksnark", "zkstark", "mpc",
+            "privacy", "private", "anonymous", "confidential", "encrypted"
+        ],
+        "SECTORS": [
+            "defi", "cefi", "gamefi", "metaverse", "socialfi", "fintech", "realfi",
+            "play-to-earn", "move-to-earn", "learn-to-earn", "x-to-earn", "defai", "depin", "desci",
+            "refi", "did", "dedata", "dedao", "deid", "deai", "degov", "decloud", "dehealth", 
+            "decex", "deinsurance", "deworkplace", "public goods", "zk", "ordinals", "soulbound",
+            "onchain gaming", "ai agents", "infrastructure", "credentials", "restaking", "modular blockchain",
+            "liquid staking", "real world assets", "rwa", "synthetic assets", "account abstraction"
+        ]
+    },
+    
+    "ACTION": {
+        "TRADING": [
+            "buy", "sell", "long", "short", "margin", "leverage", "trade", "swap",
+            "arbitrage", "dca", "ape", "pump", "dump", "moon", "ath", "atl", "breakout",
+            "correction", "consolidation", "accumulate", "distribute", "front run", "front runner", 
+            "front running", "mev", "sandwich attack"
+        ],
+        "DEFI": [
+            "stake", "yield", "farm", "lend", "borrow", "supply", "withdraw", "claim",
+            "harvest", "flash loan", "liquidate", "collateralize", "wrap", "unwrap", "bridge",
+            "provide liquidity", "withdraw liquidity", "impermanent loss"
+        ],
+        "GOVERNANCE": [
+            "delegate", "vote", "propose", "governance", "dao", "snapshot", "quorum",
+            "execution", "timelock", "veto"
+        ],
+        "NFT": [
+            "mint", "airdrop", "whitelist", "burn", "floor price", "rarity", "trait", "pfp",
+            "collection", "secondary", "flip"
+        ],
+        "DEVELOPMENT": [
+            "deploy", "audit", "fork", "bootstrap", "initiate", "merge", "split",
+            "rebase", "optimize", "gas optimization", "implement", "compile"
+        ]
+    },
+    
+    "PLATFORM": {
+        "EXCHANGE": [
+            "coinbase", "binance", "kraken", "kucoin", "ftx", "okx", "bybit", "bitfinex", 
+            "huobi", "gate", "gemini", "bitstamp", "bittrex", "crypto.com", "cex", "dex"
+        ],
+        "WALLET": [
+            "metamask", "phantom", "trust wallet", "ledger", "trezor", "argent", "rainbow",
+            "wallet", "hot wallet", "cold storage", "hardware wallet", "seed phrase"
+        ],
+        "NFT_MARKET": [
+            "opensea", "rarible", "foundation", "superrare", "looksrare", "blur", "magic eden",
+            "nifty gateway", "zora", "x2y2", "element"
+        ],
+        "INFRA": [
+            "alchemy", "infura", "moralis", "quicknode", "ceramic", "arweave", "ipfs",
+            "node", "rpc", "api", "indexer", "subgraph"
+        ]
+    },
+    
+    "NETWORK": {
+        "LAYER1": [
+            "ethereum", "bitcoin", "solana", "avalanche", "polygon", "bnb chain", "bsc",
+            "cardano", "polkadot", "cosmos", "algorand", "tezos", "flow", "near", "tron"
+        ],
+        "LAYER2": [
+            "arbitrum", "optimism", "zksync", "starknet", "base", "polygon", "loopring",
+            "immutablex", "metis", "boba", "aztec", "validium", "zkevm"
+        ],
+        "INTEROPERABILITY": [
+            "cosmos", "polkadot", "kusama", "moonbeam", "moonriver", "parachains", "relay chain",
+            "ibc", "cross-chain", "bridge"
+        ]
+    },
+    
+    "EVENTS": {
+        "MARKET": [
+            "bull market", "bear market", "bull run", "bear trap", "bull trap", "halving",
+            "capitulation", "golden cross", "death cross", "breakout", "resistance", "support"
+        ],
+        "SECURITY": [
+            "hack", "exploit", "vulnerability", "scam", "phishing", "rug pull", "honeypot",
+            "flash crash", "attack", "51% attack", "front running", "sandwich attack", "mev extraction"
+        ],
+        "TOKEN_EVENTS": [
+            "airdrop", "token unlock", "vesting", "ico", "ido", "ito", "ieo", "fair launch",
+            "private sale", "seed round", "listing", "delisting"
+        ]
+    },
+    
+    "METRICS": {
+        "FINANCIAL": [
+            "apy", "apr", "roi", "tvl", "market cap", "mcap", "volume", "liquidity", "supply",
+            "circulating supply", "total supply", "max supply", "inflation", "deflation",
+            "volatility", "dominance"
+        ],
+        "TECHNICAL": [
+            "gas fee", "gas price", "gas limit", "slippage", "impermanent loss", "yield",
+            "hashrate", "difficulty", "tps", "latency", "finality", "block time", "block size",
+            "block reward"
+        ]
+    },
+    
+    "COMMUNITY": {
+        "ROLES": [
+            "whale", "degen", "anon", "influencer", "kol", "thought leader", "ambassador",
+            "advocate", "og", "contributor", "dev", "builder", "founder", "investor", "vc",
+            "angel", "team", "core team", "front runner", "mev bot", "searcher", "validator",
+            "miner", "node operator", "liquidity provider", "market maker", "arbitrageur"
+        ],
+        "SLANG": [
+            "diamond hands", "paper hands", "wagmi", "ngmi", "gm", "gn", "ser", "based",
+            "crypto twitter", "ct", "alpha", "dyor", "fomo", "fud", "hodl", "rekt"
+        ]
+    }
+}
+
+# ==============================================
+# Helper Functions
+# ==============================================
+
+def clear_gpu_memory():
+    """Clear GPU memory to prevent OOM errors"""
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+    gc.collect()
+
+def load_checkpoint():
+    """Load processing checkpoint if it exists"""
+    if os.path.exists(CHECKPOINT_FILE):
+        with open(CHECKPOINT_FILE, 'r') as f:
+            return json.load(f)
+    return {'last_processed_index': 0}
+
+def save_checkpoint(index):
+    """Save the current processing index to a checkpoint file"""
+    with open(CHECKPOINT_FILE, 'w') as f:
+        json.dump({'last_processed_index': index}, f)
+
+def identify_crypto_entities(text: str) -> list:
+    """
+    Identify crypto-specific entities in text using the hierarchical taxonomy.
+    
+    Args:
+        text (str): Text to analyze
+    
+    Returns:
+        list: List of tuples (entity, main_category, sub_category)
+    """
+    if not isinstance(text, str):
+        return []
+    
+    text_lower = text.lower()
+    found_entities = []
+    
+    # Search for each entity in the taxonomy
+    for main_cat, subcats in CRYPTO_TAXONOMY.items():
+        for subcat, terms in subcats.items():
+            for term in terms:
+                # Avoid matching partial words (ensure word boundaries)
+                pattern = r'\b' + re.escape(term) + r'\b'
+                if re.search(pattern, text_lower):
+                    found_entities.append((term, main_cat, subcat))
+    
+    return found_entities
+
+def clean_text(text: str) -> str:
+    """Clean text while preserving mentions and hashtags"""
+    if not isinstance(text, str):
+        return ""
+    
+    # Remove URLs
+    text = re.sub(r'http\S+', '', text)
+    
+    # Remove non-alphanumeric characters (except mentions, hashtags, and spaces)
+    text = re.sub(r'[^\w\s@#]', ' ', text)
+    
+    # Remove extra whitespace
+    text = re.sub(r'\s+', ' ', text).strip()
+    
+    return text.lower()
+
+def process_nlp_text(text: str) -> str:
+    """Process text with advanced NLP (lemmatization, stopword removal)"""
+    if not isinstance(text, str):
+        return ""
+    
+    # Basic cleaning
+    text = clean_text(text)
+    
+    if SPACY_AVAILABLE:
+        # Process with spaCy for advanced NLP
+        doc = spacy_nlp(text)
+        
+        # Lemmatize and remove stopwords
+        processed_tokens = [token.lemma_ for token in doc if not token.is_stop and not token.is_punct]
+        
+        return " ".join(processed_tokens)
+    else:
+        # Fallback to basic cleaning if spaCy is not available
+        return text
+
+def extract_mentions(text: str) -> list:
+    """Extract @mentions from text"""
+    if not isinstance(text, str):
+        return []
+    return re.findall(r'@(\w+)', text)
+
+def extract_hashtags(text: str) -> list:
+    """Extract #hashtags from text"""
+    if not isinstance(text, str):
+        return []
+    return re.findall(r'#(\w+)', text)
+
+def extract_urls(text: str) -> list:
+    """Extract URLs from text"""
+    if not isinstance(text, str):
+        return []
+    urls = re.findall(r'(https?://\S+)', text)
+    return urls
+
+def analyze_sentiment(text: str) -> dict:
+    """
+    Analyze the sentiment of a text using the sentiment analysis pipeline.
+    
+    Args:
+        text (str): The text to analyze
+        
+    Returns:
+        dict: A dictionary containing sentiment label and score
+    """
+    if not SENTIMENT_AVAILABLE or not text.strip():
+        return {"sentiment": "unknown", "sentiment_score": 0.0, "sentiment_magnitude": 0.0}
+    
+    try:
+        # Pre-process the text to improve sentiment analysis accuracy
+        # Limit text length to avoid errors with very long tweets
+        truncated_text = text[:512] if len(text) > 512 else text
+        
+        # Get sentiment prediction
+        sentiment_result = sentiment_pipeline(truncated_text)[0]
+        label = sentiment_result['label']
+        score = sentiment_result['score']
+        
+        # Map to standardized format (positive, negative, neutral)
+        sentiment_mapping = {
+            'LABEL_0': 'negative',
+            'LABEL_1': 'neutral',
+            'LABEL_2': 'positive',
+            'NEGATIVE': 'negative',
+            'NEUTRAL': 'neutral',
+            'POSITIVE': 'positive'
+        }
+        
+        standardized_sentiment = sentiment_mapping.get(label, label.lower())
+        
+        # Calculate magnitude (confidence) - useful for filtering high-confidence sentiments
+        magnitude = abs(score - 0.5) * 2 if standardized_sentiment != 'neutral' else score
+        
+        return {
+            "sentiment": standardized_sentiment,
+            "sentiment_score": score,
+            "sentiment_magnitude": magnitude
+        }
+    except Exception as e:
+        print(f"Error in sentiment analysis: {e}")
+        return {"sentiment": "error", "sentiment_score": 0.0, "sentiment_magnitude": 0.0}
+
+def process_with_nlp(text: str) -> dict:
+    """
+    Process text with NLP to extract named entities, key phrases, etc.
+    
+    Args:
+        text (str): The text to process
+        
+    Returns:
+        dict: A dictionary containing NLP processing results
+    """
+    results = {
+        "named_entities": [],
+        "pos_tags": [],
+        "lemmatized_tokens": [],
+        "key_phrases": [],
+        "important_nouns": [],
+        "sentiment_analysis": {"sentiment": "unknown", "sentiment_score": 0.0, "sentiment_magnitude": 0.0}
+    }
+    
+    if not text or text.isspace():
+        return results
+    
+    # First, analyze sentiment
+    results["sentiment_analysis"] = analyze_sentiment(text)
+    
+    try:
+        # Use spaCy for advanced NLP if available
+        if SPACY_AVAILABLE:
+            doc = spacy_nlp(text)
+            
+            # Extract named entities (excluding crypto entities which are handled separately)
+            results["named_entities"] = [(ent.text, ent.label_) for ent in doc.ents]
+            
+            # Extract POS tags for content words
+            results["pos_tags"] = [(token.text, token.pos_) for token in doc 
+                                   if token.pos_ in ["NOUN", "VERB", "ADJ", "ADV"] and not token.is_stop]
+            
+            # Get lemmatized tokens (normalized words)
+            results["lemmatized_tokens"] = [token.lemma_ for token in doc 
+                                           if not token.is_stop and not token.is_punct and token.text.strip()]
+            
+            # Extract important nouns (potential topics)
+            results["important_nouns"] = [token.text for token in doc 
+                                         if token.pos_ == "NOUN" and not token.is_stop]
+            
+            # Try to extract key phrases using noun chunks
+            results["key_phrases"] = [chunk.text for chunk in doc.noun_chunks 
+                                     if len(chunk.text.split()) > 1]
+        
+        # If key phrases are empty, use RoBERTa to attempt extraction
+        if not results["key_phrases"] and len(text.split()) > 3:
+            try:
+                # Create a masked sentence from the text
+                words = text.split()
+                if len(words) > 5:
+                    # Get 3 random positions to mask
+                    import random
+                    positions = sorted(random.sample(range(len(words)), min(3, len(words))))
+                    
+                    # Create masked sentences
+                    key_terms = []
+                    for pos in positions:
+                        words_copy = words.copy()
+                        words_copy[pos] = tokenizer.mask_token
+                        masked_text = " ".join(words_copy)
+                        
+                        # Get predictions for the masked token
+                        predictions = nlp_pipeline(masked_text, top_k=2)
+                        for prediction in predictions:
+                            key_terms.append(prediction["token_str"].strip())
+                    
+                    results["key_phrases"].extend(key_terms)
+            except Exception as e:
+                print(f"Error in key phrase extraction: {e}")
+                
+        # Ensure all results are strings for CSV output
+        results["named_entities"] = ";".join([f"{ent[0]}:{ent[1]}" for ent in results["named_entities"]])
+        results["pos_tags"] = ";".join([f"{tag[0]}:{tag[1]}" for tag in results["pos_tags"]])
+        results["lemmatized_tokens"] = ";".join(results["lemmatized_tokens"])
+        results["key_phrases"] = ";".join(list(set(results["key_phrases"])))  # Remove duplicates
+        results["important_nouns"] = ";".join(list(set(results["important_nouns"])))  # Remove duplicates
+            
+    except Exception as e:
+        print(f"Error in NLP processing: {e}")
+    
+    # Clear GPU memory after processing
+    if (results["named_entities"].count(";") > 100) or (len(text) > 1000):
+        clear_gpu_memory()
+        
+    return results
+
+def process_tweet(text: str) -> tuple:
+    """
+    Process a tweet to extract mentions, hashtags, URLs, crypto entities, and perform NLP analysis.
+    Also performs sentiment analysis.
+    
+    Args:
+        text (str): The tweet text to process
+        
+    Returns:
+        tuple: A tuple containing mentions, hashtags, URLs, NLP results, and sentiment analysis
+    """
+    if not text or not isinstance(text, str):
+        return [], [], [], "", "", {}, {"sentiment": "unknown", "sentiment_score": 0.0, "sentiment_magnitude": 0.0}
+    
+    # Clean the text while preserving mentions and hashtags
+    cleaned_text = clean_text(text)
+    
+    # Process text with NLP
+    processed_text = process_nlp_text(text)
+    
+    # Extract mentions, hashtags, and URLs
+    mentions = extract_mentions(text)
+    hashtags = extract_hashtags(text)
+    urls = extract_urls(text)
+    
+    # Identify crypto entities
+    crypto_entities = identify_crypto_entities(text)
+    
+    # Process with NLP models
+    nlp_results = process_with_nlp(text)
+    
+    # Ensure we have the sentiment analysis results
+    sentiment_results = nlp_results.pop("sentiment_analysis", {"sentiment": "unknown", "sentiment_score": 0.0, "sentiment_magnitude": 0.0})
+    
+    # Add crypto entities to the named entities
+    formatted_crypto_entities = [f"{entity}:{main_cat}.{sub_cat}" for entity, main_cat, sub_cat in crypto_entities]
+    
+    # If named_entities is a string (joined with semicolons), we need to handle differently
+    if isinstance(nlp_results.get("named_entities", ""), str):
+        nlp_results["named_entities"] = nlp_results.get("named_entities", "")
+        if nlp_results["named_entities"] and formatted_crypto_entities:
+            nlp_results["named_entities"] += ";" + ";".join(formatted_crypto_entities)
+        elif formatted_crypto_entities:
+            nlp_results["named_entities"] = ";".join(formatted_crypto_entities)
+    
+    return mentions, hashtags, urls, cleaned_text, processed_text, nlp_results, sentiment_results
+
+def process_batch(df_batch):
+    """Process a batch of tweets"""
+    processed_data = []
+    
+    for idx, row in df_batch.iterrows():
+        text = row.get('text', '')
+        
+        # Process the tweet
+        mentions, hashtags, urls, cleaned_text, processed_text, nlp_results, sentiment_results = process_tweet(text)
+        
+        # Create a dictionary with the results
+        result = {
+            'id': row.get('id', ''),
+            'original_text': text,  # Store the original text
+            'cleaned_text': cleaned_text,
+            'nlp_processed_text': processed_text,
+            'extracted_mentions': ';'.join(mentions),
+            'extracted_hashtags': ';'.join(hashtags),
+            'extracted_urls': ';'.join(urls),
+            'named_entities': nlp_results.get('named_entities', ''),
+            'pos_tags': nlp_results.get('pos_tags', ''),
+            'lemmatized_tokens': nlp_results.get('lemmatized_tokens', ''),
+            'key_phrases': nlp_results.get('key_phrases', ''),
+            'important_nouns': nlp_results.get('important_nouns', ''),
+            'sentiment': sentiment_results.get('sentiment', 'unknown'),
+            'sentiment_score': sentiment_results.get('sentiment_score', 0.0),
+            'sentiment_magnitude': sentiment_results.get('sentiment_magnitude', 0.0)
+        }
+        
+        processed_data.append(result)
+    
+    return pd.DataFrame(processed_data)
+
+# ==============================================
+# Main Processing Function
+# ==============================================
+
+def main(reset_checkpoint=False, input_file=None):
+    """
+    Main function to process tweets
+    
+    Args:
+        reset_checkpoint (bool): Whether to reset the checkpoint and process all data
+        input_file (str): Optional specific input file to process, otherwise processes all CSV files
+    """
+    if reset_checkpoint and os.path.exists(CHECKPOINT_FILE):
+        os.remove(CHECKPOINT_FILE)
+        print("Checkpoint reset. Will process all data from the beginning.")
+    
+    # Get list of CSV files to process
+    if input_file:
+        # Process a specific file
+        input_files = [input_file]
+    else:
+        # Find all CSV files in the OUTPUT_FOLDER
+        import glob
+        input_files = glob.glob(f"{OUTPUT_FOLDER}/*.csv")
+        
+        # Exclude our output files
+        input_files = [f for f in input_files if not any(x in f for x in ["_processed.csv", "_analysis.csv"])]
+    
+    if not input_files:
+        print(f"No input CSV files found in {OUTPUT_FOLDER}")
+        return
+    
+    print(f"Found {len(input_files)} files to process: {[os.path.basename(f) for f in input_files]}")
+    
+    # Process each file
+    for input_csv in input_files:
+        print(f"\nProcessing file: {os.path.basename(input_csv)}")
+        
+        print("Loading dataset...")
+        # Check if input file exists
+        if not os.path.exists(input_csv):
+            print(f"Input file {input_csv} not found. Skipping.")
+            continue
+        
+        # Load the dataset
+        try:
+            df = pd.read_csv(input_csv)
+            print(f"Loaded dataset with {len(df)} records and {len(df.columns)} columns.")
+        except Exception as e:
+            print(f"Error loading {input_csv}: {e}")
+            continue
+        
+        # Load checkpoint if it exists
+        checkpoint = load_checkpoint()
+        start_idx = checkpoint['last_processed_index']
+        
+        # For simplicity, reset checkpoints between files
+        start_idx = 0
+        save_checkpoint(0)
+        
+        print("\nProcessing tweets...")
+        print(f"Starting from index {start_idx}")
+        
+        # Filter to only unprocessed rows
+        df_to_process = df.iloc[start_idx:]
+        
+        if len(df_to_process) == 0:
+            print("No new data to process in this file.")
+            continue
+        
+        # Process in batches for memory efficiency
+        batch_size = BATCH_SIZE
+        num_batches = math.ceil(len(df_to_process) / batch_size)
+        print(f"Processing in {num_batches} batches of {batch_size} records each")
+        
+        processed_batches = []
+        
+        # Create progress bar
+        for i in tqdm(range(num_batches)):
+            batch_start = i * batch_size
+            batch_end = min((i + 1) * batch_size, len(df_to_process))
+            
+            # Get current batch
+            df_batch = df_to_process.iloc[batch_start:batch_end]
+            
+            # Process the batch
+            processed_batch = process_batch(df_batch)
+            processed_batches.append(processed_batch)
+            
+            # Save checkpoint
+            save_checkpoint(start_idx + batch_end)
+            
+            # Save intermediate results every 5 batches to prevent data loss in case of session timeout
+            if i % 5 == 0 and i > 0:
+                file_basename = os.path.splitext(os.path.basename(input_csv))[0]
+                interim_df = pd.concat(processed_batches, ignore_index=True)
+                interim_file = f"{OUTPUT_FOLDER}/{file_basename}_interim_{i}.csv"
+                interim_df.to_csv(interim_file, index=False)
+                print(f"\nSaved interim results to {interim_file}")
+                
+                # Clear memory
+                clear_gpu_memory()
+        
+        # Combine all batches
+        if processed_batches:
+            file_basename = os.path.splitext(os.path.basename(input_csv))[0]
+            
+            final_df = pd.concat(processed_batches, ignore_index=True)
+            
+            # Calculate statistics columns
+            final_df["mention_count"] = final_df["extracted_mentions"].str.count(";") + (final_df["extracted_mentions"] != "").astype(int)
+            final_df["hashtag_count"] = final_df["extracted_hashtags"].str.count(";") + (final_df["extracted_hashtags"] != "").astype(int)
+            final_df["entity_count"] = final_df["named_entities"].str.count(";") + (final_df["named_entities"] != "").astype(int)
+            
+            # Save the full processed dataset
+            output_file = f"{OUTPUT_FOLDER}/{file_basename}_processed.csv"
+            final_df.to_csv(output_file, index=False)
+            print(f"Processed data saved to {output_file}")
+            
+            # Create a lighter version with just the analysis
+            analysis_columns = [
+                "id", "original_text", "cleaned_text", "nlp_processed_text", 
+                "extracted_mentions", "extracted_hashtags", "extracted_urls",
+                "named_entities", "key_phrases", "important_nouns",
+                "sentiment", "sentiment_score", "sentiment_magnitude",
+                "mention_count", "hashtag_count", "entity_count"
+            ]
+            
+            # Ensure all columns exist before subsetting
+            available_columns = [col for col in analysis_columns if col in final_df.columns]
+            analysis_df = final_df[available_columns]
+            analysis_file = f"{OUTPUT_FOLDER}/{file_basename}_analysis.csv"
+            analysis_df.to_csv(analysis_file, index=False)
+            print(f"Analysis results saved to {analysis_file}")
+            
+            # Print statistics
+            print(f"\nAnalysis completed successfully!")
+            print(f"Total records: {len(final_df)}")
+            print(f"Tweets with Mentions: {(final_df['extracted_mentions'] != '').sum()}")
+            print(f"Tweets with Hashtags: {(final_df['extracted_hashtags'] != '').sum()}")
+            print(f"Tweets with Named Entities: {(final_df['named_entities'] != '').sum()}")
+            
+            # Print sentiment statistics
+            sentiment_counts = final_df['sentiment'].value_counts()
+            print("\nSentiment Distribution:")
+            for sentiment, count in sentiment_counts.items():
+                percentage = (count / len(final_df)) * 100
+                print(f"  {sentiment}: {count} tweets ({percentage:.1f}%)")
+            
+            # Get average sentiment scores
+            avg_score = final_df['sentiment_score'].mean()
+            avg_magnitude = final_df['sentiment_magnitude'].mean()
+            print(f"\nAverage sentiment score: {avg_score:.3f}")
+            print(f"Average sentiment magnitude: {avg_magnitude:.3f}")
+            
+            # Get top entities by sentiment
+            positive_entities = []
+            for idx, row in final_df[final_df['sentiment'] == 'positive'].iterrows():
+                entities = row['named_entities'].split(';') if isinstance(row['named_entities'], str) and row['named_entities'] else []
+                for entity in entities:
+                    if entity and ':' in entity:
+                        entity_name = entity.split(':')[0]
+                        positive_entities.append(entity_name)
+            
+            # Get the most common positive entities
+            from collections import Counter
+            top_positive = Counter(positive_entities).most_common(5)
+            if top_positive:
+                print("\nTop entities with positive sentiment:")
+                for entity, count in top_positive:
+                    print(f"  {entity}: {count} mentions")
+            
+            # Print sample results
+            print("\nSample of processing results:")
+            for i, row in analysis_df.head(3).iterrows():
+                print(f"\nOriginal Text: {row['original_text']}")
+                print(f"Cleaned Text: {row['cleaned_text']}")
+                print(f"NLP Processed Text: {row['nlp_processed_text']}")
+                print(f"Mentions: {row['extracted_mentions']}")
+                print(f"Hashtags: {row['extracted_hashtags']}")
+                print(f"Named Entities: {row['named_entities']}")
+                print(f"Key Phrases: {row['key_phrases']}")
+                print(f"Sentiment: {row['sentiment']} (Score: {row['sentiment_score']:.3f}, Magnitude: {row['sentiment_magnitude']:.3f})")
+                print("-" * 80)
+            
+            # Delete interim files
+            import glob
+            interim_files = glob.glob(f"{OUTPUT_FOLDER}/{file_basename}_interim_*.csv")
+            for f in interim_files:
+                try:
+                    os.remove(f)
+                    print(f"Deleted interim file: {os.path.basename(f)}")
+                except:
+                    pass
+                    
+            # Clear memory after processing each file
+            clear_gpu_memory()
+        else:
+            print("No data processed for this file.")
+    
+    # Clean up checkpoint file after successful processing
+    if os.path.exists(CHECKPOINT_FILE):
+        os.remove(CHECKPOINT_FILE)
+    print("\nAll files processed successfully!")
+
+# ==============================================
+# Colab Usage Example
+# ==============================================
+
+"""
+# EXAMPLE USAGE IN COLAB:
+
+# 1. Install packages and mount drive
+from google.colab import drive
+drive.mount('/content/drive')
+
+# 2. Process one specific file
+input_file = "/content/drive/MyDrive/projects_twitter_post/zilliqa.csv"
+main(reset_checkpoint=True, input_file=input_file)
+
+# 3. Process all files
+main(reset_checkpoint=True)
+"""
+
+if __name__ == "__main__":
+    import sys
+    
+    # Check if --reset flag is provided
+    reset_checkpoint = "--reset" in sys.argv
+    
+    # Check if --file flag is provided
+    input_file = None
+    if "--file" in sys.argv:
+        try:
+            input_file = sys.argv[sys.argv.index("--file") + 1]
+        except IndexError:
+            print("Error: --file flag requires a filename argument")
+            sys.exit(1)
+    
+    # Run the main function
+    main(reset_checkpoint=reset_checkpoint, input_file=input_file) )
+   
+   demo.launch()