nse_modules/data_summary.py

"""
NSE F&O Data Summary Module - Enhanced with Date Range and Pure Nifty Filter
"""

import gradio as gr
import datetime
import pandas as pd
import os
from typing import Dict, Optional, List
import json

class DataSummary:
    def __init__(self):
        self.processed_dir = "processed_derivatives"

    def get_derivatives_summary(self, from_date: str, to_date: str, selected_stock: str) -> str:
        """Get comprehensive derivatives data summary for date range"""
        try:
            if not from_date or not to_date:
                return "❌ Please select both from and to dates", None, None, None
            
            from_date_obj = datetime.datetime.strptime(from_date, "%Y-%m-%d").date()
            to_date_obj = datetime.datetime.strptime(to_date, "%Y-%m-%d").date()
            
            if from_date_obj > to_date_obj:
                return "❌ From date cannot be after To date", None, None, None
            
            # Get data for the latest date in the range for summary
            latest_date = to_date_obj
            existence_check = self._check_file_exists(latest_date)
            
            if not existence_check['extract_exists']:
                return f"❌ No derivatives data available for the selected date range", None, None, None
            
            # Read and analyze data for latest date
            df_latest = self._read_derivatives_csv(existence_check['csv_path'])
            if df_latest is None or df_latest.empty:
                return f"❌ Could not read derivatives data for the selected date range", None, None, None
            
            # Filter data if stock is selected
            filtered_df = df_latest.copy()
            if selected_stock and selected_stock.strip() and selected_stock != "All Stocks":
                filtered_df = self._filter_by_stock(df_latest, selected_stock.strip())
            
            # Get combined data for the entire date range
            combined_df = self._get_combined_data(from_date_obj, to_date_obj, selected_stock.strip() if selected_stock else "")
            
            summary = self._generate_summary(filtered_df, latest_date)
            table_data = self._generate_table_data(filtered_df)
            nifty_table_data = self._generate_pure_nifty_table_data(combined_df)
            
            output = f"## 📊 Derivatives Summary\n\n"
            output += f"**Date Range:** {from_date} to {to_date}\n"
            output += f"**Summary Date:** {summary['date']}\n"
            
            if selected_stock and selected_stock.strip() and selected_stock != "All Stocks":
                output += f"**Filtered for:** {selected_stock.strip()}\n\n"
            
            output += f"**Data Quality:** {summary['data_quality']}\n"
            output += f"**Total Records:** {summary['total_records']:,}\n"
            output += f"**Total Volume:** {summary['total_volume']:,}\n"
            output += f"**Total Turnover:** ₹{summary['total_turnover']:,.2f}\n"
            output += f"**Total Open Interest:** {summary['total_open_interest']:,}\n"
            output += f"**Call Options:** {summary['call_options_count']:,}\n"
            output += f"**Put Options:** {summary['put_options_count']:,}\n"
            output += f"**Put/Call Ratio:** {summary['put_call_ratio']:.2f}\n\n"
            
            output += "### 📈 Top 5 Stocks by Volume\n"
            for i, stock in enumerate(summary['top_volume_stocks'][:5]):
                output += f"{i+1}. **{stock['SYMBOL']}**: {stock['VOLUME']:,}\n"
            
            output += "\n### 🔥 Top 5 OI Changes\n"
            for i, stock in enumerate(summary['top_oi_changes'][:5]):
                change_icon = "🟢" if stock['CHG_IN_OI'] >= 0 else "🔴"
                output += f"{i+1}. **{stock['SYMBOL']}**: {change_icon} {stock['CHG_IN_OI']:+,}\n"
            
            return output, table_data, nifty_table_data, combined_df
            
        except Exception as e:
            return f"❌ Error generating summary: {str(e)}", None, None, None

    def _get_combined_data(self, from_date: datetime.date, to_date: datetime.date, selected_stock: str) -> pd.DataFrame:
        """Get combined data for the entire date range"""
        combined_data = []
        
        current_date = from_date
        while current_date <= to_date:
            existence_check = self._check_file_exists(current_date)
            
            if existence_check['extract_exists']:
                df = self._read_derivatives_csv(existence_check['csv_path'])
                if df is not None and not df.empty:
                    # Filter by stock if selected
                    if selected_stock and selected_stock.strip() and selected_stock != "All Stocks":
                        df = self._filter_by_stock(df, selected_stock.strip())
                    
                    # Add date column
                    df['TRADE_DATE'] = current_date.strftime('%Y-%m-%d')
                    combined_data.append(df)
            
            current_date += datetime.timedelta(days=1)
        
        if combined_data:
            return pd.concat(combined_data, ignore_index=True)
        return pd.DataFrame()

    def _filter_by_stock(self, df: pd.DataFrame, stock_name: str) -> pd.DataFrame:
        """Filter dataframe by stock name"""
        df.columns = [col.upper() if isinstance(col, str) else col for col in df.columns]
        if 'TCKRSYMB' in df.columns:
            return df[df['TCKRSYMB'] == stock_name].copy()
        return df

    def _generate_pure_nifty_table_data(self, df: pd.DataFrame) -> pd.DataFrame:
        """Generate Nifty-specific data table - ONLY NIFTY, no other indices"""
        if df.empty:
            return pd.DataFrame(columns=[
                'Stock Name', 'OI', 'Strike', 'Settlement Price', 'Option Type', 
                'Expiry Date', 'Volume', 'Change in OI', 'Trade Date'
            ])
        
        # Convert column names to uppercase
        df.columns = [col.upper() if isinstance(col, str) else col for col in df.columns]
        
        # Filter for ONLY NIFTY (exclude BankNifty, FinNifty, Midcap, etc.)
        nifty_df = df.copy()
        if 'TCKRSYMB' in df.columns:
            # Strict filter for only NIFTY
            nifty_df = df[df['TCKRSYMB'].str.upper() == 'NIFTY'].copy()
        
        if nifty_df.empty:
            # Also try common Nifty variations
            nifty_variations = ['NIFTY', 'NIFTY 50', 'NIFTY50']
            for variation in nifty_variations:
                nifty_df = df[df['TCKRSYMB'].str.upper().str.contains(variation, na=False)].copy()
                if not nifty_df.empty:
                    break
        
        if nifty_df.empty:
            return pd.DataFrame(columns=[
                'Stock Name', 'OI', 'Strike', 'Settlement Price', 'Option Type', 
                'Expiry Date', 'Volume', 'Change in OI', 'Trade Date'
            ])
        
        # Map NSE column names to user-friendly names
        column_mapping = {
            'TCKRSYMB': 'Stock Name',
            'OPNINTRST': 'OI',
            'STRKPRIC': 'Strike',
            'XPRYDT': 'Expiry Date',
            'OPTNTP': 'Option Type',
            'SETTLPR': 'Settlement Price',
            'TTLTRADGVOL': 'Volume',
            'CHNGINOPNINTRST': 'Change in OI',
            'TRADE_DATE': 'Trade Date'
        }
        
        # Select and rename columns for display
        display_columns = []
        for col in ['TCKRSYMB', 'OPNINTRST', 'STRKPRIC', 'SETTLPR', 'OPTNTP', 'XPRYDT', 'TTLTRADGVOL', 'CHNGINOPNINTRST', 'TRADE_DATE']:
            if col in nifty_df.columns:
                display_columns.append(col)
        
        table_df = nifty_df[display_columns].copy()
        table_df.columns = [column_mapping.get(col, col) for col in table_df.columns]
        
        # Format numeric columns
        numeric_columns = ['OI', 'Strike', 'Settlement Price', 'Volume', 'Change in OI']
        for col in numeric_columns:
            if col in table_df.columns:
                table_df[col] = pd.to_numeric(table_df[col], errors='coerce')
        
        # Sort by Trade Date, Strike Price and Option Type for better readability
        sort_columns = []
        if 'Trade Date' in table_df.columns:
            sort_columns.append('Trade Date')
        if 'Strike' in table_df.columns:
            sort_columns.append('Strike')
        if 'Option Type' in table_df.columns:
            sort_columns.append('Option Type')
        
        if sort_columns:
            table_df = table_df.sort_values(sort_columns)
        
        return table_df

    def _generate_table_data(self, df: pd.DataFrame) -> pd.DataFrame:
        """Generate tabular data for display and export"""
        # Convert column names to uppercase for consistency
        df.columns = [col.upper() if isinstance(col, str) else col for col in df.columns]
        
        # Map NSE column names to user-friendly names
        column_mapping = {
            'TCKRSYMB': 'Stock Name',
            'OPNINTRST': 'OI',
            'STRKPRIC': 'Strike',
            'XPRYDT': 'Expiry Date',
            'OPTNTP': 'Option Type',
            'SETTLPR': 'Settlement Price',
            'TTLTRADGVOL': 'Volume',
            'CHNGINOPNINTRST': 'Change in OI'
        }
        
        # Select and rename columns for display
        display_columns = []
        for col in ['TCKRSYMB', 'OPNINTRST', 'STRKPRIC', 'SETTLPR', 'OPTNTP', 'XPRYDT', 'TTLTRADGVOL', 'CHNGINOPNINTRST']:
            if col in df.columns:
                display_columns.append(col)
        
        table_df = df[display_columns].copy()
        table_df.columns = [column_mapping.get(col, col) for col in table_df.columns]
        
        # Add trade date column (current date)
        table_df['Trade Date'] = datetime.date.today().strftime('%Y-%m-%d')
        
        # Format numeric columns
        numeric_columns = ['OI', 'Strike', 'Settlement Price', 'Volume', 'Change in OI']
        for col in numeric_columns:
            if col in table_df.columns:
                table_df[col] = pd.to_numeric(table_df[col], errors='coerce')
        
        return table_df

    def export_to_csv(self, from_date: str, to_date: str, selected_stock: str, current_df: pd.DataFrame) -> str:
        """Export data to CSV file"""
        try:
            if not from_date or not to_date:
                return "❌ Please select date range first"
            
            if current_df is None or current_df.empty:
                return "❌ No data available to export"
            
            # Create exports directory
            export_dir = "exports"
            os.makedirs(export_dir, exist_ok=True)
            
            # Generate filename
            stock_suffix = f"_{selected_stock}" if selected_stock and selected_stock != "All Stocks" else "_AllStocks"
            filename = f"nse_fo_data_{from_date}_to_{to_date}{stock_suffix}.csv"
            filepath = os.path.join(export_dir, filename)
            
            # Export to CSV
            current_df.to_csv(filepath, index=False)
            
            return f"✅ Data exported successfully to: {filepath}"
            
        except Exception as e:
            return f"❌ Export failed: {str(e)}"

    def export_nifty_to_csv(self, from_date: str, to_date: str, nifty_df: pd.DataFrame) -> str:
        """Export Nifty data to CSV file"""
        try:
            if not from_date or not to_date:
                return "❌ Please select date range first"
            
            if nifty_df is None or nifty_df.empty:
                return "❌ No Nifty data available to export"
            
            # Create exports directory
            export_dir = "exports"
            os.makedirs(export_dir, exist_ok=True)
            
            # Generate filename
            filename = f"nifty_options_{from_date}_to_{to_date}.csv"
            filepath = os.path.join(export_dir, filename)
            
            # Export to CSV
            nifty_df.to_csv(filepath, index=False)
            
            return f"✅ Nifty data exported successfully to: {filepath}"
            
        except Exception as e:
            return f"❌ Export failed: {str(e)}"

    def _check_file_exists(self, date: datetime.date) -> Dict:
        """Check if file exists"""
        yyyymmdd = date.strftime('%Y%m%d')
        extract_dir = os.path.join(self.processed_dir, yyyymmdd)
        
        extract_exists = False
        csv_path = None
        
        if os.path.exists(extract_dir):
            csv_files = [f for f in os.listdir(extract_dir) if f.endswith('.csv')]
            if csv_files:
                csv_path = os.path.join(extract_dir, csv_files[0])
                extract_exists = True
        
        return {
            'extract_exists': extract_exists,
            'csv_path': csv_path,
            'extract_dir': extract_dir
        }

    def _read_derivatives_csv(self, csv_path: str) -> Optional[pd.DataFrame]:
        """Read derivatives CSV file"""
        try:
            # Try different separators
            for separator in ['\t', ',', ';']:
                try:
                    df = pd.read_csv(csv_path, sep=separator, nrows=1)
                    if len(df.columns) > 3:
                        return pd.read_csv(csv_path, sep=separator)
                except:
                    continue
            return None
        except Exception:
            return None

    def _generate_summary(self, df: pd.DataFrame, date: datetime.date) -> Dict:
        """Generate summary statistics from DataFrame"""
        # Convert column names to uppercase for consistency
        df.columns = [col.upper() if isinstance(col, str) else col for col in df.columns]
        
        # Initialize summary
        total_records = len(df)
        
        # Map NSE column names
        volume_col = 'TTLTRADGVOL'
        turnover_col = 'TTLTRFVAL'
        oi_col = 'OPNINTRST'
        oi_change_col = 'CHNGINOPNINTRST'
        symbol_col = 'TCKRSYMB'
        option_type_col = 'OPTNTP'
        
        # Calculate metrics
        total_volume = df[volume_col].sum() if volume_col in df.columns else 0
        total_turnover = df[turnover_col].sum() if turnover_col in df.columns else 0
        total_oi = df[oi_col].sum() if oi_col in df.columns else 0
        
        # Count options
        call_options_count = 0
        put_options_count = 0
        if option_type_col in df.columns:
            call_options_count = len(df[df[option_type_col].str.upper().str.contains('CE', na=False)])
            put_options_count = len(df[df[option_type_col].str.upper().str.contains('PE', na=False)])
        
        # Put/Call ratio
        put_call_ratio = put_options_count / call_options_count if call_options_count > 0 else 0
        
        # Top stocks by volume
        top_volume_stocks = []
        if symbol_col in df.columns and volume_col in df.columns:
            volume_by_stock = df.groupby(symbol_col)[volume_col].sum().nlargest(5)
            top_volume_stocks = [{'SYMBOL': str(sym), 'VOLUME': float(vol)} for sym, vol in volume_by_stock.items()]
        
        # Top OI changes
        top_oi_changes = []
        if symbol_col in df.columns and oi_change_col in df.columns:
            oi_changes = df.groupby(symbol_col)[oi_change_col].sum().nlargest(5)
            top_oi_changes = [{'SYMBOL': str(sym), 'CHG_IN_OI': float(oi)} for sym, oi in oi_changes.items()]
        
        # Data quality assessment
        data_quality = "Good" if total_records > 1000 else "Limited"
        
        return {
            "date": date.strftime('%Y-%m-%d'),
            "total_records": total_records,
            "total_volume": total_volume,
            "total_turnover": total_turnover,
            "total_open_interest": total_oi,
            "call_options_count": call_options_count,
            "put_options_count": put_options_count,
            "put_call_ratio": put_call_ratio,
            "top_volume_stocks": top_volume_stocks,
            "top_oi_changes": top_oi_changes,
            "data_quality": data_quality
        }

    def create_interface(self):
        """Create data summary interface with date range and pure Nifty filter"""
        gr.Markdown("### 📊 F&O Data Summary & Export")
        gr.Markdown("View detailed summary for date range, filter by stock, and export data to CSV")
        
        with gr.Row():
            from_date = gr.Textbox(
                label="From Date (YYYY-MM-DD)",
                placeholder="e.g., 2024-01-01",
                value=(datetime.date.today() - datetime.timedelta(days=7)).strftime("%Y-%m-%d")
            )
            to_date = gr.Textbox(
                label="To Date (YYYY-MM-DD)",
                placeholder="e.g., 2024-01-15",
                value=datetime.date.today().strftime("%Y-%m-%d")
            )
            stock_input = gr.Textbox(
                label="Stock Symbol (Optional)",
                placeholder="e.g., RELIANCE, TCS, INFY (leave empty for all stocks)",
                value="",
                info="Enter stock symbol to filter data"
            )
        
        with gr.Row():
            summary_btn = gr.Button("📈 Generate Summary", variant="primary")
        
        summary_output = gr.Markdown()
        
        # Data table display
        gr.Markdown("### 📋 Detailed Data Table")
        data_table = gr.Dataframe(
            label="F&O Data",
            headers=["Stock Name", "OI", "Strike", "Price", "Option Type", "Expiry Date", "Volume", "Change in OI", "Trade Date"],
            wrap=True,
            max_height=400
        )
        
        # Pure Nifty table (ONLY NIFTY, no other indices)
        gr.Markdown("### 🇮🇳 PURE NIFTY Options Data (No BankNifty/Other Indices)")
        nifty_table = gr.Dataframe(
            label="NIFTY Options",
            headers=["Stock Name", "OI", "Strike", "Price", "Option Type", "Expiry Date", "Volume", "Change in OI", "Trade Date"],
            wrap=True,
            max_height=400
        )
        
        with gr.Row():
            export_btn = gr.Button("📥 Export Filtered Data to CSV", variant="secondary")
            export_nifty_btn = gr.Button("📥 Export Nifty Data to CSV", variant="secondary")
        
        with gr.Row():
            export_output = gr.Markdown()
            export_nifty_output = gr.Markdown()
        
        # Store the current dataframes for export
        current_df = gr.State()
        current_nifty_df = gr.State()
        
        summary_btn.click(
            fn=self.get_derivatives_summary,
            inputs=[from_date, to_date, stock_input],
            outputs=[summary_output, data_table, nifty_table, current_df]
        )
        
        export_btn.click(
            fn=self.export_to_csv,
            inputs=[from_date, to_date, stock_input, current_df],
            outputs=[export_output]
        )
        
        export_nifty_btn.click(
            fn=self.export_nifty_to_csv,
            inputs=[from_date, to_date, current_nifty_df],
            outputs=[export_nifty_output]
        )
        
        gr.Markdown("---")
        gr.Markdown("### 📋 Features:")
        gr.Markdown("- **Date Range**: Analyze data across multiple dates")
        gr.Markdown("- **Stock Filter**: Enter stock symbol to filter data (leave empty for all stocks)")
        gr.Markdown("- **Pure Nifty Table**: Dedicated table for ONLY NIFTY options (excludes BankNifty, FinNifty, etc.)")
        gr.Markdown("- **CSV Export**: Export both filtered data and pure Nifty data")
        gr.Markdown("- **Fast Performance**: No slow dropdown loading")