Create app.py

app.py

@@ -0,0 +1,353 @@
+import gradio as gr
+import torch
+from transformers import (
+    BlipProcessor, BlipForConditionalGeneration,
+    TrOCRProcessor, VisionEncoderDecoderModel,
+    AutoProcessor, AutoModelForCausalLM
+)
+from PIL import Image
+import easyocr
+import matplotlib.pyplot as plt
+import pandas as pd
+import numpy as np
+import cv2
+import io
+import base64
+
+class ChartAnalyzer:
+    def __init__(self):
+        # Load models
+        self.load_models()
+        
+    def load_models(self):
+        """Load all required models"""
+        try:
+            # BLIP for image captioning and understanding
+            self.blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
+            self.blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
+            
+            # TrOCR for text extraction
+            self.trocr_processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-printed")
+            self.trocr_model = VisionEncoderDecoderModel.from_pretrained("microsoft/trocr-base-printed")
+            
+            # EasyOCR for backup text extraction
+            self.ocr_reader = easyocr.Reader(['en'])
+            
+            # Florence-2 for advanced understanding (if available)
+            try:
+                self.florence_processor = AutoProcessor.from_pretrained("microsoft/Florence-2-base")
+                self.florence_model = AutoModelForCausalLM.from_pretrained("microsoft/Florence-2-base")
+                self.florence_available = True
+            except:
+                self.florence_available = False
+                
+        except Exception as e:
+            print(f"Error loading models: {e}")
+    
+    def analyze_chart(self, image, analysis_type="comprehensive"):
+        """Main function to analyze charts"""
+        if image is None:
+            return "Please upload an image first."
+        
+        results = {}
+        
+        try:
+            # Convert to PIL Image if needed
+            if not isinstance(image, Image.Image):
+                image = Image.fromarray(image).convert('RGB')
+            
+            # Basic image understanding with BLIP
+            results['description'] = self.get_image_description(image)
+            
+            # Extract text using multiple methods
+            results['extracted_text'] = self.extract_text_multi_method(image)
+            
+            # Chart type detection
+            results['chart_type'] = self.detect_chart_type(image, results['description'])
+            
+            # Data extraction (if possible)
+            if analysis_type in ["comprehensive", "data_extraction"]:
+                results['data_points'] = self.extract_data_points(image, results['chart_type'])
+            
+            # Advanced analysis with Florence-2 (if available)
+            if self.florence_available and analysis_type == "comprehensive":
+                results['advanced_analysis'] = self.florence_analysis(image)
+            
+            return self.format_results(results)
+            
+        except Exception as e:
+            return f"Error analyzing chart: {str(e)}"
+    
+    def get_image_description(self, image):
+        """Get image description using BLIP"""
+        try:
+            inputs = self.blip_processor(image, return_tensors="pt")
+            out = self.blip_model.generate(**inputs, max_length=100)
+            description = self.blip_processor.decode(out[0], skip_special_tokens=True)
+            return description
+        except:
+            return "Unable to generate description"
+    
+    def extract_text_multi_method(self, image):
+        """Extract text using multiple OCR methods"""
+        extracted_texts = {}
+        
+        # Method 1: TrOCR
+        try:
+            pixel_values = self.trocr_processor(image, return_tensors="pt").pixel_values
+            generated_ids = self.trocr_model.generate(pixel_values)
+            trocr_text = self.trocr_processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+            extracted_texts['TrOCR'] = trocr_text
+        except:
+            extracted_texts['TrOCR'] = "Failed"
+        
+        # Method 2: EasyOCR
+        try:
+            # Convert PIL to numpy array
+            image_np = np.array(image)
+            ocr_results = self.ocr_reader.readtext(image_np)
+            easyocr_text = ' '.join([result[1] for result in ocr_results])
+            extracted_texts['EasyOCR'] = easyocr_text
+        except:
+            extracted_texts['EasyOCR'] = "Failed"
+        
+        return extracted_texts
+    
+    def detect_chart_type(self, image, description):
+        """Detect chart type based on image analysis"""
+        description_lower = description.lower()
+        
+        chart_keywords = {
+            'bar_chart': ['bar', 'column', 'histogram'],
+            'line_chart': ['line', 'trend', 'time series'],
+            'pie_chart': ['pie', 'circular', 'slice'],
+            'scatter_plot': ['scatter', 'correlation', 'points'],
+            'area_chart': ['area', 'filled'],
+            'box_plot': ['box', 'whisker'],
+            'heatmap': ['heat', 'color coded', 'matrix']
+        }
+        
+        for chart_type, keywords in chart_keywords.items():
+            if any(keyword in description_lower for keyword in keywords):
+                return chart_type.replace('_', ' ').title()
+        
+        return "Unknown Chart Type"
+    
+    def extract_data_points(self, image, chart_type):
+        """Attempt to extract data points (simplified approach)"""
+        try:
+            # This is a simplified version - real implementation would be more sophisticated
+            # Convert to grayscale for analysis
+            image_np = np.array(image.convert('L'))
+            
+            # Basic edge detection
+            edges = cv2.Canny(image_np, 50, 150)
+            
+            # Find contours
+            contours, _ = cv2.findContours(edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+            
+            data_info = {
+                'contours_found': len(contours),
+                'image_dimensions': image_np.shape,
+                'note': 'This is a simplified data extraction. Advanced algorithms needed for accurate data point extraction.'
+            }
+            
+            return data_info
+            
+        except Exception as e:
+            return f"Data extraction failed: {str(e)}"
+    
+    def florence_analysis(self, image):
+        """Advanced analysis using Florence-2"""
+        if not self.florence_available:
+            return "Florence-2 model not available"
+        
+        try:
+            # Florence-2 prompts for different tasks
+            prompts = [
+                "<OD>",  # Object Detection
+                "<DENSE_REGION_CAPTION>",  # Dense captioning
+                "<OCR_WITH_REGION>"  # OCR with regions
+            ]
+            
+            results = {}
+            for prompt in prompts:
+                inputs = self.florence_processor(text=prompt, images=image, return_tensors="pt")
+                generated_ids = self.florence_model.generate(
+                    input_ids=inputs["input_ids"],
+                    pixel_values=inputs["pixel_values"],
+                    max_new_tokens=1024,
+                    num_beams=3
+                )
+                generated_text = self.florence_processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
+                results[prompt] = generated_text
+            
+            return results
+        except:
+            return "Florence-2 analysis failed"
+    
+    def format_results(self, results):
+        """Format results for display"""
+        formatted = "# Chart Analysis Results\n\n"
+        
+        if 'description' in results:
+            formatted += f"## Image Description\n{results['description']}\n\n"
+        
+        if 'chart_type' in results:
+            formatted += f"## Chart Type\n{results['chart_type']}\n\n"
+        
+        if 'extracted_text' in results:
+            formatted += "## Extracted Text\n"
+            for method, text in results['extracted_text'].items():
+                formatted += f"**{method}:** {text}\n\n"
+        
+        if 'data_points' in results:
+            formatted += f"## Data Analysis\n{results['data_points']}\n\n"
+        
+        if 'advanced_analysis' in results:
+            formatted += f"## Advanced Analysis\n{results['advanced_analysis']}\n\n"
+        
+        return formatted
+
+# Initialize the analyzer
+analyzer = ChartAnalyzer()
+
+# Create Gradio interface
+def analyze_uploaded_chart(image, analysis_type):
+    return analyzer.analyze_chart(image, analysis_type)
+
+# Create the Gradio app
+with gr.Blocks(title="Chart Analyzer & Data Extractor", theme=gr.themes.Soft()) as demo:
+    gr.Markdown("# 📊 Chart Analyzer & Data Extractor")
+    gr.Markdown("Upload a chart image to extract data and analyze its contents using multiple AI models including BLIP, TrOCR, and Florence-2.")
+    
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown("## 📁 Upload Your Chart")
+            
+            # Multiple upload options
+            with gr.Tabs():
+                with gr.Tab("📤 Upload Image"):
+                    image_input = gr.Image(
+                        type="pil", 
+                        label="Upload Chart Image",
+                        height=400,
+                        sources=["upload", "webcam", "clipboard"],
+                        format="png"
+                    )
+                    gr.Markdown("**Supported formats:** PNG, JPG, JPEG, GIF, BMP")
+                    gr.Markdown("**Max size:** 10MB")
+                
+                with gr.Tab("🔗 From URL"):
+                    url_input = gr.Textbox(
+                        label="Image URL", 
+                        placeholder="https://example.com/chart.png"
+                    )
+                    load_url_btn = gr.Button("Load from URL")
+            
+            # Analysis options
+            gr.Markdown("## ⚙️ Analysis Settings")
+            analysis_type = gr.Dropdown(
+                choices=["basic", "comprehensive", "data_extraction"],
+                value="comprehensive",
+                label="Analysis Type",
+                info="Choose the depth of analysis"
+            )
+            
+            with gr.Accordion("Advanced Options", open=False):
+                confidence_threshold = gr.Slider(
+                    minimum=0.1,
+                    maximum=1.0,
+                    value=0.5,
+                    label="OCR Confidence Threshold"
+                )
+                use_florence = gr.Checkbox(
+                    label="Use Florence-2 (Advanced Analysis)",
+                    value=True
+                )
+            
+            analyze_btn = gr.Button("🔍 Analyze Chart", variant="primary", size="lg")
+            clear_btn = gr.Button("🗑️ Clear All", variant="secondary")
+        
+        with gr.Column(scale=2):
+            gr.Markdown("## 📊 Analysis Results")
+            output = gr.Markdown(
+                value="Upload an image and click 'Analyze Chart' to see results here.",
+                label="Results"
+            )
+            
+            # Additional output components
+            with gr.Accordion("Raw Data Export", open=False):
+                json_output = gr.JSON(label="Structured Data")
+                csv_download = gr.File(label="Download CSV", visible=False)
+    
+    # Function to load image from URL
+    def load_image_from_url(url):
+        try:
+            import requests
+            response = requests.get(url)
+            response.raise_for_status()
+            image = Image.open(io.BytesIO(response.content))
+            return image, "Image loaded successfully!"
+        except Exception as e:
+            return None, f"Error loading image: {str(e)}"
+    
+    # Enhanced analysis function
+    def analyze_uploaded_chart(image, analysis_type, confidence_threshold, use_florence):
+        if image is None:
+            return "Please upload an image first.", {}, None
+        
+        try:
+            result = analyzer.analyze_chart(image, analysis_type)
+            
+            # Create structured data for JSON output
+            structured_data = {
+                "analysis_type": analysis_type,
+                "confidence_threshold": confidence_threshold,
+                "models_used": ["BLIP", "TrOCR", "EasyOCR"],
+                "timestamp": pd.Timestamp.now().isoformat()
+            }
+            
+            if use_florence and analyzer.florence_available:
+                structured_data["models_used"].append("Florence-2")
+            
+            return result, structured_data, None
+            
+        except Exception as e:
+            error_msg = f"Error analyzing chart: {str(e)}"
+            return error_msg, {"error": error_msg}, None
+    
+    # Clear function
+    def clear_all():
+        return None, "Upload an image and click 'Analyze Chart' to see results here.", {}, None
+    
+    # Examples
+    gr.Examples(
+        examples=[
+            ["https://via.placeholder.com/600x400/0066CC/FFFFFF?text=Sample+Bar+Chart", "comprehensive"],
+            ["https://via.placeholder.com/600x400/FF6B35/FFFFFF?text=Sample+Line+Chart", "data_extraction"],
+        ],
+        inputs=[image_input, analysis_type],
+        label="Try these examples:"
+    )
+    
+    # Event handlers
+    analyze_btn.click(
+        fn=analyze_uploaded_chart,
+        inputs=[image_input, analysis_type, confidence_threshold, use_florence],
+        outputs=[output, json_output, csv_download]
+    )
+    
+    load_url_btn.click(
+        fn=load_image_from_url,
+        inputs=[url_input],
+        outputs=[image_input, output]
+    )
+    
+    clear_btn.click(
+        fn=clear_all,
+        outputs=[image_input, output, json_output, csv_download]
+    )
+
+if __name__ == "__main__":
+    demo.launch()