Create app_minimal.py

app_minimal.py

@@ -0,0 +1,514 @@
+"""
+AI Dataset Studio - Minimal Version
+Guaranteed to work with basic dependencies only
+"""
+
+import gradio as gr
+import pandas as pd
+import json
+import re
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse
+from datetime import datetime
+import logging
+from typing import Dict, List, Tuple, Optional, Any
+from dataclasses import dataclass, asdict
+import uuid
+import time
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+@dataclass
+class SimpleScrapedItem:
+    """Simplified scraped content structure"""
+    id: str
+    url: str
+    title: str
+    content: str
+    word_count: int
+    scraped_at: str
+    quality_score: float = 0.0
+
+class SimpleWebScraper:
+    """Simplified web scraper with basic functionality"""
+    
+    def __init__(self):
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (compatible; AI-DatasetStudio/1.0)',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+        })
+    
+    def scrape_url(self, url: str) -> Optional[SimpleScrapedItem]:
+        """Scrape a single URL"""
+        try:
+            if not self._validate_url(url):
+                return None
+            
+            response = self.session.get(url, timeout=10)
+            response.raise_for_status()
+            
+            soup = BeautifulSoup(response.content, 'html.parser')
+            
+            # Extract title
+            title_tag = soup.find('title')
+            title = title_tag.get_text().strip() if title_tag else "Untitled"
+            
+            # Extract content
+            # Remove unwanted elements
+            for element in soup(['script', 'style', 'nav', 'header', 'footer']):
+                element.decompose()
+            
+            # Try to find main content
+            content_element = (soup.find('article') or 
+                             soup.find('main') or 
+                             soup.find(class_='content') or
+                             soup.find('body'))
+            
+            if content_element:
+                content = content_element.get_text(separator=' ', strip=True)
+            else:
+                content = soup.get_text(separator=' ', strip=True)
+            
+            # Clean content
+            content = re.sub(r'\s+', ' ', content).strip()
+            
+            # Calculate basic metrics
+            word_count = len(content.split())
+            quality_score = min(1.0, word_count / 100) if word_count > 0 else 0.0
+            
+            return SimpleScrapedItem(
+                id=str(uuid.uuid4()),
+                url=url,
+                title=title,
+                content=content,
+                word_count=word_count,
+                scraped_at=datetime.now().isoformat(),
+                quality_score=quality_score
+            )
+            
+        except Exception as e:
+            logger.error(f"Failed to scrape {url}: {e}")
+            return None
+    
+    def _validate_url(self, url: str) -> bool:
+        """Basic URL validation"""
+        try:
+            parsed = urlparse(url)
+            return parsed.scheme in ['http', 'https'] and parsed.netloc
+        except:
+            return False
+    
+    def batch_scrape(self, urls: List[str], progress_callback=None) -> List[SimpleScrapedItem]:
+        """Scrape multiple URLs"""
+        results = []
+        total = len(urls)
+        
+        for i, url in enumerate(urls):
+            if progress_callback:
+                progress_callback((i + 1) / total, f"Scraping {i+1}/{total}")
+            
+            item = self.scrape_url(url)
+            if item:
+                results.append(item)
+            
+            time.sleep(1)  # Rate limiting
+        
+        return results
+
+class SimpleDataProcessor:
+    """Basic data processing"""
+    
+    def process_items(self, items: List[SimpleScrapedItem], options: Dict[str, bool]) -> List[SimpleScrapedItem]:
+        """Process scraped items"""
+        processed = []
+        
+        for item in items:
+            # Apply quality filter
+            if options.get('quality_filter', True) and item.quality_score < 0.3:
+                continue
+            
+            # Clean text if requested
+            if options.get('clean_text', True):
+                item.content = self._clean_text(item.content)
+            
+            processed.append(item)
+        
+        return processed
+    
+    def _clean_text(self, text: str) -> str:
+        """Basic text cleaning"""
+        # Remove URLs
+        text = re.sub(r'http\S+', '', text)
+        # Remove extra whitespace
+        text = re.sub(r'\s+', ' ', text)
+        # Remove common navigation text
+        text = re.sub(r'(Click here|Read more|Subscribe|Advertisement)', '', text, flags=re.IGNORECASE)
+        return text.strip()
+
+class SimpleExporter:
+    """Basic export functionality"""
+    
+    def export_dataset(self, items: List[SimpleScrapedItem], format_type: str) -> str:
+        """Export dataset"""
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        
+        if format_type == "json":
+            filename = f"dataset_{timestamp}.json"
+            data = [asdict(item) for item in items]
+            with open(filename, 'w', encoding='utf-8') as f:
+                json.dump(data, f, indent=2, ensure_ascii=False)
+            return filename
+        
+        elif format_type == "csv":
+            filename = f"dataset_{timestamp}.csv"
+            data = [asdict(item) for item in items]
+            df = pd.DataFrame(data)
+            df.to_csv(filename, index=False)
+            return filename
+        
+        else:
+            raise ValueError(f"Unsupported format: {format_type}")
+
+class SimpleDatasetStudio:
+    """Simplified main application"""
+    
+    def __init__(self):
+        self.scraper = SimpleWebScraper()
+        self.processor = SimpleDataProcessor()
+        self.exporter = SimpleExporter()
+        
+        self.scraped_items = []
+        self.processed_items = []
+        self.current_project = None
+    
+    def create_project(self, name: str) -> Dict[str, Any]:
+        """Create a new project"""
+        self.current_project = {
+            'name': name,
+            'id': str(uuid.uuid4()),
+            'created_at': datetime.now().isoformat()
+        }
+        self.scraped_items = []
+        self.processed_items = []
+        return self.current_project
+    
+    def scrape_urls(self, urls: List[str], progress_callback=None) -> Tuple[int, List[str]]:
+        """Scrape URLs"""
+        url_list = [url.strip() for url in urls if url.strip()]
+        if not url_list:
+            return 0, ["No valid URLs provided"]
+        
+        self.scraped_items = self.scraper.batch_scrape(url_list, progress_callback)
+        success_count = len(self.scraped_items)
+        failed_count = len(url_list) - success_count
+        
+        errors = []
+        if failed_count > 0:
+            errors.append(f"{failed_count} URLs failed")
+        
+        return success_count, errors
+    
+    def process_data(self, options: Dict[str, bool]) -> int:
+        """Process scraped data"""
+        if not self.scraped_items:
+            return 0
+        
+        self.processed_items = self.processor.process_items(self.scraped_items, options)
+        return len(self.processed_items)
+    
+    def get_preview(self) -> List[Dict[str, Any]]:
+        """Get data preview"""
+        items = self.processed_items or self.scraped_items
+        preview = []
+        
+        for item in items[:5]:
+            preview.append({
+                'Title': item.title[:50] + "..." if len(item.title) > 50 else item.title,
+                'Content Preview': item.content[:100] + "..." if len(item.content) > 100 else item.content,
+                'Word Count': item.word_count,
+                'Quality Score': round(item.quality_score, 2),
+                'URL': item.url[:50] + "..." if len(item.url) > 50 else item.url
+            })
+        
+        return preview
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get dataset statistics"""
+        items = self.processed_items or self.scraped_items
+        if not items:
+            return {}
+        
+        word_counts = [item.word_count for item in items]
+        quality_scores = [item.quality_score for item in items]
+        
+        return {
+            'total_items': len(items),
+            'avg_word_count': round(sum(word_counts) / len(word_counts)),
+            'avg_quality': round(sum(quality_scores) / len(quality_scores), 2),
+            'min_words': min(word_counts),
+            'max_words': max(word_counts)
+        }
+    
+    def export_data(self, format_type: str) -> str:
+        """Export dataset"""
+        items = self.processed_items or self.scraped_items
+        if not items:
+            raise ValueError("No data to export")
+        
+        return self.exporter.export_dataset(items, format_type)
+
+def create_simple_interface():
+    """Create simplified Gradio interface"""
+    
+    studio = SimpleDatasetStudio()
+    
+    # Custom CSS
+    css = """
+    .container { max-width: 1200px; margin: auto; }
+    .header { 
+        background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+        color: white; padding: 2rem; border-radius: 10px; 
+        text-align: center; margin-bottom: 2rem;
+    }
+    .step-box {
+        background: #f8f9ff; border: 1px solid #e1e5ff;
+        border-radius: 8px; padding: 1.5rem; margin: 1rem 0;
+    }
+    """
+    
+    with gr.Blocks(css=css, title="AI Dataset Studio - Simple") as interface:
+        
+        # Header
+        gr.HTML("""
+        <div class="header">
+            <h1>🚀 AI Dataset Studio - Simple Version</h1>
+            <p>Create datasets from web content - No complex setup required!</p>
+        </div>
+        """)
+        
+        # Project state
+        project_state = gr.State({})
+        
+        with gr.Tabs():
+            
+            # Project Setup
+            with gr.Tab("📋 Project Setup"):
+                gr.HTML('<div class="step-box"><h3>Step 1: Create Your Project</h3></div>')
+                
+                project_name = gr.Textbox(
+                    label="Project Name",
+                    placeholder="e.g., News Articles Dataset",
+                    value="My Dataset"
+                )
+                
+                create_btn = gr.Button("Create Project", variant="primary")
+                project_status = gr.Markdown("")
+                
+                def create_project_handler(name):
+                    if not name.strip():
+                        return "❌ Please enter a project name", {}
+                    
+                    project = studio.create_project(name.strip())
+                    status = f"""
+                    ✅ **Project Created!**
+                    
+                    **Name:** {project['name']}  
+                    **ID:** {project['id'][:8]}...  
+                    **Created:** {project['created_at'][:19]}
+                    
+                    👉 Next: Go to Data Collection tab
+                    """
+                    return status, project
+                
+                create_btn.click(
+                    fn=create_project_handler,
+                    inputs=[project_name],
+                    outputs=[project_status, project_state]
+                )
+            
+            # Data Collection
+            with gr.Tab("🕷️ Data Collection"):
+                gr.HTML('<div class="step-box"><h3>Step 2: Scrape Web Content</h3></div>')
+                
+                urls_input = gr.Textbox(
+                    label="URLs to Scrape (one per line)",
+                    placeholder="https://example.com/article1\nhttps://example.com/article2",
+                    lines=6
+                )
+                
+                scrape_btn = gr.Button("Start Scraping", variant="primary")
+                scrape_status = gr.Markdown("")
+                
+                def scrape_handler(urls_text, project, progress=gr.Progress()):
+                    if not project:
+                        return "❌ Create a project first"
+                    
+                    urls = [url.strip() for url in urls_text.split('\n') if url.strip()]
+                    if not urls:
+                        return "❌ No URLs provided"
+                    
+                    def progress_callback(pct, msg):
+                        progress(pct, desc=msg)
+                    
+                    success_count, errors = studio.scrape_urls(urls, progress_callback)
+                    
+                    if success_count > 0:
+                        return f"""
+                        ✅ **Scraping Complete!**
+                        
+                        **Success:** {success_count} URLs  
+                        **Failed:** {len(urls) - success_count} URLs  
+                        
+                        👉 Next: Go to Data Processing tab
+                        """
+                    else:
+                        return f"❌ Scraping failed: {', '.join(errors)}"
+                
+                scrape_btn.click(
+                    fn=scrape_handler,
+                    inputs=[urls_input, project_state],
+                    outputs=[scrape_status]
+                )
+            
+            # Data Processing
+            with gr.Tab("⚙️ Data Processing"):
+                gr.HTML('<div class="step-box"><h3>Step 3: Clean and Process Data</h3></div>')
+                
+                with gr.Row():
+                    clean_text = gr.Checkbox(label="Clean Text", value=True)
+                    quality_filter = gr.Checkbox(label="Quality Filter", value=True)
+                
+                process_btn = gr.Button("Process Data", variant="primary")
+                process_status = gr.Markdown("")
+                
+                def process_handler(clean, quality, project):
+                    if not project:
+                        return "❌ Create a project first"
+                    
+                    options = {
+                        'clean_text': clean,
+                        'quality_filter': quality
+                    }
+                    
+                    processed_count = studio.process_data(options)
+                    
+                    if processed_count > 0:
+                        return f"""
+                        ✅ **Processing Complete!**
+                        
+                        **Processed:** {processed_count} items  
+                        
+                        👉 Next: Check Data Preview tab
+                        """
+                    else:
+                        return "❌ No items passed processing filters"
+                
+                process_btn.click(
+                    fn=process_handler,
+                    inputs=[clean_text, quality_filter, project_state],
+                    outputs=[process_status]
+                )
+            
+            # Data Preview
+            with gr.Tab("👀 Data Preview"):
+                gr.HTML('<div class="step-box"><h3>Step 4: Review Your Dataset</h3></div>')
+                
+                refresh_btn = gr.Button("Refresh Preview")
+                preview_table = gr.DataFrame(label="Dataset Preview")
+                stats_display = gr.JSON(label="Statistics")
+                
+                def refresh_handler(project):
+                    if not project:
+                        return None, {}
+                    
+                    preview = studio.get_preview()
+                    stats = studio.get_stats()
+                    return preview, stats
+                
+                refresh_btn.click(
+                    fn=refresh_handler,
+                    inputs=[project_state],
+                    outputs=[preview_table, stats_display]
+                )
+            
+            # Export
+            with gr.Tab("📤 Export Dataset"):
+                gr.HTML('<div class="step-box"><h3>Step 5: Export Your Dataset</h3></div>')
+                
+                export_format = gr.Radio(
+                    choices=["JSON", "CSV"],
+                    label="Export Format",
+                    value="JSON"
+                )
+                
+                export_btn = gr.Button("Export Dataset", variant="primary")
+                export_status = gr.Markdown("")
+                export_file = gr.File(label="Download", visible=False)
+                
+                def export_handler(format_type, project):
+                    if not project:
+                        return "❌ Create a project first", None
+                    
+                    try:
+                        filename = studio.export_data(format_type.lower())
+                        return f"✅ Export successful! File: {filename}", filename
+                    except Exception as e:
+                        return f"❌ Export failed: {str(e)}", None
+                
+                export_btn.click(
+                    fn=export_handler,
+                    inputs=[export_format, project_state],
+                    outputs=[export_status, export_file]
+                )
+        
+        # Instructions
+        with gr.Accordion("📚 Quick Guide", open=False):
+            gr.Markdown("""
+            ## How to Use
+            
+            1. **Create Project** - Give your dataset a name
+            2. **Add URLs** - Paste URLs of web pages to scrape
+            3. **Process Data** - Clean and filter the content
+            4. **Review** - Check the quality of your dataset
+            5. **Export** - Download in JSON or CSV format
+            
+            ## Features
+            - ✅ Smart content extraction
+            - ✅ Quality filtering
+            - ✅ Text cleaning
+            - ✅ JSON/CSV export
+            - ✅ Preview and statistics
+            
+            ## Tips
+            - Use high-quality source URLs
+            - Enable quality filtering for better results
+            - Review your data before exporting
+            - Start with 5-10 URLs to test
+            """)
+    
+    return interface
+
+# Launch application
+if __name__ == "__main__":
+    logger.info("🚀 Starting AI Dataset Studio (Simple Version)")
+    
+    try:
+        interface = create_simple_interface()
+        logger.info("✅ Simple interface created successfully")
+        
+        interface.launch(
+            server_name="0.0.0.0",
+            server_port=7860,
+            share=False,
+            show_error=True
+        )
+        
+    except Exception as e:
+        logger.error(f"❌ Failed to launch: {e}")
+        print("\n💡 If you see import errors, try installing:")
+        print("pip install gradio pandas requests beautifulsoup4")
+        raise