Update app.py

app.py

@@ -1,82 +1,346 @@
 import streamlit as st
-import requests
 import pdfplumber
+import pandas as pd
+from bs4 import BeautifulSoup
+import re
+import json
+import requests
+import tempfile
 import os
-import fitz  # PyMuPDF
+from typing import List, Dict, Any, Union
+from urllib.parse import urlparse, unquote
+import html
+
+try:
+    import fitz  # PyMuPDF
+    PYMUPDF_AVAILABLE = True
+except ImportError:
+    PYMUPDF_AVAILABLE = False
+    print("PyMuPDF not available, using pdfplumber only")
+
+from lxml import html as lxml_html, etree
+from dataclasses import dataclass, asdict
+
+
+@dataclass
+class PDFElement:
+    """Represents an element extracted from PDF"""
+    type: str
+    content: Any
+    page: int
+    bbox: tuple = None
+    style: Dict = None
+    level: int = None
 
-def download_pdf(pdf_path):
-    """Downloads PDF from URL or returns local path if it exists."""
-    if os.path.isfile(pdf_path):
-        return pdf_path
+
+class PDFProcessor:
+    """Simplified PDF processor"""
     
-    try:
-        response = requests.get(pdf_path)
+    def __init__(self):
+        self.elements = []
+        self.html_content = ""
+        self.element_counter = 0
+        
+    def process_pdf(self, pdf_url: str) -> Dict:
+        """Process PDF from URL"""
+        temp_file = None
+        
+        try:
+            temp_file = self._download_pdf(pdf_url)
+            
+            # Extract content
+            self.elements = self._extract_content(temp_file)
+            self.html_content = self._convert_to_html()
+            
+            # Get summary
+            summary = {
+                'total_elements': len(self.elements),
+                'pages': max([e.page for e in self.elements]) if self.elements else 0,
+                'headings': len([e for e in self.elements if e.type == 'heading']),
+                'tables': len([e for e in self.elements if e.type == 'table']),
+                'paragraphs': len([e for e in self.elements if e.type == 'paragraph'])
+            }
+            
+            return summary
+            
+        finally:
+            if temp_file and os.path.exists(temp_file):
+                try:
+                    os.unlink(temp_file)
+                except:
+                    pass
+    
+    def _download_pdf(self, url: str) -> str:
+        """Download PDF from URL"""
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+        }
+        
+        response = requests.get(url, headers=headers, timeout=30)
         response.raise_for_status()
         
-        pdf_filename = 'downloaded_document.pdf'
-        with open(pdf_filename, 'wb') as pdf_file:
-            pdf_file.write(response.content)
-        return pdf_filename
-    except Exception as e:
-        st.error(f"Error downloading PDF: {e}")
-        return None
-
-def extract_content(pdf_path):
-    """Extracts raw text using pdfplumber and HTML using PyMuPDF."""
-    text_data = ""
-    html_data = ""
+        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.pdf')
+        temp_file.write(response.content)
+        temp_file.close()
+        
+        return temp_file.name
+    
+    def _get_element_id(self, element_type: str) -> str:
+        """Generate unique ID for element"""
+        self.element_counter += 1
+        return f"{element_type}-{self.element_counter}"
     
-    # 1. Extract Raw Text using pdfplumber (keeping existing logic)
-    try:
+    def _extract_content(self, pdf_path: str) -> List[PDFElement]:
+        """Extract structured content from PDF"""
+        elements = []
+        
+        if PYMUPDF_AVAILABLE:
+            try:
+                # Try PyMuPDF first for better structure detection
+                doc = fitz.open(pdf_path)
+                
+                for page_num, page in enumerate(doc, 1):
+                    blocks = page.get_text("dict")
+                    
+                    for block in blocks["blocks"]:
+                        if block["type"] == 0:  # Text block
+                            for line in block["lines"]:
+                                for span in line["spans"]:
+                                    text = span["text"].strip()
+                                    if not text:
+                                        continue
+                                    
+                                    font_size = span["size"]
+                                    
+                                    # Simple classification
+                                    if font_size > 14:
+                                        element_type = "heading"
+                                        level = 1 if font_size > 18 else 2
+                                    elif re.match(r'^[\d\-\•\*]+\.?\s+', text):
+                                        element_type = "list"
+                                        level = None
+                                    else:
+                                        element_type = "paragraph"
+                                        level = None
+                                    
+                                    elements.append(PDFElement(
+                                        type=element_type,
+                                        content=text,
+                                        page=page_num,
+                                        level=level
+                                    ))
+                
+                doc.close()
+                
+                # Also get tables with pdfplumber
+                with pdfplumber.open(pdf_path) as pdf:
+                    for page_num, page in enumerate(pdf.pages, 1):
+                        tables = page.extract_tables()
+                        for table in tables:
+                            if table:
+                                elements.append(PDFElement(
+                                    type="table",
+                                    content=table,
+                                    page=page_num
+                                ))
+                
+                return elements
+                
+            except Exception as e:
+                print(f"PyMuPDF failed: {e}, falling back to pdfplumber")
+        
+        # Fallback to pdfplumber only
         with pdfplumber.open(pdf_path) as pdf:
-            for page in pdf.pages:
-                text_data += (page.extract_text() or "") + "\n"
+            for page_num, page in enumerate(pdf.pages, 1):
+                text = page.extract_text() or ""
+                lines = text.split('\n')
+                
+                for line in lines:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    
+                    if line.isupper() and len(line) < 100:
+                        element_type = "heading"
+                        level = 1
+                    elif re.match(r'^[\d\-\•\*]+\.?\s+', line):
+                        element_type = "list"
+                        level = None
+                    else:
+                        element_type = "paragraph"
+                        level = None
+                    
+                    elements.append(PDFElement(
+                        type=element_type,
+                        content=line,
+                        page=page_num,
+                        level=level
+                    ))
+                
+                # Extract tables
                 tables = page.extract_tables()
                 for table in tables:
-                    for row in table:
-                        # Handle None cells in tables
-                        row_text = "\t".join(str(cell) if cell is not None else "" for cell in row)
-                        text_data += row_text + "\n"
-    except Exception as e:
-        st.error(f"Error extracting text with pdfplumber: {e}")
-
-    # 2. Extract HTML using PyMuPDF (fitz)
-    try:
-        doc = fitz.open(pdf_path)
-        for page in doc:
-            html_data += page.get_text("html")
-        doc.close()
-    except Exception as e:
-        st.error(f"Error extracting HTML with PyMuPDF: {e}")
-
-    return text_data, html_data
-
-vert_space = '<div style="padding: 3rem 1rem;"></div>'
-st.markdown(vert_space, unsafe_allow_html=True)
-st.title("PDF Content Scraper")
-st.write("Extract full text and HTML from PDF URL")
+                    if table:
+                        elements.append(PDFElement(
+                            type="table",
+                            content=table,
+                            page=page_num
+                        ))
+        
+        return elements
+    
+    def _convert_to_html(self) -> str:
+        """Convert elements to HTML with IDs"""
+        html_parts = ['''
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            line-height: 1.6;
+            padding: 20px;
+            max-width: 900px;
+            margin: 0 auto;
+        }
+        h1, h2, h3 { color: #333; margin-top: 20px; }
+        table {
+            border-collapse: collapse;
+            width: 100%;
+            margin: 20px 0;
+        }
+        th, td {
+            border: 1px solid #ddd;
+            padding: 8px;
+            text-align: left;
+        }
+        th { background-color: #f2f2f2; }
+        p { margin: 10px 0; }
+        li { margin: 5px 0; }
+        .page-marker {
+            color: #888;
+            font-size: 0.9em;
+            margin-top: 30px;
+            padding-top: 10px;
+            border-top: 2px solid #eee;
+        }
+    </style>
+</head>
+<body>
+''']
+        
+        current_page = 0
+        
+        for elem in self.elements:
+            # Add page marker
+            if elem.page != current_page:
+                current_page = elem.page
+                html_parts.append(f'<div class="page-marker" id="page-{current_page}">Page {current_page}</div>')
+            
+            if elem.type == "heading":
+                level = elem.level or 2
+                elem_id = self._get_element_id('heading')
+                content = html.escape(elem.content)
+                html_parts.append(f'<h{level} id="{elem_id}" data-page="{elem.page}">{content}</h{level}>')
+            
+            elif elem.type == "paragraph":
+                elem_id = self._get_element_id('paragraph')
+                content = html.escape(elem.content)
+                html_parts.append(f'<p id="{elem_id}" data-page="{elem.page}">{content}</p>')
+            
+            elif elem.type == "list":
+                elem_id = self._get_element_id('list-item')
+                content = html.escape(elem.content)
+                html_parts.append(f'<li id="{elem_id}" data-page="{elem.page}">{content}</li>')
+            
+            elif elem.type == "table":
+                elem_id = self._get_element_id('table')
+                html_parts.append(f'<table id="{elem_id}" data-page="{elem.page}">')
+                for i, row in enumerate(elem.content):
+                    row_id = self._get_element_id('table-row')
+                    html_parts.append(f'<tr id="{row_id}">')
+                    tag = 'th' if i == 0 else 'td'
+                    for j, cell in enumerate(row):
+                        cell_id = self._get_element_id('table-cell')
+                        cell_content = html.escape(str(cell)) if cell else ""
+                        html_parts.append(f'<{tag} id="{cell_id}">{cell_content}</{tag}>')
+                    html_parts.append('</tr>')
+                html_parts.append('</table>')
+        
+        html_parts.append('</body></html>')
+        return '\n'.join(html_parts)
 
-pdfURL = st.text_input(label="PDF URL", value="", placeholder="Enter PDF URL here")
-button = st.button(label='Extract')
 
-if button and pdfURL:
-    with st.spinner("Downloading and extracting..."):
-        local_pdf = download_pdf(pdfURL)
+# Streamlit App
+def main():
+    st.set_page_config(
+        page_title="PDF to HTML Converter",
+        page_icon="📄",
+        layout="wide"
+    )
+    
+    st.title("📄 PDF to HTML Converter")
+    st.markdown("Extract PDF content and view as structured HTML")
+    
+    # Initialize session state
+    if 'processor' not in st.session_state:
+        st.session_state.processor = None
+    if 'html_content' not in st.session_state:
+        st.session_state.html_content = None
+    
+    # Input section
+    pdf_url = st.text_input(
+        "Enter PDF URL",
+        placeholder="https://example.com/document.pdf",
+        help="Enter the URL of the PDF you want to process"
+    )
+    
+    if st.button("Process PDF", type="primary"):
+        if not pdf_url:
+            st.error("Please enter a PDF URL")
+        else:
+            with st.spinner("Processing PDF..."):
+                try:
+                    processor = PDFProcessor()
+                    summary = processor.process_pdf(pdf_url)
+                    
+                    st.session_state.processor = processor
+                    st.session_state.html_content = processor.html_content
+                    
+                    st.success(f"✅ PDF processed successfully! ({summary['total_elements']} elements extracted)")
+                    
+                except Exception as e:
+                    st.error(f"❌ Error processing PDF: {str(e)}")
+    
+    # Display HTML in iframe
+    if st.session_state.html_content:
+        st.markdown("---")
+        st.subheader("📋 Extracted HTML Content")
         
-        if local_pdf:
-            text, html = extract_content(local_pdf)
+        # Create tabs for different views
+        tab1, tab2 = st.tabs(["HTML Preview", "HTML Source"])
+        
+        with tab1:
+            # Display in iframe
+            st.components.v1.html(
+                st.session_state.html_content,
+                height=800,
+                scrolling=True
+            )
+        
+        with tab2:
+            # Show source code
+            st.code(st.session_state.html_content, language='html')
             
-            # Clean up downloaded file if it was downloaded
-            if local_pdf == 'downloaded_document.pdf' and os.path.exists(local_pdf):
-                os.remove(local_pdf)
+            # Download button
+            st.download_button(
+                label="📥 Download HTML",
+                data=st.session_state.html_content,
+                file_name="extracted_content.html",
+                mime="text/html"
+            )
 
-            st.subheader("Raw Text Content")
-            st.text_area("Extracted Text", text, height=300)
-            
-            st.subheader("HTML Content")
-            st.write("Rendered HTML Preview:")
-            import streamlit.components.v1 as components
-            components.html(html, height=600, scrolling=True)
-            
-            with st.expander("View HTML Source"):
-                st.code(html, language='html')
+
+if __name__ == "__main__":
+    main()