Upload app_simplified.py with huggingface_hub

app_simplified.py

@@ -698,43 +698,84 @@ with col2:
         
         # Execute the one-click data collection
         with st.spinner("🔄 Collecting data from all APIs..."):
-            results = st.session_state.harvester.fetch_all_apis(update_progress)
-            st.session_state.last_results = results
+            try:
+                results = st.session_state.harvester.fetch_all_apis(update_progress)
+                st.session_state.last_results = results
+                
+                progress_bar.progress(1.0)
+                status_container.success("✅ Collection completed!")
+                
+            except Exception as e:
+                progress_bar.progress(0.0)
+                status_container.error(f"❌ Collection failed: {str(e)}")
+                st.error("Data collection encountered an error. Please try again.")
+                st.stop()  # Stop execution here if collection failed
         
-        progress_bar.progress(1.0)
-        status_container.text("✅ Collection completed!")
+        # Show results only if collection was successful
+        if 'results' not in locals() or not results:
+            st.error("No results to display")
+            st.stop()
         
-        # Show results
-        summary = results['summary']
+        summary = results.get('summary', {})
         
         # Success metrics
         col1, col2, col3, col4 = st.columns(4)
         
         with col1:
-            st.metric("✅ Successful APIs", summary['successful'])
+            st.metric("✅ Successful APIs", summary.get('successful', 0))
         
         with col2:
-            st.metric("❌ Failed APIs", summary['failed'])
+            st.metric("❌ Failed APIs", summary.get('failed', 0))
         
         with col3:
-            st.metric("📊 Success Rate", f"{summary['success_rate']:.1f}%")
+            st.metric("📊 Success Rate", f"{summary.get('success_rate', 0):.1f}%")
         
         with col4:
-            total_records = sum(api_data['total_records'] for api_data in results['results'].values())
-            st.metric("📄 Total Records", f"{total_records:,}")
+            try:
+                total_records = sum(api_data.get('total_records', 0) for api_data in results.get('results', {}).values())
+                st.metric("📄 Total Records", f"{total_records:,}")
+            except Exception:
+                st.metric("📄 Total Records", "Error")
         
-        # Detailed results
+        # Detailed results with progress indicator
         st.markdown("### 📋 Detailed Results")
         
-        for api_name, api_data in results['results'].items():
-            with st.expander(f"✅ {SIMPLIFIED_API_CONFIG[api_name]['name']} - {api_data['total_records']} records"):
-                for endpoint in api_data['endpoints']:
-                    st.write(f"**URL:** {endpoint['endpoint_url']}")
-                    st.write(f"**Records:** {endpoint['records']} | **Duration:** {endpoint['duration_ms']}ms | **Size:** {endpoint['size_bytes']} bytes")
-                    
-                    if endpoint.get('data_preview'):
-                        st.write("**Data Preview:**")
-                        st.code(endpoint['data_preview'], language="json")
+        if len(results['results']) > 5:
+            with st.spinner("Loading detailed results..."):
+                time.sleep(0.1)  # Small delay to show spinner
+        
+        try:
+            # Limit to first 8 APIs to prevent UI overload
+            apis_to_show = list(results['results'].items())[:8]
+            
+            for api_name, api_data in apis_to_show:
+                with st.expander(f"✅ {SIMPLIFIED_API_CONFIG[api_name]['name']} - {api_data['total_records']} records"):
+                    for endpoint in api_data['endpoints']:
+                        col1, col2 = st.columns([3, 1])
+                        
+                        with col1:
+                            st.write(f"**URL:** {endpoint['endpoint_url']}")
+                            st.write(f"**Records:** {endpoint['records']} | **Duration:** {endpoint['duration_ms']}ms | **Size:** {endpoint['size_bytes']} bytes")
+                        
+                        with col2:
+                            if endpoint.get('status') == 'success':
+                                st.success("✅ Success")
+                            else:
+                                st.error("❌ Failed")
+                        
+                        # Show a very limited preview
+                        if endpoint.get('data_preview') and len(str(endpoint.get('data_preview', ''))) > 10:
+                            with st.container():
+                                st.write("**Sample Data:**")
+                                preview_text = str(endpoint.get('data_preview', ''))[:100]
+                                st.code(preview_text + "..." if len(preview_text) == 100 else preview_text)
+            
+            if len(results['results']) > 8:
+                st.info(f"Showing first 8 APIs. Total: {len(results['results'])} APIs processed successfully.")
+                
+        except Exception as e:
+            st.error(f"Error displaying results: {str(e)}")
+            st.info("✅ Data collection completed successfully! Check the database viewer below for stored data.")
         
         # Error details
         if results['errors']:
@@ -749,29 +790,57 @@ if st.session_state.last_results:
     
     results = st.session_state.last_results
     
-    # Create simple visualizations
-    if results['results']:
-        # API success chart
-        api_names = []
-        record_counts = []
-        
-        for api_name, api_data in results['results'].items():
-            api_names.append(SIMPLIFIED_API_CONFIG[api_name]['name'])
-            record_counts.append(api_data['total_records'])
-        
-        if record_counts and any(count > 0 for count in record_counts):
-            fig = px.bar(
-                x=api_names,
-                y=record_counts,
-                title="📊 Records Collected by API",
-                labels={'x': 'API', 'y': 'Records'}
-            )
-            fig.update_layout(
-                paper_bgcolor="rgba(255,255,255,0.9)",
-                plot_bgcolor="rgba(255,255,255,0.9)",
-                font_color="#2c3e50"
-            )
-            st.plotly_chart(fig, use_container_width=True)
+    # Create simple visualizations with error handling
+    try:
+        if results.get('results') and len(results['results']) > 0:
+            # Simple metrics display instead of complex charts
+            col1, col2, col3 = st.columns(3)
+            
+            with col1:
+                total_apis = len(results['results'])
+                st.metric("🌍 APIs Processed", total_apis)
+            
+            with col2:
+                total_records = sum(api_data.get('total_records', 0) for api_data in results['results'].values())
+                st.metric("📄 Total Records", f"{total_records:,}")
+            
+            with col3:
+                total_size = sum(api_data.get('total_size', 0) for api_data in results['results'].values())
+                size_mb = total_size / (1024 * 1024)
+                st.metric("💾 Data Size", f"{size_mb:.2f} MB")
+            
+            # Simple bar chart with limited data
+            if len(results['results']) <= 10:  # Only show chart for reasonable number of APIs
+                api_names = []
+                record_counts = []
+                
+                for api_name, api_data in list(results['results'].items())[:8]:  # Limit to 8 APIs
+                    api_names.append(SIMPLIFIED_API_CONFIG[api_name]['name'][:20])  # Truncate names
+                    record_counts.append(api_data.get('total_records', 0))
+                
+                if record_counts and any(count > 0 for count in record_counts):
+                    try:
+                        fig = px.bar(
+                            x=api_names,
+                            y=record_counts,
+                            title="📊 Records by API (Top 8)",
+                            labels={'x': 'API', 'y': 'Records'}
+                        )
+                        fig.update_layout(
+                            paper_bgcolor="rgba(255,255,255,0.9)",
+                            plot_bgcolor="rgba(255,255,255,0.9)",
+                            font_color="#2c3e50",
+                            height=400
+                        )
+                        st.plotly_chart(fig, use_container_width=True)
+                    except Exception as chart_error:
+                        st.info("Chart generation skipped due to data size")
+            else:
+                st.info("📊 Chart display skipped - too many APIs for optimal viewing")
+                
+    except Exception as e:
+        st.warning("Analytics display skipped due to data processing complexity")
+        st.info("✅ Data collection was successful - check database viewer below")
 
 # Enhanced Database viewer
 with st.expander("🗄️ Enhanced Database Viewer"):