Upload app_ultimate.py with huggingface_hub

app_ultimate.py

@@ -26,6 +26,9 @@ from sklearn.metrics.pairwise import cosine_similarity
 import warnings
 warnings.filterwarnings('ignore')
 
+# Global ML availability flag
+ML_AVAILABLE = False
+
 # AI/ML Imports for enhanced functionality
 try:
     from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
@@ -33,7 +36,6 @@ try:
     ML_AVAILABLE = True
 except ImportError:
     ML_AVAILABLE = False
-    st.warning("⚠️ ML libraries not available. Some AI features will be disabled.")
 
 # Enhanced Page Configuration
 st.set_page_config(
@@ -43,99 +45,134 @@ st.set_page_config(
     initial_sidebar_state="collapsed"
 )
 
-# Enhanced CSS with modern animations
+# Enhanced CSS with modern, professional styling
 st.markdown("""
 <style>
     .main > div {
         padding-top: 1rem;
     }
     .stApp {
-        background: linear-gradient(135deg, #667eea 0%, #764ba2 50%, #f093fb 100%);
-        color: white;
+        background: linear-gradient(135deg, #f5f7fa 0%, #c3cfe2 100%);
+        color: #2c3e50;
     }
     .metric-card {
-        background: rgba(255, 255, 255, 0.15);
-        backdrop-filter: blur(15px);
-        border-radius: 20px;
+        background: rgba(255, 255, 255, 0.95);
+        border-radius: 12px;
         padding: 1.5rem;
         margin: 0.5rem 0;
-        border: 1px solid rgba(255, 255, 255, 0.3);
-        box-shadow: 0 8px 32px 0 rgba(31, 38, 135, 0.37);
+        border: 1px solid rgba(52, 73, 94, 0.1);
+        box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
         transition: all 0.3s ease;
     }
     .metric-card:hover {
-        transform: translateY(-3px);
-        box-shadow: 0 12px 40px 0 rgba(31, 38, 135, 0.6);
+        transform: translateY(-2px);
+        box-shadow: 0 8px 25px rgba(0, 0, 0, 0.15);
     }
     .api-card {
-        background: rgba(255, 255, 255, 0.12);
-        backdrop-filter: blur(10px);
-        border-radius: 15px;
+        background: rgba(255, 255, 255, 0.9);
+        border-radius: 10px;
         padding: 1.2rem;
         margin: 0.5rem;
-        border: 1px solid rgba(255, 255, 255, 0.25);
-        transition: all 0.4s cubic-bezier(0.4, 0, 0.2, 1);
+        border: 1px solid rgba(52, 73, 94, 0.15);
+        transition: all 0.3s ease;
         position: relative;
         overflow: hidden;
     }
-    .api-card::before {
-        content: '';
-        position: absolute;
-        top: 0;
-        left: -100%;
-        width: 100%;
-        height: 100%;
-        background: linear-gradient(90deg, transparent, rgba(255,255,255,0.1), transparent);
-        transition: left 0.5s;
-    }
-    .api-card:hover::before {
-        left: 100%;
-    }
     .api-card:hover {
-        transform: translateY(-5px) scale(1.02);
-        box-shadow: 0 15px 45px 0 rgba(31, 38, 135, 0.6);
-        border-color: rgba(255, 255, 255, 0.4);
+        transform: translateY(-3px);
+        box-shadow: 0 8px 20px rgba(0, 0, 0, 0.12);
+        border-color: #3498db;
     }
     .title-container {
         text-align: center;
         padding: 2rem 0;
-        background: rgba(255, 255, 255, 0.08);
-        border-radius: 25px;
+        background: rgba(255, 255, 255, 0.9);
+        border-radius: 15px;
         margin-bottom: 2rem;
-        backdrop-filter: blur(20px);
-        border: 1px solid rgba(255, 255, 255, 0.2);
+        border: 1px solid rgba(52, 73, 94, 0.1);
+        box-shadow: 0 4px 12px rgba(0, 0, 0, 0.05);
     }
     .status-indicator {
-        width: 12px;
-        height: 12px;
+        width: 10px;
+        height: 10px;
         border-radius: 50%;
         display: inline-block;
         margin-right: 8px;
-        animation: pulse 2s infinite;
     }
-    @keyframes pulse {
-        0% { opacity: 1; }
-        50% { opacity: 0.5; }
-        100% { opacity: 1; }
+    .status-active { background-color: #27ae60; }
+    .status-discovering { background-color: #f39c12; }
+    .status-error { background-color: #e74c3c; }
+    .status-paused { background-color: #95a5a6; }
+    
+    .ai-panel {
+        background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+        border-radius: 10px;
+        padding: 1rem;
+        margin: 1rem 0;
+        color: white;
+        border: none;
     }
-    .status-active { background-color: #4ade80; }
-    .status-discovering { background-color: #fbbf24; }
-    .status-error { background-color: #ef4444; }
-    .status-paused { background-color: #6b7280; }
     .discovery-progress {
-        background: rgba(255, 255, 255, 0.1);
-        border-radius: 10px;
+        background: rgba(255, 255, 255, 0.95);
+        border-radius: 8px;
         padding: 1rem;
         margin: 1rem 0;
-        border: 1px solid rgba(255, 255, 255, 0.2);
+        border: 1px solid rgba(52, 73, 94, 0.1);
+        box-shadow: 0 2px 8px rgba(0, 0, 0, 0.05);
     }
     .endpoint-item {
-        background: rgba(255, 255, 255, 0.05);
-        border-radius: 8px;
+        background: rgba(255, 255, 255, 0.8);
+        border-radius: 6px;
         padding: 0.5rem;
         margin: 0.3rem 0;
-        border-left: 3px solid #4ade80;
+        border-left: 3px solid #3498db;
         font-size: 0.9rem;
+        color: #34495e;
+    }
+    
+    /* Custom button styling */
+    .stButton > button {
+        background: linear-gradient(135deg, #3498db, #2980b9);
+        color: white;
+        border: none;
+        border-radius: 8px;
+        padding: 0.5rem 1rem;
+        font-weight: 500;
+        transition: all 0.3s ease;
+    }
+    .stButton > button:hover {
+        background: linear-gradient(135deg, #2980b9, #1f4e79);
+        transform: translateY(-1px);
+        box-shadow: 0 4px 12px rgba(52, 152, 219, 0.3);
+    }
+    
+    /* Tab styling */
+    .stTabs [data-baseweb="tab-list"] {
+        gap: 8px;
+    }
+    .stTabs [data-baseweb="tab"] {
+        background-color: rgba(255, 255, 255, 0.7);
+        border-radius: 8px;
+        color: #2c3e50;
+        font-weight: 500;
+    }
+    .stTabs [aria-selected="true"] {
+        background-color: #3498db;
+        color: white;
+    }
+    
+    /* Metrics styling */
+    [data-testid="metric-container"] {
+        background: rgba(255, 255, 255, 0.9);
+        border: 1px solid rgba(52, 73, 94, 0.1);
+        padding: 1rem;
+        border-radius: 8px;
+        box-shadow: 0 2px 8px rgba(0, 0, 0, 0.05);
+    }
+    
+    /* Sidebar styling */
+    .css-1d391kg {
+        background: linear-gradient(135deg, #ecf0f1 0%, #bdc3c7 100%);
     }
 </style>
 """, unsafe_allow_html=True)
@@ -156,6 +193,7 @@ class AIDataQualityAssessor:
     
     def _initialize_models(self):
         """Initialize AI models for quality assessment"""
+        global ML_AVAILABLE
         if ML_AVAILABLE:
             try:
                 # Initialize quality classifier
@@ -168,10 +206,10 @@ class AIDataQualityAssessor:
                 # Initialize embeddings model for similarity
                 self.embeddings_model = SentenceTransformer('all-MiniLM-L6-v2')
                 
-                st.success("✅ AI models loaded successfully!")
             except Exception as e:
-                st.warning(f"⚠️ Failed to load AI models: {e}")
                 ML_AVAILABLE = False
+                self.quality_model = None
+                self.embeddings_model = None
     
     def assess_data_quality(self, data: Any, api_name: str) -> Dict:
         """Comprehensive AI-powered data quality assessment"""
@@ -210,7 +248,6 @@ class AIDataQualityAssessor:
             }
         
         except Exception as e:
-            st.warning(f"AI quality assessment failed: {e}")
             return self._basic_quality_assessment(data, api_name)
     
     def _data_to_text(self, data: Any) -> str:
@@ -362,11 +399,13 @@ class SemanticDataAnalyzer:
     
     def _initialize_model(self):
         """Initialize sentence transformer model"""
+        global ML_AVAILABLE
         if ML_AVAILABLE:
             try:
                 self.embeddings_model = SentenceTransformer('all-MiniLM-L6-v2')
             except Exception as e:
-                st.warning(f"Failed to load embeddings model: {e}")
+                ML_AVAILABLE = False
+                self.embeddings_model = None
     
     def find_similar_datasets(self, new_data: Any, api_name: str, threshold: float = 0.85) -> List[Dict]:
         """Find semantically similar datasets"""
@@ -396,7 +435,6 @@ class SemanticDataAnalyzer:
             return sorted(similar_datasets, key=lambda x: x['similarity'], reverse=True)
         
         except Exception as e:
-            st.warning(f"Semantic analysis failed: {e}")
             return []
     
     def _data_to_text(self, data: Any) -> str:
@@ -509,7 +547,7 @@ class APIHealthMonitor:
                 return float(anomaly_score)
         
         except Exception as e:
-            st.warning(f"Anomaly detection failed: {e}")
+            pass  # Silent fail for anomaly detection
         
         return 0.0
     
@@ -1702,20 +1740,29 @@ if 'last_session_info' not in st.session_state:
 # Enhanced Header
 st.markdown("""
 <div class="title-container">
-    <h1 style="font-size: 3.5rem; margin: 0; background: linear-gradient(45deg, #fff, #f0f8ff, #e6f3ff); -webkit-background-clip: text; -webkit-text-fill-color: transparent;">
+    <h1 style="font-size: 2.5rem; margin: 0; color: #2c3e50;">
         🚀 Ultimate Data Harvester
     </h1>
-    <p style="font-size: 1.3rem; margin: 0.5rem 0 0 0; opacity: 0.9;">
-        Deep endpoint discovery • Session resumption • Intelligent storage
+    <p style="font-size: 1.1rem; margin: 0.5rem 0 0 0; color: #34495e;">
+        AI-Enhanced Deep Discovery • Session Resumption • Intelligent Storage
+    </p>
+    <p style="font-size: 0.95rem; margin: 0.3rem 0 0 0; color: #7f8c8d;">
+        Comprehensive data collection from 10 international APIs with advanced analytics
     </p>
     <div style="margin-top: 1rem;">
-        <span style="background: rgba(255,255,255,0.25); padding: 0.3rem 0.8rem; border-radius: 15px; margin: 0 0.3rem; font-size: 0.9rem;">Recursive Discovery</span>
-        <span style="background: rgba(255,255,255,0.25); padding: 0.3rem 0.8rem; border-radius: 15px; margin: 0 0.3rem; font-size: 0.9rem;">Auto-Resume</span>
-        <span style="background: rgba(255,255,255,0.25); padding: 0.3rem 0.8rem; border-radius: 15px; margin: 0 0.3rem; font-size: 0.9rem;">Smart Storage</span>
+        <span style="background: #ecf0f1; color: #2c3e50; padding: 0.3rem 0.8rem; border-radius: 15px; margin: 0 0.3rem; font-size: 0.9rem;">🔍 Recursive Discovery</span>
+        <span style="background: #ecf0f1; color: #2c3e50; padding: 0.3rem 0.8rem; border-radius: 15px; margin: 0 0.3rem; font-size: 0.9rem;">🎯 Auto-Resume</span>
+        <span style="background: #ecf0f1; color: #2c3e50; padding: 0.3rem 0.8rem; border-radius: 15px; margin: 0 0.3rem; font-size: 0.9rem;">💾 Smart Storage</span>
     </div>
 </div>
 """, unsafe_allow_html=True)
 
+# Display ML status prominently
+if ML_AVAILABLE:
+    st.success("🤖 **AI Enhanced Mode Active** - Advanced quality assessment and semantic analysis enabled")
+else:
+    st.info("📊 **Standard Mode** - Basic functionality available. Install transformers and sentence-transformers for AI features.")
+
 # Session Management Section
 st.markdown("### 🎯 Session Management")
 
@@ -1969,9 +2016,9 @@ with tab3:
                     color_continuous_scale='viridis'
                 )
                 fig_records.update_layout(
-                    paper_bgcolor="rgba(0,0,0,0)",
-                    plot_bgcolor="rgba(0,0,0,0)",
-                    font_color="white"
+                    paper_bgcolor="rgba(255,255,255,0.9)",
+                    plot_bgcolor="rgba(255,255,255,0.9)",
+                    font_color="#2c3e50"
                 )
                 st.plotly_chart(fig_records, use_container_width=True)
             
@@ -1985,9 +2032,9 @@ with tab3:
                     title="💾 Data Size Distribution (MB)"
                 )
                 fig_size.update_layout(
-                    paper_bgcolor="rgba(0,0,0,0)",
-                    plot_bgcolor="rgba(0,0,0,0)",
-                    font_color="white"
+                    paper_bgcolor="rgba(255,255,255,0.9)",
+                    plot_bgcolor="rgba(255,255,255,0.9)",
+                    font_color="#2c3e50"
                 )
                 st.plotly_chart(fig_size, use_container_width=True)
             
@@ -2003,9 +2050,9 @@ with tab3:
                 color_continuous_scale='plasma'
             )
             fig_perf.update_layout(
-                paper_bgcolor="rgba(0,0,0,0)",
-                plot_bgcolor="rgba(0,0,0,0)",
-                font_color="white"
+                paper_bgcolor="rgba(255,255,255,0.9)",
+                plot_bgcolor="rgba(255,255,255,0.9)",
+                font_color="#2c3e50"
             )
             st.plotly_chart(fig_perf, use_container_width=True)
         
@@ -2016,41 +2063,42 @@ with tab3:
         conn.close()
 
 # AI Enhancement Panel
-if ML_AVAILABLE:
-    st.markdown("---")
-    with st.expander("🤖 AI Enhancement Status", expanded=False):
-        col1, col2, col3 = st.columns(3)
-        
-        with col1:
-            st.markdown("**🎯 Quality Assessment**")
-            if ai_quality_assessor and ai_quality_assessor.quality_model:
-                st.success("✅ Active - DistilBERT")
-            else:
-                st.error("❌ Not Available")
-        
-        with col2:
-            st.markdown("**🔍 Semantic Analysis**")
-            if semantic_analyzer and semantic_analyzer.embeddings_model:
-                st.success("✅ Active - MiniLM-L6-v2")
-            else:
-                st.error("❌ Not Available")
-        
-        with col3:
-            st.markdown("**📊 Health Monitoring**")
-            if health_monitor:
-                st.success("✅ Active - Isolation Forest")
-            else:
-                st.error("❌ Not Available")
-        
-        if ai_quality_assessor and hasattr(ai_quality_assessor, 'quality_model'):
-            st.info("💡 AI models are loaded and ready for enhanced data analysis!")
+st.markdown("---")
+with st.expander("🤖 AI Enhancement Status", expanded=False):
+    col1, col2, col3 = st.columns(3)
+    
+    with col1:
+        st.markdown("**🎯 Quality Assessment**")
+        if ML_AVAILABLE and ai_quality_assessor and ai_quality_assessor.quality_model:
+            st.success("✅ Active - DistilBERT")
+        else:
+            st.error("❌ Not Available")
+    
+    with col2:
+        st.markdown("**🔍 Semantic Analysis**")
+        if ML_AVAILABLE and semantic_analyzer and semantic_analyzer.embeddings_model:
+            st.success("✅ Active - MiniLM-L6-v2")
+        else:
+            st.error("❌ Not Available")
+    
+    with col3:
+        st.markdown("**📊 Health Monitoring**")
+        if health_monitor:
+            st.success("✅ Active - Isolation Forest")
+        else:
+            st.error("❌ Not Available")
+    
+    if ML_AVAILABLE:
+        st.info("💡 AI models are loaded and ready for enhanced data analysis!")
+    else:
+        st.warning("⚠️ Install ML libraries (transformers, sentence-transformers) for AI features")
 
 # Footer
 st.markdown("---")
 st.markdown("""
-<div style="text-align: center; padding: 1rem; opacity: 0.9;">
-    <p><strong>🚀 Ultimate Data Harvester with AI</strong> - Deep discovery, session resumption, intelligent storage</p>
-    <p style="font-size: 0.9rem;">
+<div style="text-align: center; padding: 1rem; background: rgba(255,255,255,0.8); border-radius: 10px; color: #2c3e50;">
+    <p><strong>🚀 Ultimate Data Harvester with AI</strong> - Professional data collection platform</p>
+    <p style="font-size: 0.9rem; color: #7f8c8d;">
         🔍 Recursive endpoint discovery • 🤖 AI quality assessment • 🎯 Session management • 💾 Smart database storage • 📊 Real-time analytics
     </p>
 </div>