Upload 2 files

app.py

@@ -0,0 +1,401 @@
+import streamlit as st
+import duckdb
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+import numpy as np
+
+# Configuration de la page
+st.set_page_config(
+    page_title="DuckDB Database Analyzer",
+    page_icon="🦆",
+    layout="wide",
+    initial_sidebar_state="expanded"
+)
+
+st.title("🦆 DuckDB Database Analyzer")
+st.markdown("**Analysez vos bases de données sans les importer !**")
+
+# Sidebar
+st.sidebar.header("⚙️ Paramètres de connexion")
+
+# Gestion du reset
+if "reset_counter" not in st.session_state:
+    st.session_state.reset_counter = 0
+if "test_url" not in st.session_state:
+    st.session_state.test_url = ""
+if "analysis_done" not in st.session_state:
+    st.session_state.analysis_done = False
+if "analysis_data" not in st.session_state:
+    st.session_state.analysis_data = {}
+
+# Champ URL avec clé dynamique
+url_input = st.sidebar.text_input(
+    "📍 URL de la base de données",
+    value=st.session_state.test_url,
+    placeholder="https://example.com/data.parquet",
+    help="Formats supportés : Parquet, CSV, JSON, HTTP, S3, etc.",
+    key=f"url_input_{st.session_state.reset_counter}"
+)
+
+# Bouton Reset
+col1, col2 = st.sidebar.columns([4, 1])
+with col2:
+    if st.sidebar.button("🗑️ Reset"):
+        st.session_state.reset_counter += 1
+        st.session_state.test_url = ""
+        st.session_state.analysis_done = False
+        st.session_state.analysis_data = {}
+        st.rerun()
+
+# Options
+with st.sidebar.expander("🔧 Options avancées"):
+    max_rows_sample = st.slider("Lignes échantillon", 50, 2000, 100)
+
+# Bouton d'analyse
+if st.sidebar.button("🚀 Analyser la base de données", type="primary"):
+    if url_input:
+        st.session_state.test_url = ""
+        
+        with st.spinner("🔍 Analyse en cours..."):
+            try:
+                con = duckdb.connect()
+                con.execute("INSTALL httpfs; LOAD httpfs;")
+
+                # Test de lecture
+                formats_to_try = [
+                    ("parquet", f"read_parquet('{url_input}')"),
+                    ("csv", f"read_csv_auto('{url_input}')"),
+                    ("json", f"read_json_auto('{url_input}')")
+                ]
+                
+                read_func = ""
+                detected_format = ""
+                
+                for fmt_name, fmt in formats_to_try:
+                    try:
+                        result = con.execute(f"SELECT COUNT(*) FROM {fmt}").fetchone()
+                        if result and result[0] is not None:
+                            read_func = fmt
+                            detected_format = fmt_name
+                            st.success(f"✅ Format détecté : {fmt_name}")
+                            break
+                    except:
+                        continue
+
+                if not read_func:
+                    st.error("❌ Impossible de lire le fichier. Vérifiez l'URL.")
+                    st.stop()
+
+                # Nombre total de lignes
+                total_rows = con.execute(f"SELECT COUNT(*) FROM {read_func}").fetchone()[0]
+
+                # Nombre de colonnes
+                sample_df = con.execute(f"SELECT * FROM {read_func} LIMIT 1").df()
+                num_columns = len(sample_df.columns)
+                
+                # TAILLE FICHIER
+                file_size = "N/A"
+                try:
+                    if detected_format == "parquet":
+                        metadata_result = con.execute(f"""
+                            SELECT COUNT(*) as row_groups 
+                            FROM parquet_metadata('{url_input}')
+                        """).fetchone()
+                        if metadata_result:
+                            row_groups = metadata_result[0]
+                            estimated_mb = row_groups * 4.5
+                            file_size = f"~{estimated_mb:.0f} MB"
+                except:
+                    pass
+
+                # Analyse des variables
+                sample_1000 = con.execute(f"SELECT * FROM {read_func} LIMIT 1000").df()
+                
+                columns_info = []
+                for col in sample_1000.columns:
+                    col_data = sample_1000[col].dropna()
+                    
+                    # Détection type
+                    if len(col_data) == 0:
+                        col_type = "UNKNOWN"
+                        detail_type = "VIDE"
+                    elif pd.api.types.is_integer_dtype(col_data):
+                        col_type = "INTEGER"
+                        detail_type = "ENTIER"
+                    elif pd.api.types.is_float_dtype(col_data):
+                        col_type = "FLOAT"
+                        detail_type = "DÉCIMAL"
+                    elif pd.api.types.is_datetime64_any_dtype(col_data):
+                        col_type = "DATETIME"
+                        detail_type = "DATE/HEURE"
+                    elif pd.api.types.is_bool_dtype(col_data):
+                        col_type = "BOOLEAN"
+                        detail_type = "BOOLEEN"
+                    else:
+                        col_type = "TEXT"
+                        try:
+                            pd.to_numeric(col_data, errors='raise')
+                            detail_type = "NUMÉRIQUE"
+                        except:
+                            detail_type = "TEXTE"
+                    
+                    # Taux de remplissage sur l'échantillon
+                    null_count_sample = sample_1000[col].isna().sum()
+                    fill_rate = ((1000 - null_count_sample) / 1000 * 100)
+                    
+                    example = str(col_data.iloc[0])[:30] if len(col_data) > 0 else "N/A"
+                    
+                    columns_info.append({
+                        'Variable': col,
+                        'Type': col_type,
+                        'Type_Détaillé': detail_type,
+                        'Valeurs_Manquantes': null_count_sample,
+                        'Taux_Remplissage': round(fill_rate, 1),
+                        'Exemple': example
+                    })
+                
+                columns_df = pd.DataFrame(columns_info)
+                
+                # Échantillon pour affichage
+                sample_display = con.execute(f"SELECT * FROM {read_func} LIMIT {max_rows_sample}").df()
+
+                # Sauvegarder les résultats
+                st.session_state.analysis_data = {
+                    'total_rows': total_rows,
+                    'num_columns': num_columns,
+                    'file_size': file_size,
+                    'detected_format': detected_format,
+                    'columns_df': columns_df,
+                    'sample_display': sample_display,
+                    'read_func': read_func,
+                    'url_input': url_input
+                }
+                st.session_state.analysis_done = True
+                
+                con.close()
+                st.success("✅ **Analyse terminée avec succès !**")
+                st.rerun()
+
+            except Exception as e:
+                st.error(f"❌ Erreur lors de l'analyse : {str(e)}")
+                st.info("💡 Vérifiez que l'URL est accessible et publique")
+    else:
+        st.warning("⚠️ Veuillez saisir une URL valide")
+
+# URLs de test
+with st.sidebar.expander("🧪 URLs de test"):
+    st.markdown("**URL fonctionnelles pour tester :**")
+    
+    test_urls = [
+        ("SIREN Entreprises France", "https://object.files.data.gouv.fr/data-pipeline-open/siren/stock/StockUniteLegale_utf8.parquet"),
+        ("NYC Taxi Oct 2025", "https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-10.parquet"),
+        ("Open Data Paris Ilôts de fraîcheur", r"https://opendata.paris.fr/api/explore/v2.1/catalog/datasets/ilots-de-fraicheur-equipements-activites/exports/csv?lang=fr&timezone=Europe%2FBerlin&use_labels=true&delimiter=%3B")
+    ]
+    
+    for i, (name, url) in enumerate(test_urls):
+        if st.button(f"📊 {name}", key=f"test_{i}", width='stretch'):
+            st.session_state.reset_counter += 1
+            st.session_state.test_url = url
+            st.rerun()
+
+# AFFICHAGE DES RÉSULTATS AVEC ONGLETS
+if st.session_state.analysis_done:
+    data = st.session_state.analysis_data
+    
+    tab1, tab2, tab3, tab4 = st.tabs(["📊 Dashboard", "📋 Variables", "💾 Données", "💻 Code"])
+    
+    # ============================================
+    # ONGLET 1: DASHBOARD
+    # ============================================
+    with tab1:
+        # Calcul des métriques pour le rapport de qualité
+        avg_fill = data['columns_df']['Taux_Remplissage'].mean()
+        missing_cols = len(data['columns_df'][data['columns_df']['Taux_Remplissage'] < 100])
+        complete_cols = len(data['columns_df']) - missing_cols
+        
+        # Style CSS pour les cards
+        st.markdown("""
+        <style>
+        .metric-card {
+            background-color: #f0f2f6;
+            border-radius: 10px;
+            padding: 15px;
+            text-align: center;
+            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+            height: 100px;
+            display: flex;
+            flex-direction: column;
+            justify-content: center;
+            align-items: center;
+        }
+        .metric-label {
+            font-size: 0.85em;
+            color: #666;
+            margin-bottom: 5px;
+            line-height: 1.2;
+            min-height: 32px;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+        }
+        .metric-value {
+            font-size: 1.8em;
+            font-weight: bold;
+            color: #262730;
+        }
+        </style>
+        """, unsafe_allow_html=True)
+        
+        # Cards en haut - 7 colonnes
+        col1, col2, col3, col4, col5, col6, col7 = st.columns(7)
+        
+        with col1:
+            st.markdown(f"""
+            <div class="metric-card">
+                <div class="metric-label">📊 Observations</div>
+                <div class="metric-value">{data['total_rows']:,}</div>
+            </div>
+            """, unsafe_allow_html=True)
+        
+        with col2:
+            st.markdown(f"""
+            <div class="metric-card">
+                <div class="metric-label">📋 Colonnes</div>
+                <div class="metric-value">{data['num_columns']}</div>
+            </div>
+            """, unsafe_allow_html=True)
+        
+        with col3:
+            st.markdown(f"""
+            <div class="metric-card">
+                <div class="metric-label">💾 Taille fichier</div>
+                <div class="metric-value">{data['file_size']}</div>
+            </div>
+            """, unsafe_allow_html=True)
+        
+        with col4:
+            st.markdown(f"""
+            <div class="metric-card">
+                <div class="metric-label">📄 Format</div>
+                <div class="metric-value">{data['detected_format'].upper()}</div>
+            </div>
+            """, unsafe_allow_html=True)
+        
+        with col5:
+            st.markdown(f"""
+            <div class="metric-card">
+                <div class="metric-label">✅ Taux moyen</div>
+                <div class="metric-value">{avg_fill:.1f}%</div>
+            </div>
+            """, unsafe_allow_html=True)
+        
+        with col6:
+            st.markdown(f"""
+            <div class="metric-card">
+                <div class="metric-label">⚠️ Colonnes incomplètes</div>
+                <div class="metric-value">{missing_cols}</div>
+            </div>
+            """, unsafe_allow_html=True)
+        
+        with col7:
+            st.markdown(f"""
+            <div class="metric-card">
+                <div class="metric-label">✔️ Colonnes complètes</div>
+                <div class="metric-value">{complete_cols}</div>
+            </div>
+            """, unsafe_allow_html=True)
+        
+        st.markdown("<br>", unsafe_allow_html=True)
+        
+        # Graphiques côte à côte
+        col_left, col_right = st.columns([2, 1])
+        
+        with col_left:
+            # Graphique vertical du taux de remplissage
+            fig_fill = px.bar(
+                data['columns_df'].sort_values('Taux_Remplissage'), 
+                y='Variable', 
+                x='Taux_Remplissage',
+                title="Taux de remplissage par variable (1000 premières lignes)",
+                color='Taux_Remplissage',
+                color_continuous_scale='RdYlGn',
+                orientation='h',
+                range_color=[0, 100],
+                height=500
+            )
+            
+            fig_fill.update_layout(
+                showlegend=False,
+                xaxis_title="Taux de Remplissage (%)",
+                yaxis_title="",
+                margin=dict(t=50, b=50, l=200, r=20)
+            )
+            fig_fill.update_traces(marker_line_width=0, marker_cornerradius=5)
+            fig_fill.update_yaxes(tickmode='linear')
+            
+            st.plotly_chart(fig_fill, width='stretch')
+        
+        with col_right:
+            # Camembert des types
+            type_counts = data['columns_df']['Type_Détaillé'].value_counts()
+            fig_pie = px.pie(
+                values=type_counts.values,
+                names=type_counts.index,
+                title="Répartition des types"
+            )
+            fig_pie.update_traces(textposition='inside', textinfo='percent+label')
+            fig_pie.update_layout(height=500)
+            st.plotly_chart(fig_pie, width='stretch')
+    
+    # ============================================
+    # ONGLET 2: VARIABLES
+    # ============================================
+    with tab2:
+        st.header("📋 Structure des Variables")
+        
+        # Tableau fusionné
+        display_df = data['columns_df'][['Variable', 'Type', 'Type_Détaillé', 'Valeurs_Manquantes', 'Taux_Remplissage', 'Exemple']].copy()
+        display_df.columns = ['Variable', 'Type', 'Type Détaillé', 'Valeurs Manquantes (sur 1000)', 'Taux de Remplissage (%)', 'Exemple']
+        
+        st.dataframe(display_df, width='stretch', height=600)
+    
+    # ============================================
+    # ONGLET 3: DONNÉES
+    # ============================================
+    with tab3:
+        st.header("💾 Échantillon des Données")
+        
+        col1, col2 = st.columns([1, 3])
+        with col1:
+            st.metric("Lignes affichées", f"{len(data['sample_display']):,}")
+        with col2:
+            st.caption(f"sur {data['total_rows']:,} total")
+        
+        st.dataframe(data['sample_display'], width='stretch', height=600)
+    
+    # ============================================
+    # ONGLET 4: CODE
+    # ============================================
+    with tab4:
+        st.header("💻 Code Python prêt à l'emploi")
+        
+        st.code(f"""
+import duckdb
+
+# Connexion
+con = duckdb.connect()
+con.execute("INSTALL httpfs; LOAD httpfs;")
+
+# Lecture des données
+df = con.execute("SELECT * FROM {data['read_func']} LIMIT 1000").df()
+print(f"Forme: {{df.shape}}")
+print("Colonnes:", df.columns.tolist())
+
+# Nombre total de lignes
+total_rows = con.execute("SELECT COUNT(*) FROM {data['read_func']}").fetchone()[0]
+print(f"Total lignes: {{total_rows:,}}")
+        """, language="python")
+
+else:
+    st.info("👆 Veuillez saisir une URL et cliquer sur **Analyser la base de données** pour commencer l'analyse")

requirements.txt

@@ -1,3 +1,5 @@
-altair
-pandas
-streamlit
+streamlit==1.31.0
+duckdb==0.10.0
+pandas==2.2.0
+plotly==5.18.0
+numpy==1.26.3