Update app.py

app.py

@@ -3,166 +3,222 @@ import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 import seaborn as sns
+import logging
+import requests
+from typing import List, Tuple, Dict, Optional
 from datasets import load_dataset
 from sklearn.ensemble import RandomForestRegressor
 from sklearn.metrics import mean_absolute_error, r2_score
 from sklearn.model_selection import train_test_split
 
-# =========================================================
-# CONFIG
-# =========================================================
-DATASET_MAP = {
-    "Core (Clean)": "QSBench/QSBench-Core-v1.0.0-demo",
-    "Depolarizing Noise": "QSBench/QSBench-Depolarizing-Demo-v1.0.0",
-    "Amplitude Damping": "QSBench/QSBench-Amplitude-v1.0.0-demo",
-    "Transpilation (10q)": "QSBench/QSBench-Transpilation-v1.0.0-demo"
+# Setup production-style logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Configuration for datasets and their specific metadata branches
+REPO_CONFIG = {
+    "Core (Clean)": {
+        "repo": "QSBench/QSBench-Core-v1.0.0-demo",
+        "meta_url": "https://huggingface.co/datasets/QSBench/QSBench-Core-v1.0.0-demo/raw/metadata/meta/meta.json",
+        "report_url": "https://huggingface.co/datasets/QSBench/QSBench-Core-v1.0.0-demo/raw/metadata/meta/report.json"
+    },
+    "Depolarizing Noise": {
+        "repo": "QSBench/QSBench-Depolarizing-Demo-v1.0.0",
+        "meta_url": "https://huggingface.co/datasets/QSBench/QSBench-Depolarizing-Demo-v1.0.0/raw/meta/meta/meta.json",
+        "report_url": "https://huggingface.co/datasets/QSBench/QSBench-Depolarizing-Demo-v1.0.0/raw/meta/meta/report.json"
+    },
+    "Amplitude Damping": {
+        "repo": "QSBench/QSBench-Amplitude-v1.0.0-demo",
+        "meta_url": "https://huggingface.co/datasets/QSBench/QSBench-Amplitude-v1.0.0-demo/raw/meta/meta/meta.json",
+        "report_url": "https://huggingface.co/datasets/QSBench/QSBench-Amplitude-v1.0.0-demo/raw/meta/meta/report.json"
+    },
+    "Transpilation (10q)": {
+        "repo": "QSBench/QSBench-Transpilation-v1.0.0-demo",
+        "meta_url": "https://huggingface.co/datasets/QSBench/QSBench-Transpilation-v1.0.0-demo/raw/meta/meta/meta.json",
+        "report_url": "https://huggingface.co/datasets/QSBench/QSBench-Transpilation-v1.0.0-demo/raw/meta/meta/report.json"
+    }
 }
 
-TARGET_COL = "ideal_expval_Z_global"
-
-# Колонки, которые никогда не должны быть признаками (фичами)
-EXCLUDE_COLS = {
+# Features that should never be used as training inputs
+NON_FEATURE_COLS = {
     "sample_id", "sample_seed", "circuit_hash", "split", "circuit_qasm", 
     "qasm_raw", "qasm_transpiled", "circuit_type_resolved", "circuit_type_requested",
-    "noise_type", "observable_bases", "observable_mode", "backend_device", 
+    "noise_type", "noise_prob", "observable_bases", "observable_mode", "backend_device", 
     "precision_mode", "circuit_signature", "ideal_expval_Z_global", "noisy_expval_Z_global"
 }
 
-dataset_cache = {}
-
-# =========================================================
-# UTILS
-# =========================================================
-def get_df(dataset_key):
-    if dataset_key not in dataset_cache:
-        repo_id = DATASET_MAP[dataset_key]
-        ds = load_dataset(repo_id)
-        dataset_cache[dataset_key] = pd.DataFrame(ds["train"])
-    return dataset_cache[dataset_key]
-
-def get_numeric_feature_cols(df: pd.DataFrame) -> list[str]:
-    numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
-    # Оставляем только структурные метрики, убираем таргеты и ошибки
-    return [c for c in numeric_cols if c not in EXCLUDE_COLS and not c.startswith("error_") and "expval" not in c]
-
-# =========================================================
-# LOGIC
-# =========================================================
-
-# Функция для обновления первой вкладки (Explorer)
-def update_explorer_tab(dataset_name, split_name):
-    df = get_df(dataset_name)
-    splits = df["split"].unique().tolist() if "split" in df.columns else ["train"]
-    filtered = df[df["split"] == split_name].head(10) if "split" in df.columns else df.head(10)
+_ASSET_CACHE = {}
+
+def fetch_remote_json(url: str) -> Optional[dict]:
+    """Helper to fetch JSON files from raw Hugging Face branches."""
+    try:
+        response = requests.get(url, timeout=5)
+        return response.json() if response.status_code == 200 else None
+    except Exception as e:
+        logger.error(f"Error fetching metadata from {url}: {e}")
+        return None
+
+def load_all_assets(key: str) -> Dict:
+    """Fetch dataframe and metadata with memory caching."""
+    if key not in _ASSET_CACHE:
+        logger.info(f"Loading assets for dataset: {key}")
+        # Load main parquet/csv data
+        ds = load_dataset(REPO_CONFIG[key]["repo"])
+        # Fetch metadata from dedicated branches
+        meta = fetch_remote_json(REPO_CONFIG[key]["meta_url"])
+        report = fetch_remote_json(REPO_CONFIG[key]["report_url"])
+        
+        _ASSET_CACHE[key] = {
+            "df": pd.DataFrame(ds["train"]),
+            "meta": meta,
+            "report": report
+        }
+    return _ASSET_CACHE[key]
+
+def generate_meta_markdown(assets: Dict) -> str:
+    """Parse JSON metadata into a human-readable research summary."""
+    meta = assets.get("meta", {})
+    params = meta.get("parameters", {})
+    report = assets.get("report", {})
+    
+    if not meta:
+        return "⚠️ *Metadata currently unavailable for this dataset branch.*"
+    
+    # Format family distribution from report.json
+    families = report.get("families", {})
+    fam_info = ", ".join([f"{k.upper()}: {v}" for k, v in families.items()])
     
-    qasm_raw = filtered["qasm_raw"].iloc[0] if "qasm_raw" in filtered.columns else "// N/A"
-    qasm_tr = filtered["qasm_transpiled"].iloc[0] if "qasm_transpiled" in filtered.columns else "// N/A"
+    md = (
+        f"### 📋 Dataset Release: {meta.get('dataset_version', '1.0.0')}\n"
+        f"**Hardware Config:** {params.get('n_qubits')} Qubits | Depth: {params.get('depth')} | "
+        f"Shots: {params.get('shots')} | Device: {meta.get('backend_device', 'GPU')}\n\n"
+        f"**Noise Model:** `{params.get('noise', 'Clean')}` (p={params.get('noise_prob', 0.0)}) | "
+        f"**Circuit Coverage:** {fam_info}"
+    )
+    return md
+
+def update_explorer_view(ds_name: str, split_name: str):
+    """Main callback for the Explorer tab."""
+    assets = load_all_assets(ds_name)
+    df = assets["df"]
     
-    return gr.update(choices=splits), filtered, qasm_raw, qasm_tr
-
-# Функция для обновления списка фичей во второй вкладке (ML Demo)
-def update_ml_features(dataset_name):
-    df = get_df(dataset_name)
-    features = get_numeric_feature_cols(df)
-    # По умолчанию выбираем первые несколько важных метрик
-    default_selection = [f for f in ["n_qubits", "depth", "total_gates", "gate_entropy", "meyer_wallach"] if f in features]
-    if not default_selection: default_selection = features[:5]
+    splits = df["split"].unique().tolist() if "split" in df.columns else ["train"]
+    display_df = df[df["split"] == split_name].head(10) if "split" in df.columns else df.head(10)
+    
+    # QASM Sample Extraction
+    raw_qasm = display_df["qasm_raw"].iloc[0] if "qasm_raw" in display_df.columns else "// No data"
+    tr_qasm = display_df["qasm_transpiled"].iloc[0] if "qasm_transpiled" in display_df.columns else "// No data"
     
-    return gr.update(choices=features, value=default_selection)
+    return gr.update(choices=splits), display_df, raw_qasm, tr_qasm, generate_meta_markdown(assets)
 
-def run_model_demo(dataset_name, selected_features):
-    df = get_df(dataset_name)
+def sync_ml_inputs(ds_name: str):
+    """Callback to update feature checkboxes when dataset changes."""
+    assets = load_all_assets(ds_name)
+    df = assets["df"]
     
-    # Защита от несуществующих колонок (KeyError)
-    valid_features = [f for f in selected_features if f in df.columns]
+    numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+    valid_features = [c for c in numeric_cols if c not in NON_FEATURE_COLS and not c.startswith(("error_", "sign_", "ideal_", "noisy_"))]
     
-    if not valid_features:
-        return None, "### ⚠️ Ошибка: Выбранные признаки не найдены в этом датасете."
+    # Default selection of core structural metrics
+    top_picks = [f for f in ["gate_entropy", "meyer_wallach", "n_qubits", "depth", "total_gates"] if f in valid_features]
     
-    target = TARGET_COL if TARGET_COL in df.columns else df.filter(like="expval").columns[0]
+    return gr.update(choices=valid_features, value=top_picks or valid_features[:5])
+
+def train_baseline_model(ds_name: str, selected_features: List[str]):
+    """Train a Random Forest regressor and generate analytics plots."""
+    if not selected_features:
+        return None, "### ❌ Error: Please select at least one feature."
     
-    work_df = df.dropna(subset=valid_features + [target]).reset_index(drop=True)
-    X, y = work_df[valid_features], work_df[target]
+    assets = load_all_assets(ds_name)
+    df = assets["df"]
+    target = "ideal_expval_Z_global" if "ideal_expval_Z_global" in df.columns else df.filter(like="expval").columns[0]
     
-    if len(work_df) < 20:
-        return None, "### ⚠️ Недостаточно данных для обучения."
+    # Data cleaning
+    train_df = df.dropna(subset=selected_features + [target])
+    if len(train_df) < 50:
+        return None, "### ⚠️ Warning: Dataset too small for reliable training."
 
+    X, y = train_df[selected_features], train_df[target]
     X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
 
-    model = RandomForestRegressor(n_estimators=50, max_depth=10, n_jobs=-1, random_state=42)
+    # Model Pipeline
+    model = RandomForestRegressor(n_estimators=100, max_depth=12, n_jobs=-1, random_state=42)
     model.fit(X_train, y_train)
     preds = model.predict(X_test)
 
-    sns.set_theme(style="whitegrid")
-    fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(18, 5))
+    # Plotting
+    sns.set_theme(style="whitegrid", context="notebook")
+    fig, axes = plt.subplots(1, 3, figsize=(20, 6))
+    
+    # 1. Parity Plot
+    axes[0].scatter(y_test, preds, alpha=0.4, color='#34495e')
+    axes[0].plot([y.min(), y.max()], [y.min(), y.max()], 'r--', lw=2)
+    axes[0].set_title(f"Regression Accuracy (R²: {r2_score(y_test, preds):.3f})")
+    axes[0].set_xlabel("Actual")
+    axes[0].set_ylabel("Predicted")
+
+    # 2. Importance
+    importances = model.feature_importances_
+    indices = np.argsort(importances)[-12:]
+    axes[1].barh([selected_features[i] for i in indices], importances[indices], color='#1abc9c')
+    axes[1].set_title("Top Structural Predictors")
 
-    # График предсказаний
-    ax1.scatter(y_test, preds, alpha=0.4, color='#636EFA')
-    ax1.plot([y.min(), y.max()], [y.min(), y.max()], 'r--', lw=2)
-    ax1.set_title(f"R² Score: {r2_score(y_test, preds):.3f}")
-    ax1.set_xlabel("Actual")
-    ax1.set_ylabel("Predicted")
+    # 3. Error Analysis
+    sns.histplot(y_test - preds, kde=True, ax=axes[2], color='#e67e22')
+    axes[2].set_title("Residuals Distribution")
 
-    # Важность признаков (топ-10)
-    importances = model.feature_importances_
-    indices = np.argsort(importances)[-10:]
-    ax2.barh(range(len(indices)), importances[indices], color='#EF553B')
-    ax2.set_yticks(range(len(indices)))
-    ax2.set_yticklabels([valid_features[i] for i in indices])
-    ax2.set_title("Feature Importance")
-
-    # Распределение ошибок
-    sns.histplot(y_test - preds, kde=True, ax=ax3, color='#00CC96')
-    ax3.set_title("Residuals")
-    
     plt.tight_layout()
-    return fig, f"### Отчет по датасету: {dataset_name}\n**MAE:** {mean_absolute_error(y_test, preds):.4f}"
+    result_text = f"**Model Performance on {ds_name}**\n**MAE:** {mean_absolute_error(y_test, preds):.4f}"
+    
+    return fig, result_text
 
-# =========================================================
-# UI
-# =========================================================
-with gr.Blocks(title="QSBench Explorer") as demo:
-    gr.Markdown("# 🌌 QSBench: Quantum Synthetic Benchmark")
+# --- GRADIO INTERFACE ---
+
+with gr.Blocks(theme=gr.themes.Soft(), title="QSBench Analytics") as demo:
+    gr.Markdown("# 🌌 QSBench: Quantum Synthetic Benchmark Suite")
     
     with gr.Tabs():
-        # ВКЛАДКА 1: ПРОСМОТР ДАННЫХ
-        with gr.TabItem("🔎 Explorer"):
+        with gr.TabItem("🔎 Dataset Explorer"):
+            # Header with parsed metadata from JSON
+            metadata_box = gr.Markdown("### Synchronizing with Hugging Face...")
+            
             with gr.Row():
-                ds_selector = gr.Dropdown(choices=list(DATASET_MAP.keys()), value="Core (Clean)", label="Dataset")
-                split_selector = gr.Dropdown(choices=["train"], value="train", label="Split")
+                ds_select = gr.Dropdown(choices=list(REPO_CONFIG.keys()), value="Core (Clean)", label="Dataset Pack")
+                split_select = gr.Dropdown(choices=["train"], value="train", label="Subset")
             
             data_table = gr.Dataframe(interactive=False)
             
             with gr.Row():
-                qasm_raw_view = gr.Code(label="Raw QASM", language="python", lines=10)
-                qasm_tr_view = gr.Code(label="Transpiled QASM", language="python", lines=10)
+                code_raw = gr.Code(label="Source Circuit (QASM)", language="python")
+                code_tr = gr.Code(label="Transpiled (Hardware-Ready)", language="python")
 
-        # ВКЛАДКА 2: МАШИННОЕ ОБУЧЕНИЕ
-        with gr.TabItem("🤖 ML Demo"):
+        with gr.TabItem("🤖 ML Training Baseline"):
             with gr.Row():
                 with gr.Column(scale=1):
-                    gr.Markdown("### Настройка обучения")
-                    m_ds_selector = gr.Dropdown(choices=list(DATASET_MAP.keys()), value="Core (Clean)", label="Dataset for ML")
-                    f_selector = gr.CheckboxGroup(label="Признаки (Features)", choices=[])
-                    train_btn = gr.Button("Запустить обучение", variant="primary")
+                    gr.Markdown("### Training Configuration")
+                    ml_ds_select = gr.Dropdown(choices=list(REPO_CONFIG.keys()), value="Core (Clean)", label="Source Dataset")
+                    ml_features = gr.CheckboxGroup(label="Structural Metrics", choices=[])
+                    btn_train = gr.Button("Execute Baseline Training", variant="primary")
                 with gr.Column(scale=2):
-                    plot_out = gr.Plot()
-                    text_out = gr.Markdown()
-
-    # --- ЛОГИКА СОБЫТИЙ ---
-
-    # При измене��ии датасета в Explorer — обновляем таблицу и QASM
-    ds_selector.change(update_explorer_tab, [ds_selector, split_selector], [split_selector, data_table, qasm_raw_view, qasm_tr_view])
-    
-    # ПРИНЦИПИАЛЬНО: При изменении датасета в ML Demo — обновляем список чекбоксов
-    m_ds_selector.change(update_ml_features, inputs=[m_ds_selector], outputs=[f_selector])
-    
-    # Кнопка обучения
-    train_btn.click(run_model_demo, [m_ds_selector, f_selector], [plot_out, text_out])
+                    plot_output = gr.Plot()
+                    txt_output = gr.Markdown()
+
+    # Footer/Resources
+    gr.Markdown("""
+    ---
+    ### 🔬 Research Credits
+    **QSBench** is an open-source framework for noise-aware Quantum Machine Learning benchmarking.
+    - [GitHub Repository](https://github.com/QSBench/QSBench-Demo) | [Official Website](https://qsbench.github.io)
+    """)
+
+    # Event Handlers
+    ds_select.change(update_explorer_view, [ds_select, split_select], [split_select, data_table, code_raw, code_tr, metadata_box])
+    ml_ds_select.change(sync_ml_inputs, [ml_ds_select], [ml_features])
+    btn_train.click(train_baseline_model, [ml_ds_select, ml_features], [plot_output, txt_output])
     
-    # Инициализация при старте
-    demo.load(update_explorer_tab, [ds_selector, split_selector], [split_selector, data_table, qasm_raw_view, qasm_tr_view])
-    demo.load(update_ml_features, [m_ds_selector], [f_selector])
+    # Initial Load
+    demo.load(update_explorer_view, [ds_select, split_select], [split_select, data_table, code_raw, code_tr, metadata_box])
+    demo.load(sync_ml_inputs, [ml_ds_select], [ml_features])
 
 if __name__ == "__main__":
-    demo.launch(theme=gr.themes.Soft())
+    demo.launch()