Deploy weighted clustering presets with 11 physics perspectives

app.py

@@ -16,7 +16,6 @@ import argparse
 import json
 import os
 import sys
-from functools import lru_cache
 from pathlib import Path
 
 from dotenv import load_dotenv
@@ -56,8 +55,8 @@ DASH_I18N = {
         ),
         "c1_title": "Band gap distribution over material families",
         "c2_title": "Compare band gap distribution of material types",
-        "c3_title": "Principal Component Analysis (PCA) for high-dimensional material embeddings",
-        "c3_desc": "Reduces 32-dimensional material embeddings to 2D/3D to reveal clustering patterns.",
+        "c3_title": "Material Embedding Space Visualization",
+        "c3_desc": "Visualize via supervised UMAP (preset mode) or PCA (custom/fallback).",
         "c4_title": "Look up materials by chemical family",
         "c5_title": "Band gap by material family and material type",
         "c6_title": "Show the top N material families",
@@ -101,9 +100,8 @@ DASH_I18N = {
         "chart_type_avg_cluster": "{type} — Avg Band Gap by Cluster",
         "chart_pca_2d": "PCA 2D Projection",
         "chart_pca_3d": "PCA 3D Projection",
-        "lbl_weighted_pc1": "Weighted PC1 (Physics-Adaptive)",
-        "lbl_weighted_pc2": "Weighted PC2 (Physics-Adaptive)",
-        "lbl_weighted_pc3": "Weighted PC3 (Physics-Adaptive)",
+        "chart_umap_2d": "UMAP 2D Projection (Supervised)",
+        "lbl_projection": "Projection Method",
         "chart_variance": "Explained Variance (%)",
         "chart_component": "Component",
         "chart_variance_pct": "Variance (%)",
@@ -159,8 +157,8 @@ DASH_I18N = {
         ),
         "c1_title": "材料ファミリーごとのバンドギャップ分布",
         "c2_title": "材料タイプ別バンドギャップ分布の比較",
-        "c3_title": "高次元材料埋め込みの主成分分析（PCA）",
-        "c3_desc": "32次元の材料埋め込みを2D/3Dに次元削減し、クラスタリングパターンを可視化。",
+        "c3_title": "材料埋め込み空間の可視化",
+        "c3_desc": "教師付きUMAP（プリセット）またはPCA（カスタム/フォールバック）で可視化。",
         "c4_title": "化学ファミリーで材料を検索",
         "c5_title": "材料ファミリーと材料タイプ別バンドギャップ",
         "c6_title": "上位N材料ファミリーを表示",
@@ -204,9 +202,8 @@ DASH_I18N = {
         "chart_type_avg_cluster": "{type} — クラスター別平均バンドギャップ",
         "chart_pca_2d": "PCA 2D射影",
         "chart_pca_3d": "PCA 3D射影",
-        "lbl_weighted_pc1": "重み付きPC1（物理適応軸）",
-        "lbl_weighted_pc2": "重み付きPC2（物理適応軸）",
-        "lbl_weighted_pc3": "重み付きPC3（物理適応軸）",
+        "chart_umap_2d": "UMAP 2D射影（教師付き）",
+        "lbl_projection": "射影法",
         "chart_variance": "寄与率（%）",
         "chart_component": "成分",
         "chart_variance_pct": "寄与率（%）",
@@ -292,35 +289,7 @@ df = pd.read_csv(CSV)
 DIM_COLS = [f"dim_{i}" for i in range(32)]
 ALL_TYPES = ["Metallic", "Semiconductor", "Insulator"]
 
-# Load raw embedding vectors for runtime weighted PCA
-print("Loading raw embedding vectors for weighted PCA...")
-VECTORS_PATH = CACHE / "vectors.npy"
-ESEN_PATH = CACHE / "esen_vectors.npy"
-
-if VECTORS_PATH.exists() and ESEN_PATH.exists():
-    vectors = np.load(VECTORS_PATH).astype(np.float32)  # (33973, 2738)
-    esen = np.load(ESEN_PATH).astype(np.float32)        # (33973, 128)
-
-    # Split into embedding spaces
-    RAW_EMBEDDINGS = {
-        "orb": vectors[:, :1792],           # (33973, 1792)
-        "mm": vectors[:, 1792:2550],        # (33973, 758)
-        "ofm": vectors[:, 2550:2738],       # (33973, 188)
-        "esen": esen,                       # (33973, 128)
-    }
-
-    mem_mb = sum(v.nbytes for v in RAW_EMBEDDINGS.values()) / (1024 * 1024)
-    print(f"  Loaded raw vectors: {mem_mb:.1f} MB in memory")
-    print(f"    orb: {RAW_EMBEDDINGS['orb'].shape}")
-    print(f"    mm: {RAW_EMBEDDINGS['mm'].shape}")
-    print(f"    ofm: {RAW_EMBEDDINGS['ofm'].shape}")
-    print(f"    esen: {RAW_EMBEDDINGS['esen'].shape}")
-else:
-    print("  WARNING: Raw vectors not found. Runtime PCA disabled.")
-    RAW_EMBEDDINGS = None
-
 # Compute PCA on balanced 32D UMAP for variance bar chart
-# Chart 3 scatter will use runtime weighted PCA instead
 X = StandardScaler().fit_transform(df[DIM_COLS].values)
 pca_full = PCA(n_components=20, random_state=42).fit(X)
 pc_all = pca_full.transform(X)  # (33973, 20)
@@ -418,76 +387,20 @@ if S_ORB is not None and S_MM is not None and S_OFM is not None and S_ESEN is no
 else:
     print("  WARNING: Centroid similarity matrices not loaded. Custom mode unavailable.")
 
-# ── Weighted PCA computation ───────────────────────────────────────────
-
-def compute_weighted_pca(weights, n_dims=2):
-    """
-    Compute PCA on weighted concatenated vectors.
-
-    This is the mathematically correct approach: weight-concatenate the raw
-    embedding vectors, then compute PCA on the weighted space. Guarantees
-    that PC0, PC1 capture maximum variance in the weighted combination.
-
-    Args:
-        weights: dict with keys 'orb', 'mm', 'ofm', 'esen' (normalized to sum=1)
-        n_dims: int, 2 for 2D scatter, 3 for 3D scatter
-
-    Returns:
-        np.ndarray of shape (33973, n_dims) - PCA coordinates
-
-    Performance: ~500ms (StandardScaler + PCA on 33,973 × 2,866 matrix)
-    """
-    if RAW_EMBEDDINGS is None:
-        # Fallback: return zeros if raw vectors not loaded
-        print("WARNING: RAW_EMBEDDINGS not available, returning zeros")
-        return np.zeros((len(df), n_dims))
-
-    # Weight-concatenate raw embedding vectors
-    # Scale by sqrt(dim) to give equal importance per dimension, not per space
-    weighted_vectors = np.concatenate([
-        weights["orb"] * RAW_EMBEDDINGS["orb"] / np.sqrt(1792),
-        weights["mm"] * RAW_EMBEDDINGS["mm"] / np.sqrt(758),
-        weights["ofm"] * RAW_EMBEDDINGS["ofm"] / np.sqrt(188),
-        weights["esen"] * RAW_EMBEDDINGS["esen"] / np.sqrt(128),
-    ], axis=1)  # Shape: (33973, 2866)
-
-    # Standardize and compute PCA
-    # StandardScaler ensures each dimension has mean=0, std=1
-    scaler = StandardScaler()
-    scaled = scaler.fit_transform(weighted_vectors)
-
-    # PCA to extract top n_dims components
-    pca = PCA(n_components=n_dims, random_state=42)
-    pca_coords = pca.fit_transform(scaled)  # Shape: (33973, n_dims)
-
-    # Log explained variance for debugging
-    explained_var = pca.explained_variance_ratio_.sum() * 100
-    print(f"  Weighted PCA: {explained_var:.1f}% variance explained (n_dims={n_dims})")
-
-    return pca_coords
-
-
-@lru_cache(maxsize=128)
-def compute_weighted_pca_cached(weights_tuple, n_dims):
-    """
-    Cached wrapper for compute_weighted_pca.
-
-    Args:
-        weights_tuple: tuple (w_orb, w_mm, w_ofm, w_esen) - must be hashable
-        n_dims: int
+# ── Load per-preset UMAP 2D coordinates ────────────────────────────────
+print("Loading per-preset UMAP 2D projections...")
+PRESET_UMAP_2D = {}
+for key in PRESET_KEYS:
+    umap_path = CACHE / f"umap_2d_{key}.npy"
+    if umap_path.exists():
+        PRESET_UMAP_2D[key] = np.load(umap_path).astype(np.float32)
 
-    Returns:
-        np.ndarray of shape (33973, n_dims)
-    """
-    # Convert tuple back to dict
-    weights = {
-        "orb": weights_tuple[0],
-        "mm": weights_tuple[1],
-        "ofm": weights_tuple[2],
-        "esen": weights_tuple[3],
-    }
+if PRESET_UMAP_2D:
+    print(f"  Loaded {len(PRESET_UMAP_2D)} UMAP 2D projections (~{sum(v.nbytes for v in PRESET_UMAP_2D.values()) / (1024*1024):.1f} MB)")
+else:
+    print("  WARNING: No UMAP 2D files found. Will use PCA fallback.")
 
-    return compute_weighted_pca(weights, n_dims)
+HAS_UMAP = bool(PRESET_UMAP_2D)
 
 # ── Label resolution ───────────────────────────────────────────────────
 
@@ -700,10 +613,6 @@ app.layout = html.Div(
         dcc.Store(id="active-labels",
                   data={"mode": "preset", "key": "balanced"}),
 
-        # ── Debouncing for weight sliders ──────────────────────────────
-        dcc.Store(id="weight-buffer", data=None),
-        dcc.Interval(id="debounce-interval", interval=300, max_intervals=0),
-
         # ── Clustering control panel ────────────────────────────────────
         html.Div(style={**CARD, "borderLeft": "4px solid #1976d2"}, children=[
             html.H3(T["lbl_cluster_mode"],
@@ -769,11 +678,21 @@ app.layout = html.Div(
             html.Div(style={"display": "flex", "gap": "16px",
                             "flexWrap": "wrap", "marginBottom": "12px"}, children=[
                 html.Div([
-                    html.Div(T["lbl_components"], style=LABEL),
-                    dcc.Dropdown(id="c3-ndim", options=["2D", "3D"],
-                                 value="2D", clearable=False,
-                                 style={"width": "100px"}),
-                ]),
+                    html.Div(T["lbl_projection"], style=LABEL),
+                    dcc.Dropdown(
+                        id="c3-ndim",
+                        options=(
+                            [{"label": "UMAP 2D", "value": "UMAP_2D"}]
+                            if HAS_UMAP else []
+                        ) + [
+                            {"label": "PCA 2D", "value": "PCA_2D"},
+                            {"label": "PCA 3D", "value": "PCA_3D"},
+                        ],
+                        value="UMAP_2D" if HAS_UMAP else "PCA_2D",
+                        clearable=False,
+                        style={"width": "150px"}
+                    ),
+                ], style={"display": "flex", "gap": "8px", "alignItems": "center"}),
                 html.Div([
                     html.Div(T["lbl_color_by"], style=LABEL),
                     dcc.Dropdown(id="c3-color", options=color_by_options,
@@ -806,20 +725,12 @@ app.layout = html.Div(
             ]),
             html.Div(style={"display": "flex", "gap": "16px"}, children=[
                 html.Div(
-                    dcc.Loading(
-                        id="loading-c3-scatter",
-                        type="circle",
-                        children=dcc.Graph(id="c3-scatter")
-                    ),
+                    dcc.Graph(id="c3-scatter"),
                     id="c3-scatter-container",
                     style={"flex": "1", "minWidth": 0}
                 ),
                 html.Div(
-                    dcc.Loading(
-                        id="loading-c3-variance",
-                        type="circle",
-                        children=dcc.Graph(id="c3-variance")
-                    ),
+                    dcc.Graph(id="c3-variance"),
                     id="c3-variance-container",
                     style={"flex": "1", "minWidth": 0, "display": "none"}
                 ),
@@ -1059,50 +970,6 @@ app.layout = html.Div(
 # CALLBACKS
 # ══════════════════════════════════════════════════════════════════════════
 
-# ── Buffer weight changes (instant, no PCA) ──────────────────────────
-@callback(
-    Output("weight-buffer", "data"),
-    Output("debounce-interval", "max_intervals"),
-    Input("cw-orb", "value"),
-    Input("cw-mm", "value"),
-    Input("cw-ofm", "value"),
-    Input("cw-esen", "value"),
-    State("debounce-interval", "n_intervals"),
-)
-def buffer_weights(w_orb, w_mm, w_ofm, w_esen, n_intervals):
-    """Store weight changes immediately without triggering expensive PCA."""
-    import time
-    weights = [w_orb or 0.25, w_mm or 0.25, w_ofm or 0.25, w_esen or 0.25]
-    timestamp = time.time()
-
-    # Reset interval counter to restart debounce timer
-    # max_intervals=1 means interval will fire once after 300ms
-    return {"weights": weights, "timestamp": timestamp}, 1
-
-
-# ── Apply debounced weights (after 300ms of no changes) ─────────────
-@callback(
-    Output("active-labels", "data", allow_duplicate=True),
-    Input("debounce-interval", "n_intervals"),
-    State("weight-buffer", "data"),
-    State("cluster-mode", "value"),
-    prevent_initial_call=True,
-)
-def apply_debounced_weights(n_intervals, buffer, mode):
-    """Apply weights after debounce delay (300ms of no slider changes)."""
-    if mode != "custom" or not buffer:
-        from dash.exceptions import PreventUpdate
-        raise PreventUpdate
-
-    w = buffer["weights"]
-    total = sum(w)
-    if total > 0:
-        w = [x / total for x in w]
-    else:
-        w = [0.25, 0.25, 0.25, 0.25]
-
-    return {"mode": "custom", "weights": w}
-
 
 # ── Active labels computation (preset mode or initial) ────────────────
 @callback(
@@ -1216,40 +1083,22 @@ def chart3(ndim, color_by, filter_type, topn_str, active_data):
     clusters, families, displays = resolve_labels(active_data)
     work = df.assign(Cluster=clusters, Family=families, FamilyDisplay=displays)
 
-    # ===== Compute weighted PCA coordinates =====
-    # Extract weights from active_data
-    if active_data and active_data.get("mode") == "preset":
-        preset_key = active_data.get("key", "balanced")
-        weights_tuple = PRESETS.get(preset_key, (0.25, 0.25, 0.25, 0.25))
-    elif active_data and active_data.get("mode") == "custom":
-        weights_tuple = tuple(active_data.get("weights", [0.25, 0.25, 0.25, 0.25]))
-    else:
-        # Fallback to balanced
-        weights_tuple = (0.25, 0.25, 0.25, 0.25)
-
-    # Normalize weights
-    w_sum = sum(weights_tuple)
-    if w_sum > 0:
-        weights = {
-            "orb": weights_tuple[0] / w_sum,
-            "mm": weights_tuple[1] / w_sum,
-            "ofm": weights_tuple[2] / w_sum,
-            "esen": weights_tuple[3] / w_sum,
-        }
-    else:
-        weights = {"orb": 0.25, "mm": 0.25, "ofm": 0.25, "esen": 0.25}
+    # ===== Assign UMAP 2D coordinates if using UMAP mode =====
+    if ndim == "UMAP_2D" and HAS_UMAP:
+        # Determine which preset's UMAP to use
+        preset_key = "balanced"  # Default fallback
+
+        if active_data and active_data.get("mode") == "preset":
+            preset_key = active_data.get("key", "balanced")
+        # Custom mode: use balanced UMAP for fixed layout
 
-    # Compute weighted PCA coordinates (cached for performance)
-    n_dims = 3 if ndim == "3D" else 2
-    weights_normalized = (weights["orb"], weights["mm"], weights["ofm"], weights["esen"])
-    pca_coords = compute_weighted_pca_cached(weights_normalized, n_dims=n_dims)
+        # Load the appropriate UMAP coordinates
+        umap_2d = PRESET_UMAP_2D.get(preset_key, PRESET_UMAP_2D.get("balanced"))
 
-    # Add dynamic coordinates to DataFrame
-    work = work.copy()
-    work["PC1_dynamic"] = pca_coords[:, 0]
-    work["PC2_dynamic"] = pca_coords[:, 1]
-    if n_dims == 3:
-        work["PC3_dynamic"] = pca_coords[:, 2]
+        # Assign to DataFrame
+        work = work.copy()
+        work["UMAP_X"] = umap_2d[:, 0]
+        work["UMAP_Y"] = umap_2d[:, 1]
 
     sub = work if filter_type == "All" else work[work["Type"] == filter_type]
 
@@ -1265,28 +1114,40 @@ def chart3(ndim, color_by, filter_type, topn_str, active_data):
         sub["Cluster"] = sub["Cluster"].astype(str)
         color = "Cluster"
 
-    if ndim == "3D":
-        fig1 = px.scatter_3d(sub, x="PC1_dynamic", y="PC2_dynamic", z="PC3_dynamic",
+    # ===== Generate scatter plot based on projection mode =====
+    if ndim == "UMAP_2D":
+        fig1 = px.scatter(
+            sub, x="UMAP_X", y="UMAP_Y",
+            color=color, template="plotly_white",
+            title=T["chart_umap_2d"],
+            render_mode="webgl", opacity=0.6, height=600
+        )
+        fig1.update_traces(marker_size=3)
+        fig1.update_layout(
+            xaxis_title="UMAP 1",
+            yaxis_title="UMAP 2"
+        )
+    elif ndim == "PCA_3D":
+        fig1 = px.scatter_3d(sub, x="PC1", y="PC2", z="PC3",
                              color=color, template="plotly_white",
                              title=T["chart_pca_3d"],
                              opacity=0.6, height=600)
         fig1.update_traces(marker_size=2)
-    else:
-        fig1 = px.scatter(sub, x="PC1_dynamic", y="PC2_dynamic",
+        fig1.update_layout(scene=dict(
+            xaxis_title="PC1",
+            yaxis_title="PC2",
+            zaxis_title="PC3"
+        ))
+    else:  # PCA_2D
+        fig1 = px.scatter(sub, x="PC1", y="PC2",
                           color=color, template="plotly_white",
                           title=T["chart_pca_2d"],
                           render_mode="webgl", opacity=0.6, height=600)
         fig1.update_traces(marker_size=3)
-
-    # Update axis labels to reflect weighted nature
-    fig1.update_xaxes(title_text=T.get("lbl_weighted_pc1", "Weighted PC1 (Physics-Adaptive)"))
-    fig1.update_yaxes(title_text=T.get("lbl_weighted_pc2", "Weighted PC2 (Physics-Adaptive)"))
-    if ndim == "3D":
-        fig1.update_layout(scene=dict(
-            xaxis_title=T.get("lbl_weighted_pc1", "Weighted PC1"),
-            yaxis_title=T.get("lbl_weighted_pc2", "Weighted PC2"),
-            zaxis_title=T.get("lbl_weighted_pc3", "Weighted PC3")
-        ))
+        fig1.update_layout(
+            xaxis_title="PC1",
+            yaxis_title="PC2"
+        )
 
     fig1.update_layout(legend=dict(font=dict(size=9)))
 
@@ -1307,20 +1168,41 @@ def chart3(ndim, color_by, filter_type, topn_str, active_data):
     Output("c3-variance-container", "style"),
     Output("c3-var-toggle", "children"),
     Input("c3-var-toggle", "n_clicks"),
-    prevent_initial_call=True,
+    Input("c3-ndim", "value"),
 )
-def toggle_variance(n):
-    if n and n % 2 == 1:
+def toggle_variance(n, ndim):
+    # Hide variance in UMAP mode (not meaningful for UMAP)
+    if ndim == "UMAP_2D":
+        return (
+            {"flex": "1", "minWidth": 0},
+            {"flex": "1", "minWidth": 0, "display": "none"},
+            T["chart_variance"] + " \u25b6",
+        )
+
+    # Existing toggle logic for PCA modes
+    ctx = dash.callback_context
+    if not ctx.triggered or ctx.triggered[0]["prop_id"] == "c3-ndim.value":
+        # Initial state or mode change: hide variance
+        return (
+            {"flex": "1", "minWidth": 0},
+            {"flex": "1", "minWidth": 0, "display": "none"},
+            T["chart_variance"] + " \u25b6",
+        )
+
+    # Toggle on click
+    is_hidden = (n or 0) % 2 == 0
+    if is_hidden:
         return (
             {"flex": "2", "minWidth": 0},
             {"flex": "1", "minWidth": 0},
             T["chart_variance"] + " \u25c0",
         )
-    return (
-        {"flex": "1", "minWidth": 0},
-        {"flex": "1", "minWidth": 0, "display": "none"},
-        T["chart_variance"] + " \u25b6",
-    )
+    else:
+        return (
+            {"flex": "1", "minWidth": 0},
+            {"flex": "1", "minWidth": 0, "display": "none"},
+            T["chart_variance"] + " \u25b6",
+        )
 
 
 # ── 4 Lookup table ────────────────────────────────────────────────────────

material_universe_cache/plotly_studio_export.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25ff56bbfd44a4d2d7db5af566fe2990d780a9c696ba2f50ca10ba8dfdb5e6d6
-size 20636286
+oid sha256:28d5a2616fc8dcd9469c0303f64b4eb61bb8c9bb07c8f614281076a93e87113e
+size 13630877

material_universe_cache/umap_2d_balanced.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:677b4fcdc6bd77071d7d8cc0608690f07cfbd34fceb44b75cf149a1ee375c5e9
+size 271912

material_universe_cache/umap_2d_chemical.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d1f19ef7c3de86c290a95a9795e3da53ceaec3ac7376ceb8b18953e80d2f952
+size 271912

material_universe_cache/umap_2d_coord_energy.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69df0550f20b0afff1f02c81fa216dc07b808ea22b9a2e98e4d441db4a692b89
+size 271912

material_universe_cache/umap_2d_electronic.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:473c16cea222156875cc5200b74fc21f835b7438c79b605901a6b6b023047b95
+size 271912

material_universe_cache/umap_2d_esen_only.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccd018765d5116fe982881e69fde569cabd1ca72ab5e418948e6dbb703d142fa
+size 271912

material_universe_cache/umap_2d_mechanochem.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:607a5d46e9bb74c38cdf1e87648e4bf2414811a6d56a9cc63d318ca73b10e93a
+size 271912

material_universe_cache/umap_2d_mm_only.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:990370651b0f798c0271428535b6dc09003138cd1ff7685146f65202aaecab40
+size 271912

material_universe_cache/umap_2d_ofm_only.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80133c0680f7028b2007745f1da99cb47cfceaee942f3e2b076a016ee2cdf0d9
+size 271912

material_universe_cache/umap_2d_orb_only.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65be8b7d5e99d94f6f7b07bb9c5df689a5a9ccb2e18c58f2f6e52e2e0b471f39
+size 271912

material_universe_cache/umap_2d_stability.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4653177a621333abe5bf0d219c915f1e6552ab9718da0e0921a74839346c991d
+size 271912

material_universe_cache/umap_2d_structural.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32874494dd779ca1674029c00db116144748118b44c14faab82686f93e518b87
+size 271912