Update src/streamlit_app.py

src/streamlit_app.py

@@ -37,13 +37,30 @@ ENSEMBLE_ARTIFACT = os.path.join(DATA_DIR, "ensemble_models.joblib")
 # -------------------------
 # Utility: generate advanced dataset if missing
 # -------------------------
-def generate_advanced_flatfile(n_rows=3000, random_seed=42, max_polynomial_new=60):
+def generate_advanced_flatfile(
+    n_rows=3000,
+    random_seed=42,
+    max_polynomial_new=60,
+    global_variance_multiplier=1.0,
+    variance_overrides=None,
+):
     """
     Generates a large synthetic, physics-aligned dataset with many engineered features.
-    Saves CSV and metadata JSON and a short annotated bibliography PDF (text).
+    Allows control of variability per feature (through variance_overrides) or globally
+    (via global_variance_multiplier).
+
+    Args:
+        n_rows: number of samples
+        random_seed: RNG seed
+        max_polynomial_new: limit on number of polynomial expansion features
+        global_variance_multiplier: multiplier applied to all default stddevs
+        variance_overrides: dict mapping feature name or substring → stddev multiplier
     """
     np.random.seed(random_seed)
     os.makedirs(DATA_DIR, exist_ok=True)
+    if variance_overrides is None:
+        variance_overrides = {}
+
     # --- base natural features across 8 use cases (expanded)
     natural_feats = [
         "vibration_x","vibration_y","motor_current","rpm","bearing_temp","ambient_temp","lube_pressure","power_factor",
@@ -55,49 +72,76 @@ def generate_advanced_flatfile(n_rows=3000, random_seed=42, max_polynomial_new=6
         "batch_id_numeric","weight_input","weight_output","time_in_queue","conveyor_speed",
         "shell_temp","lining_thickness","water_flow","cooling_out_temp","heat_flux"
     ]
-    # dedupe if duplicated names
-    natural_feats = list(dict.fromkeys(natural_feats))
+    natural_feats = list(dict.fromkeys(natural_feats))  # dedupe
+
+    # helper: compute adjusted stddev
+    def effective_sd(feature_name, base_sd):
+        # exact name override
+        if feature_name in variance_overrides:
+            return float(variance_overrides[feature_name])
+        # substring override
+        for key, val in variance_overrides.items():
+            if key in feature_name:
+                return float(val)
+        # fallback: scaled base
+        return float(base_sd) * float(global_variance_multiplier)
 
     # helper sampling heuristics
     def sample_col(name, n):
         name_l = name.lower()
         if "furnace_temp" in name_l or name_l.endswith("_temp") or "tap_temp" in name_l:
-            return np.random.normal(1550, 50, n)
+            sd = effective_sd("furnace_temp", 50)
+            return np.random.normal(1550, sd, n)
         if name_l in ("tap_temp","mold_temp","shell_temp","cooling_out_temp","exit_temp"):
-            return np.random.normal(200 if "mold" not in name_l else 1500, 30, n)
+            sd = effective_sd(name_l, 30)
+            return np.random.normal(200 if "mold" not in name_l else 1500, sd, n)
         if "offgas_co2" in name_l:
-            return np.abs(np.random.normal(15,4,n))
+            sd = effective_sd("offgas_co2", 4)
+            return np.abs(np.random.normal(15, sd, n))
         if "offgas_co" in name_l:
-            return np.abs(np.random.normal(20,5,n))
+            sd = effective_sd("offgas_co", 5)
+            return np.abs(np.random.normal(20, sd, n))
         if "o2" in name_l:
-            return np.clip(np.random.normal(5,1,n), 0.01, 60)
+            sd = effective_sd("o2_probe_pct", 1)
+            return np.clip(np.random.normal(5, sd, n), 0.01, 60)
         if "arc_power" in name_l or "motor_load" in name_l:
-            return np.abs(np.random.normal(600,120,n))
+            sd = effective_sd("arc_power", 120)
+            return np.abs(np.random.normal(600, sd, n))
         if "rpm" in name_l:
-            return np.abs(np.random.normal(120,30,n))
+            sd = effective_sd("rpm", 30)
+            return np.abs(np.random.normal(120, sd, n))
         if "vibration" in name_l:
-            return np.abs(np.random.normal(0.4,0.15,n))
+            sd = effective_sd("vibration", 0.15)
+            return np.abs(np.random.normal(0.4, sd, n))
         if "bearing_temp" in name_l:
-            return np.random.normal(65,5,n)
+            sd = effective_sd("bearing_temp", 5)
+            return np.random.normal(65, sd, n)
         if "chemical" in name_l or "spectro" in name_l:
-            return np.random.normal(0.7,0.15,n)
+            sd = effective_sd("chemical", 0.15)
+            return np.random.normal(0.7, sd, n)
         if "weight" in name_l:
-            return np.random.normal(1000,100,n)
+            sd = effective_sd("weight", 100)
+            return np.random.normal(1000, sd, n)
         if "conveyor_speed" in name_l or "casting_speed" in name_l:
-            return np.random.normal(2.5,0.6,n)
+            sd = effective_sd("casting_speed", 0.6)
+            return np.random.normal(2.5, sd, n)
         if "power_factor" in name_l:
-            return np.clip(np.random.normal(0.92,0.03,n),0.6,1.0)
+            sd = effective_sd("power_factor", 0.03)
+            return np.clip(np.random.normal(0.92, sd, n), 0.6, 1.0)
         if "image_entropy_proxy" in name_l:
-            return np.abs(np.random.normal(0.5,0.25,n))
+            sd = effective_sd("image_entropy_proxy", 0.25)
+            return np.abs(np.random.normal(0.5, sd, n))
         if "batch_id" in name_l:
             return np.random.randint(1000,9999,n)
         if "time_since" in name_l or "time_in_queue" in name_l:
-            return np.abs(np.random.normal(30,20,n))
+            sd = effective_sd("time_since", 20)
+            return np.abs(np.random.normal(30, sd, n))
         if "heat_flux" in name_l:
-            return np.abs(np.random.normal(1000,300,n))
-        return np.random.normal(0,1,n)
+            sd = effective_sd("heat_flux", 300)
+            return np.abs(np.random.normal(1000, sd, n))
+        return np.random.normal(0, effective_sd(name_l, 1), n)
 
-    # build DF
+    # build DataFrame
     df = pd.DataFrame({c: sample_col(c, n_rows) for c in natural_feats})
 
     # timestamps & metadata
@@ -129,19 +173,15 @@ def generate_advanced_flatfile(n_rows=3000, random_seed=42, max_polynomial_new=6
     df["carbon_power_ratio"] = df["carbon_proxy"] / (df["arc_power"] + 1e-6)
     df["temp_power_sqrt"] = df["furnace_temp"] * np.sqrt(np.abs(df["arc_power"]) + 1e-6)
 
-    # polynomial features limited to first 12 numeric columns to avoid explosion
+    # polynomial features limited to first 12 numeric columns
     numeric = df.select_dtypes(include=[np.number]).fillna(0)
     poly_source_cols = numeric.columns[:12].tolist()
     poly = PolynomialFeatures(degree=2, interaction_only=False, include_bias=False)
     poly_mat = poly.fit_transform(numeric[poly_source_cols])
     poly_names = poly.get_feature_names_out(poly_source_cols)
     poly_df = pd.DataFrame(poly_mat, columns=[f"poly__{n}" for n in poly_names], index=df.index)
-    # drop identical originals and limit new cols
     keep_poly = [c for c in poly_df.columns if c.replace("poly__","") not in poly_source_cols]
-    if len(keep_poly) > 0:
-      poly_df = poly_df[keep_poly].iloc[:, :max_polynomial_new]
-    else:
-      poly_df = poly_df.iloc[:, :0]
+    poly_df = poly_df[keep_poly].iloc[:, :max_polynomial_new] if len(keep_poly) > 0 else poly_df.iloc[:, :0]
     df = pd.concat([df, poly_df], axis=1)
 
     # PCA embeddings across numeric sensors
@@ -156,22 +196,19 @@ def generate_advanced_flatfile(n_rows=3000, random_seed=42, max_polynomial_new=6
     kmeans = KMeans(n_clusters=6, random_state=42, n_init=10)
     df["operating_mode"] = kmeans.fit_predict(scaled)
 
-    # surrogate models to create short-horizon predicted states (fast regressors)
-    # furnace_temp_next surrogate
+    # surrogate models
     surrogate_df = df.copy()
     surrogate_df["furnace_temp_next"] = surrogate_df["furnace_temp"].shift(-1).fillna(method="ffill")
     features_for_surrogate = [c for c in ["furnace_temp","arc_power","o2_probe_pct","offgas_co","offgas_co2"] if c in df.columns]
     if len(features_for_surrogate) >= 2:
         X = surrogate_df[features_for_surrogate].fillna(0)
         y = surrogate_df["furnace_temp_next"]
-        from sklearn.ensemble import RandomForestRegressor
         rf = RandomForestRegressor(n_estimators=50, random_state=42, n_jobs=-1)
         rf.fit(X, y)
         df["pred_temp_30s"] = rf.predict(X)
     else:
         df["pred_temp_30s"] = df["furnace_temp"]
 
-    # surrogate for carbon proxy
     if all(c in df.columns for c in ["offgas_co","offgas_co2","o2_probe_pct"]):
         X2 = df[["offgas_co","offgas_co2","o2_probe_pct"]].fillna(0)
         rf2 = RandomForestRegressor(n_estimators=50, random_state=1, n_jobs=-1)
@@ -184,7 +221,7 @@ def generate_advanced_flatfile(n_rows=3000, random_seed=42, max_polynomial_new=6
     df["refractory_limit_flag"] = (df["lining_thickness"] < 140).astype(int)
     df["max_allowed_power_delta"] = np.clip(df["arc_power"].diff().abs().fillna(0), 0, 2000)
 
-    # simple rule-based target action for demo
+    # rule-based target
     df["ARC_ON"] = ((df["arc_power"] > df["arc_power"].median()) & (df["carbon_proxy"] < 1.0)).astype(int)
     df["prediction_confidence"] = np.clip(np.random.beta(2,5, n_rows), 0.05, 0.99)
 
@@ -195,7 +232,6 @@ def generate_advanced_flatfile(n_rows=3000, random_seed=42, max_polynomial_new=6
 
     # save CSV & metadata
     df.to_csv(CSV_PATH, index=False)
-
     meta = []
     for col in df.columns:
         if col in natural_feats:
@@ -215,7 +251,7 @@ def generate_advanced_flatfile(n_rows=3000, random_seed=42, max_polynomial_new=6
     with open(META_PATH, "w") as f:
         json.dump(meta, f, indent=2)
 
-    # annotated bibliography text saved as simple PDF-like text (clients accept PDF)
+    # annotated bibliography
     try:
         from fpdf import FPDF
         pdf = FPDF('P','mm','A4')
@@ -241,9 +277,9 @@ def generate_advanced_flatfile(n_rows=3000, random_seed=42, max_polynomial_new=6
             pdf.ln(2)
         pdf.output(PDF_PATH)
     except Exception as e:
-        # fallback: simple text file
         with open(PDF_PATH.replace(".pdf",".txt"), "w") as tf:
             tf.write("Annotated bibliography generated. Install fpdf for PDF output.\n")
+
     return CSV_PATH, META_PATH, PDF_PATH
 
 # -------------------------
@@ -349,16 +385,59 @@ with tabs[4]:
         arc_power_sd = st.slider("Synthetic Arc Power σ (spread)", 50, 300, 120, step=10)
     st.markdown("---")
 
+    # --- Variance Controls UI ---
+    st.markdown("#### Variance controls (global & per-feature)")
+    global_var_mult = st.slider(
+        "Global variance multiplier", 0.1, 5.0, 1.0, step=0.1,
+        help="Multiply base standard deviations by this factor for all features."
+    )
+    
+    # Optional: choose features to override
+    feat_for_override = st.multiselect(
+        "Select features to override variance (optional)", numeric_cols, max_selections=8
+    )
+    variance_overrides = {}
+    if feat_for_override:
+        st.markdown("Set multipliers for selected features")
+        for f in feat_for_override:
+            mult = st.number_input(
+                f"Variance multiplier for {f}", min_value=0.1, max_value=10.0,
+                value=1.0, step=0.1, key=f"mult_{f}"
+            )
+            variance_overrides[f] = float(mult)
+    
+    st.markdown("---")
+    
+    # --- Regeneration button ---
     if st.button("Regenerate Synthetic Dataset with Updated Variance"):
         with st.spinner("Regenerating synthetic data..."):
+            variance_overrides.update({
+                "furnace_temp": furnace_temp_sd / 50,
+                "arc_power": arc_power_sd / 120
+            })
             CSV_PATH, META_PATH, PDF_PATH = generate_advanced_flatfile(
                 n_rows=3000,
-                random_seed=random_seed,
-                max_polynomial_new=60
+                random_seed=int(random_seed),
+                max_polynomial_new=60,
+                global_variance_multiplier=float(global_var_mult),
+                variance_overrides=variance_overrides,
             )
+    
+            # Clear cache and reload fresh
             st.cache_data.clear()
-            df, meta_df = load_data()
-            st.success("Synthetic dataset regenerated with new variance settings.")
+            df, meta_df = load_data(csv_path=CSV_PATH + f"?t={int(time.time())}", meta_path=META_PATH)
+    
+            numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+    
+            st.success(
+                f"Synthetic dataset regenerated — {len(df)} rows × {len(df.columns)} features "
+                f"(Global×{global_var_mult:.2f}; Overrides={len(variance_overrides)})"
+            )
+            st.caption(
+                f"Mean furnace_temp: {df['furnace_temp'].mean():.2f}, "
+                f"Std furnace_temp: {df['furnace_temp'].std():.2f}"
+            )
+
 
     if train_button:
         with st.spinner("Preparing data and training ensemble..."):