[FEAT/FIX] tab4: fix variant deactivation on ZIP reload, relocate helpers to utils, add persistent Reset tab

tabs/tab4_optimization.py

@@ -10,31 +10,16 @@ import numpy as np
 import pandas as pd
 import streamlit as st
 import plotly.express as px
-from config.config import CRRA_COLORS, PLOTLY_COLOR_SEQUENCE, PLOTLY_HEATMAP_SCALE, resource_order, resource_unit_map, resource_groups, secondary_resources, CO2_UNIT
+from config.config import CRRA_COLORS, PLOTLY_COLOR_SEQUENCE, PLOTLY_HEATMAP_SCALE, resource_order, resource_unit_map, resource_groups, secondary_resources, CO2_UNIT, method_order
 from optimization.optimization import run_optimization
 from utils.ui_helpers import apply_chart_style, register_chart_data, show_bar
-from utils.data_utils import is_valid_scenario, is_complete_scenario, load_inputs_from_excel, neutralize_zero_coefficient_methods, strip_unit_suffix, get_resource_to_group, get_resource_to_unit, is_other_method, get_other_method_variants, make_variant_name
+from utils.data_utils import (is_valid_scenario, is_complete_scenario, load_inputs_from_excel,
+                              neutralize_zero_coefficient_methods, strip_unit_suffix,
+                              get_resource_to_group, get_resource_to_unit,
+                              is_other_method, get_other_method_variants, make_variant_name,
+                              default_constraint as _dc, to_json_safe, build_results_df)
+from utils.state_utils import clear_widget_keys
 from config.config import hidden_resource_names as _hidden
-from tabs.tab2_constraints import default_constraint as _dc
-
-def to_json_safe(obj):
-    """Recursively convert numpy/pandas types to plain Python so json.dumps works."""
-    if isinstance(obj, dict):
-        return {k: to_json_safe(v) for k, v in obj.items()}
-    if isinstance(obj, list):
-        return [to_json_safe(v) for v in obj]
-    if isinstance(obj, pd.Series):
-        return obj.iloc[0] if len(obj) == 1 else obj.tolist()
-    if isinstance(obj, pd.DataFrame):
-        return obj.to_dict(orient="records")
-    if isinstance(obj, np.integer):
-        return int(obj)
-    if isinstance(obj, np.floating):
-        return float(obj)
-    if isinstance(obj, np.ndarray):
-        return obj.tolist()
-    return obj
-
 
 def panel_run_from_tabs():
     """Left sub-panel: run optimization from manually filled tabs."""
@@ -50,19 +35,7 @@ def panel_load_from_excel():
     with col_markdown:
         st.markdown("**Option 2 - From an Excel file**")
 
-    with col_upload:
-        with open("data/inputs_template.xlsx", "rb") as f:
-            st.download_button(
-                label="Excel Template",
-                data=f,
-                file_name="crra_input_template.xlsx",
-                mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-                width='stretch',
-                icon=":material/download:",
-                key="download_button_input_template"
-            )
-
-    st.markdown('<p class="caption">Fill in the template with your values, then import, load inputs and run.</p>', unsafe_allow_html=True)
+    st.markdown('<p class="caption">Upload an Excel file previously exported from the tool to reload all inputs at once.</p>', unsafe_allow_html=True)
 
     if "excel_loaded_file" in st.session_state:
         st.success(f"Loaded: **{st.session_state['excel_loaded_file']}** - All tabs updated.", icon="✅")
@@ -214,13 +187,17 @@ def parse_zip_scenario(uploaded) -> dict | None:
                     for _, row in df_enabled.iterrows():
                         enabled_methods.setdefault(str(row["resource"]), {})[str(row["method"])] = bool(row["enabled"])
 
-            safe_constraints = neutralize_zero_coefficient_methods(
-                st.session_state.costs, st.session_state.constraints
-            )
+            # Use the loaded costs/constraints, not the old session state.
+            # Old state may contain variant entries (e.g. "Method: Batch") from a
+            # previous scenario; using them here causes false deactivation.
+            # Guard: only report standard methods so variants never appear.
+            _std = set(method_order)
+            safe_constraints = neutralize_zero_coefficient_methods(costs, constraints)
             neutralized = [
                 m for m in safe_constraints
                 if safe_constraints[m].get("active") is False
-                and st.session_state.constraints.get(m, {}).get("active") is True
+                and constraints.get(m, {}).get("active") is True
+                and m in _std
             ]
             
             # Recompute detailed outputs from the reloaded inputs so the scenario
@@ -313,7 +290,7 @@ def panel_load_from_json_or_csv():
                 data["resource_actual_gain"] = result.get("resource_actual_gain", {})
                 data.setdefault("enabled_methods", {})
             else:
-                st.warning("Scenario loaded but detailed outputs could not be recomputed.")
+                st.warning("Scenario loaded but detailed outputs could not be recomputed.", icon=":material/warning:")
     else:
         data = parse_zip_scenario(uploaded_json)
         if data is None:
@@ -333,7 +310,10 @@ def panel_load_from_json_or_csv():
         st.session_state.pop("resource_last_loaded_file", None)
         st.session_state.resource_caps = data["resource_caps"].copy()
         st.session_state.constraints = data["method_constraints"].copy()
-        st.session_state.costs = data["method_costs"].copy()
+        # Use pre-expansion costs (has originals, no variants) so the next run
+        # can expand "Other" methods correctly without skipping the resource check.
+        # Fall back to method_costs for scenarios saved before this fix.
+        st.session_state.costs = data.get("method_costs_base", data["method_costs"]).copy()
         # Restore custom resource batches if present in the scenario
         st.session_state.custom_resources = data.get("custom_resources", [])
         st.session_state.enabled_methods = data.get("enabled_methods", {})
@@ -357,23 +337,9 @@ def panel_load_from_json_or_csv():
         st.session_state["excel_uploader_key"] += 1 # force reload in excel uploader
         if st.session_state.custom_resources:
             n = len(st.session_state.custom_resources)
-            st.info(f"ℹ️ {n} new resource(s) restored from scenario.")
+            st.info(f"{n} new resource(s) restored from scenario.", icon=":material/info:")
         st.rerun()
 
-def clear_widget_keys():
-    """Delete all Streamlit widget state keys for sliders, manual inputs, and toggles.
-
-    Called after loading a new file or scenario so widgets re-initialise from
-    fresh session-state values instead of showing stale cached values.
-
-    Returns:
-        None
-    """
-    prefixes = ("toggle_", "cap_type_", "slider_", "manual_")
-    for k in [k for k in st.session_state if k.startswith(prefixes)]:
-        del st.session_state[k]
-
-
 def run_and_store_result():
     """Run the LP optimisation from current session-state inputs and store the result.
 
@@ -395,7 +361,7 @@ def run_and_store_result():
 
     custom_total = sum(float(b["amount"]) for b in custom_resources)
     if std_total <= 0 and custom_total <= 0:
-        st.warning("⚠️ All resources are set to 0. Please set resource availability or load a scenario / input data.")
+        st.warning("⚠️ All resources are set to 0. Please set resource availability or load a scenario / input data.", icon=":material/warning:")
         return
     
     # Apply enabled_methods overrides to a temporary copy — session state is NOT
@@ -445,15 +411,15 @@ def run_and_store_result():
             safe_constraints.setdefault(m, {"active": True, "cap_type": "percent", "cap_value": 100})
             safe_constraints[m] = {**safe_constraints[m], "active": False}
 
-    neutralized = [
-        m for m in safe_constraints
-        if safe_constraints[m].get("active") is False
-        and st.session_state.constraints.get(m, {}).get("active") is True
-    ]
-
     # Expand "Other" methods: replace each with one variant per assigned batch.
     # Each variant uses the batch as its primary resource (standard LP constraint)
     # instead of the z-variable substitution mechanism.
+    #
+    # Snapshot taken before expansion: originals are present, variants absent.
+    # Stored in the result so scenario reload can restore the original method costs
+    # (post-expansion applied_costs deletes originals and adds variants, which would
+    # cause the variants to bypass the missing-resources check on the next load).
+    method_costs_base = {m: dict(v) for m, v in applied_costs.items()}
     resource_caps_for_lp = dict(st.session_state.resource_caps)
     custom_resources_for_lp = []
     expanded_batch_names: set[str] = set()
@@ -492,6 +458,17 @@ def run_and_store_result():
         del applied_costs[m]
         del safe_constraints[m]
 
+    # Compute after expansion so "Other" methods (deleted above) don't appear.
+    # Guard: only include standard methods so dynamically created variants (not in
+    # method_order) never show up in this message even in edge cases.
+    _standard_methods = set(method_order)
+    neutralized = [
+        m for m in safe_constraints
+        if safe_constraints[m].get("active") is False
+        and st.session_state.constraints.get(m, {}).get("active") is True
+        and m in _standard_methods
+    ]
+
     # LP receives only non-expanded batches.  Expanded batches are now standard
     # resources (in resource_caps_for_lp) so they get a type-A constraint — if
     # they stayed in custom_resources they'd land in custom_batch_names, be
@@ -550,7 +527,7 @@ def run_and_store_result():
     total_removed = result["total_removed"]
 
     if total_removed <= 0:
-        st.warning(f"❌ Optimization removed 0 {CO2_UNIT}. Relax constraints or increase resources.")
+        st.warning(f"❌ Optimization removed 0 {CO2_UNIT}. Relax constraints or increase resources.", icon=":material/warning:")
         return
 
     total_cost = sum(
@@ -567,11 +544,12 @@ def run_and_store_result():
         "resource_caps": resource_caps_for_lp,
         "method_constraints": safe_constraints,
         "method_costs": applied_costs,
+        "method_costs_base": method_costs_base,
         "custom_resources": custom_resources_for_display,
         "enabled_methods": enabled_methods,
     }
     if custom_resources:
-        st.info(f"ℹ️ {len(custom_resources)} new resource(s) included in optimisation.")
+        st.info(f"{len(custom_resources)} new resource(s) included in optimisation.", icon=":material/info:")
 
     # Warn when an active method has a primary resource whose coefficient is so
     # small (< 1e-7) that it is effectively unconstrained — the LP will run the
@@ -633,40 +611,6 @@ def show_metrics(latest):
                 })
             st.dataframe(pd.DataFrame(rows), width='stretch', hide_index=True)
 
-def build_results_df(allocations, method_constraints=None):
-    """Build a DataFrame of per-method CO₂ allocations with share and cap columns.
-
-    Args:
-        allocations (dict): {method: CO₂ removed (float)}.
-        method_constraints (dict | None): {method: {cap_type, cap_value}} for the Cap column.
-
-    Returns:
-        pd.DataFrame: indexed by method, columns CO₂ removed, Total share (%), Cap.
-                      Sorted descending by CO₂ removed.
-    """
-    df = pd.DataFrame.from_dict(allocations, orient="index", columns=[f"{CO2_UNIT} removed"])
-
-    df[f"{CO2_UNIT} removed"] = pd.to_numeric(df[f"{CO2_UNIT} removed"], errors="coerce")
-    total = df[f"{CO2_UNIT} removed"].sum()
-    if total > 0:
-        df["Total share (%)"] = (df[f"{CO2_UNIT} removed"] / total * 100).round(2)
-
-    if method_constraints:
-        caps = {}
-        for m in df.index:
-            c = method_constraints.get(m, {})
-            
-            if c.get("cap_type") == "absolute":
-                caps[m] = f"{c['cap_value']:,.2f} {CO2_UNIT}/yr"
-            elif c.get("cap_type") == "percent":
-                caps[m] = f"{int(c.get('cap_value', 100))}% of potential"
-            else:
-                caps[m] = f"⚠️ Invalid type ({c.get('cap_type')!r}) — fix in Tab 2"
-        df["Cap"] = pd.Series(caps)
-    
-    return df.sort_values(f"{CO2_UNIT} removed", ascending=False)
-
-
 def show_donut(df_result):
     """Render a donut chart of CO₂ removal by method and register its data.
 
@@ -715,7 +659,7 @@ def show_heatmap(df_result, resource_caps, method_costs, custom_resources=None,
     available = standard_available + extra_available
 
     if not available:
-        st.info("No resources with non-zero availability to display.")
+        st.info("No resources with non-zero availability to display.", icon=":material/info:")
         return
 
     matrix_pct, matrix_exact, y_labels = [], [], []
@@ -744,7 +688,7 @@ def show_heatmap(df_result, resource_caps, method_costs, custom_resources=None,
         df_heat, color_continuous_scale=PLOTLY_HEATMAP_SCALE, text_auto=".1f",
         labels=dict(x="Resource", y="Method", color="% Used"),
     )
-    matrix_exact_fmt = [[f"{v:.15f}".rstrip("0").rstrip(".") for v in row] for row in matrix_exact]
+    matrix_exact_fmt = [[f"{v:.10g}" for v in row] for row in matrix_exact]
     fig.update_traces(
         customdata=matrix_exact_fmt,
         hovertemplate="<b>%{y}</b><br>%{x}<br>% of cap: %{z:.1f}%<br>Exact: %{customdata}<extra></extra>",
@@ -956,7 +900,7 @@ def show_bottleneck_analysis(latest):
         st.dataframe(df_sp, width='stretch', hide_index=True)
         register_chart_data("Resource sensitivity (CDR gain per +1 unit)", df_sp)
     else:
-        st.info("No resources to analyse.")
+        st.info("No resources to analyse.", icon=":material/info:")
 
 def save_scenario(latest):
     """Render the save-scenario widget and persist the current result to session state.
@@ -990,9 +934,9 @@ def save_scenario(latest):
     if save_clicked:
         name = st.session_state.get("scenario_name", "").strip()
         if not name:
-            st.warning("Please enter a name.")
+            st.warning("Please enter a name.", icon=":material/warning:")
         elif name in st.session_state.scenarios:
-            st.warning("Name already exists.")
+            st.warning("Name already exists.", icon=":material/warning:")
         else:
             st.session_state.scenarios[name] = latest
             st.success(f"Scenario '{name}' saved for comparison.")
@@ -1265,7 +1209,7 @@ def export_all_charts():
 
     registry = st.session_state.get("chart_data_registry", {})
     if not registry:
-        st.info("No charts have been generated yet in this session.")
+        st.info("No charts have been generated yet in this session.", icon=":material/info:")
         return
 
     base_name = st.text_input(
@@ -1352,17 +1296,16 @@ def confirm_clear_all():
         if st.button("Cancel", width='stretch', type="secondary", key="btn-cancel"):
             st.rerun()
 
-def reset_scenario():
+def reset_scenario(tab_name):
     """Render the reset section with a button that opens the confirm_clear_all dialog.
 
     Returns:
         None
     """
-    st.divider()
     st.markdown('<p class="crra-sub-heading">Reset</p>', unsafe_allow_html=True)
     st.markdown('<p class="caption">Clear all current inputs, results and saved scenarios from the session.</p>', unsafe_allow_html=True)
 
-    if st.button("Clear all data", key="reset_scenario", type="primary", icon=":material/delete:"):
+    if st.button("Clear all data", key=f"reset_scenario_{tab_name}", type="primary", icon=":material/delete:"):
         confirm_clear_all()
 
 
@@ -1373,7 +1316,7 @@ def render_export_tab(latest):
     export_scenario()
     export_inputs_only()
     export_all_charts()
-    reset_scenario()
+    reset_scenario("export")
 
 
 def render_tab():
@@ -1415,12 +1358,13 @@ def render_tab():
     if "enabled_methods" not in st.session_state:
         st.session_state.enabled_methods = {}
 
-    tab_run, tab_results, tab_resources, tab_bottleneck, tab_export = st.tabs([
+    tab_run, tab_results, tab_resources, tab_bottleneck, tab_export, tab_reset = st.tabs([
         "▶️ Run",
         "📊 Method allocation",
         "🌿 Resource usage",
         "🔍 Bottleneck analysis",
         "💾 Export & Save",
+        "🔄 Reset"
     ])
 
     with tab_run:
@@ -1451,13 +1395,15 @@ def render_tab():
             with run_msg_area:
                 run_and_store_result()
 
-    _no_results = '<p class="caption">Run the optimization first to see results here.</p>'
+    no_results = '<p class="caption">Run the optimization first to see results here.</p>'
 
     latest = st.session_state.get("latest_result")
     if not latest:
-        for _tab in [tab_results, tab_resources, tab_bottleneck, tab_export]:
-            with _tab:
-                st.markdown(_no_results, unsafe_allow_html=True)
+        for tab in [tab_results, tab_resources, tab_bottleneck, tab_export]:
+            with tab:
+                st.markdown(no_results, unsafe_allow_html=True)
+        with tab_reset:
+            reset_scenario("reset")
         return
 
     allocations = latest["allocations"]
@@ -1465,9 +1411,11 @@ def render_tab():
     method_costs = latest["method_costs"]
 
     if not allocations or sum(allocations.values()) == 0:
-        for _tab in [tab_results, tab_resources, tab_bottleneck, tab_export]:
-            with _tab:
-                st.markdown(_no_results, unsafe_allow_html=True)
+        for tab in [tab_results, tab_resources, tab_bottleneck, tab_export]:
+            with tab:
+                st.markdown(no_results, unsafe_allow_html=True)
+        with tab_reset:
+            reset_scenario("reset")
         return
 
     df_result = build_results_df(
@@ -1501,4 +1449,7 @@ def render_tab():
         show_bottleneck_analysis(latest)
 
     with tab_export:
-        render_export_tab(latest)
+        render_export_tab(latest)
+
+    with tab_reset:
+        reset_scenario("reset")