Update app.py

app.py

@@ -1,49 +1,141 @@
-# app_market_sim.py
+# app_market_sim_full.py
 # -*- coding: utf-8 -*-
-import streamlit as st
-import pandas as pd
-import numpy as np
+"""
+Simultaneous market clearing for Japan (10 areas) with a randomized generation fleet.
+- Energy + upward reserve co-optimization (linear program).
+- Unit-level random capacities/heat-rates/minimum outputs by fuel.
+- Robust timestamp parsing (format='mixed') for data.json time axis.
+- LMP = dual of energy-balance per area/hour (JPY/MWh).
+- Reserve price = dual of reserve requirement per area/hour (JPY/MW).
+- Includes "shed" (load shed, penalized by VOLL) and "spill" (over-generation dump with tiny penalty).
+
+How to run:
+    streamlit run app_market_sim_full.py
+
+Data requirement (data.json):
+    {
+      "solar": {"x": [...timestamps...], "y": [...cf...]},
+      "onshore_wind": {"x": [...], "y": [...]},
+      "offshore_wind": {"x": [...], "y": [...]},
+      "river": {"x": [...], "y": [...]},
+      "demand": {"x": [...], "y": [...]}   # demand hourly capacity factor
+    }
+"""
+
 import json
+from io import StringIO
+
+import numpy as np
+import pandas as pd
 import pulp
 import plotly.express as px
 import plotly.graph_objs as go
-from io import StringIO
+import streamlit as st
+
 
 # -----------------------------
-# Utility: load time-series
+# Time-series loader (robust)
 # -----------------------------
 def load_timeseries():
     """
-    Load data.json with keys:
-      { "<series>": {"x": [...timestamps...], "y": [...values...] }, ... }
-    Required series (national, used for all areas):
-      - solar hourly capacity factor
-      - onshore_wind hourly capacity factor
-      - offshore_wind hourly capacity factor
-      - river hourly capacity factor
-      - demand hourly capacity factor
+    Robustly load hourly time-series from data.json and parse timestamps.
+
+    What this does
+    --------------
+    - Strip stray whitespace/newlines from timestamp strings in 'x'.
+    - Parse timestamps with format='mixed' (ISO8601/tz/offset tolerated).
+    - Coerce parse failures to NaT and drop those rows consistently.
+    - Return tidy DataFrame indexed by 'Time'.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Index: Time (naive, Asia/Tokyo converted if tz present)
+        Columns:
+          'solar hourly capacity factor',
+          'onshore_wind hourly capacity factor',
+          'offshore_wind hourly capacity factor',
+          'river hourly capacity factor',
+          'demand hourly capacity factor'
+        plus any extra series present.
     """
     with open("data.json", "r", encoding="utf-8") as f_local:
         data_local = json.load(f_local)
     if not data_local:
-        raise RuntimeError("data.json is empty")
-
-    times_local = pd.to_datetime(data_local[next(iter(data_local))]["x"])
-    df_local = pd.DataFrame({"Time": times_local})
+        raise ValueError("data.json is empty")
+
+    # master timeline from first series
+    first_key_local = next(iter(data_local))
+    raw_times_local = data_local[first_key_local].get("x", [])
+    if not raw_times_local:
+        raise ValueError("data.json has no 'x' timeline")
+
+    cleaned_times = []
+    for v_local in raw_times_local:
+        if isinstance(v_local, str):
+            s_local = (
+                v_local.replace("\r", "")
+                .replace("\n", "")
+                .replace("\t", " ")
+                .replace("\u3000", " ")
+                .strip()
+            )
+            cleaned_times.append(s_local if s_local != "" else np.nan)
+        else:
+            cleaned_times.append(v_local)
+
+    time_parsed = pd.to_datetime(
+        cleaned_times, format="mixed", errors="coerce", utc=True
+    )
+    # Convert to JST and drop tz-awareness; fallbacks for naive data
+    try:
+        time_parsed = time_parsed.tz_convert("Asia/Tokyo").tz_localize(None)
+    except Exception:
+        try:
+            time_parsed = time_parsed.tz_localize(None)
+        except Exception:
+            pass
+
+    n_rows = len(cleaned_times)
+    df_out = pd.DataFrame(index=np.arange(n_rows))
+    df_out["Time"] = time_parsed
+
+    # load all Y series to columns, length-align then drop NaT rows
     for k_local, v_local in data_local.items():
         if isinstance(v_local, dict) and "y" in v_local:
-            df_local[f"{k_local} hourly capacity factor"] = pd.to_numeric(v_local["y"], errors="coerce").fillna(0.0)
-    return df_local.set_index("Time")
+            ser = pd.Series(v_local["y"]).reindex(range(n_rows))
+            ser = pd.to_numeric(ser, errors="coerce")  # CFs or weights
+            name = f"{k_local} hourly capacity factor"
+            df_out[name] = ser
+
+    mask = df_out["Time"].notna()
+    df_out = df_out.loc[mask].copy()
+    for c_local in df_out.columns:
+        if c_local != "Time":
+            df_out[c_local] = df_out[c_local].fillna(0.0)
+
+    df_out = df_out.sort_values("Time").drop_duplicates(subset=["Time"]).set_index("Time")
+    return df_out
+
 
 # -----------------------------
-# Variable cost calculator
+# Fuel price helpers
 # -----------------------------
-def var_costs_jpy_per_mwh(usd_jpy, lng_usd_per_mmbtu, coal_usd_per_ton, oil_usd_per_bbl,
-                           hr_gas_gj_per_mwh, hr_coal_gj_per_mwh, hr_oil_gj_per_mwh,
-                           nuc_varcost_override_jpy_per_mwh):
+def fuel_prices_jpy_per_gj(usd_jpy, lng_usd_per_mmbtu, coal_usd_per_ton, oil_usd_per_bbl):
     """
-    Compute JPY/MWh variable costs for thermal fuels.
-    1 MMBtu = 1.055056 GJ, 1 bbl ≈ 6.12 GJ, coal energy ≈ 25.12 GJ/t
+    Compute fuel prices in JPY/GJ for LNG, coal, oil.
+
+    Parameters
+    ----------
+    usd_jpy : float
+    lng_usd_per_mmbtu : float
+    coal_usd_per_ton : float
+    oil_usd_per_bbl : float
+
+    Returns
+    -------
+    dict
+        {'lng': JPY/GJ, 'coal': JPY/GJ, 'oil': JPY/GJ}
     """
     mmbtu_to_gj = 1.055056
     bbl_to_gj = 6.12
@@ -53,63 +145,82 @@ def var_costs_jpy_per_mwh(usd_jpy, lng_usd_per_mmbtu, coal_usd_per_ton, oil_usd_
     coal_usd_per_gj = coal_usd_per_ton / coal_gj_per_ton
     oil_usd_per_gj = oil_usd_per_bbl / bbl_to_gj
 
-    gas_jpy_mwh = gas_usd_per_gj * hr_gas_gj_per_mwh * usd_jpy
-    coal_jpy_mwh = coal_usd_per_gj * hr_coal_gj_per_mwh * usd_jpy
-    oil_jpy_mwh = oil_usd_per_gj * hr_oil_gj_per_mwh * usd_jpy
-    nuc_jpy_mwh = nuc_varcost_override_jpy_per_mwh
+    return {
+        "lng": float(gas_usd_per_gj * usd_jpy),
+        "coal": float(coal_usd_per_gj * usd_jpy),
+        "oil": float(oil_usd_per_gj * usd_jpy),
+    }
 
-    return {"lng": gas_jpy_mwh, "coal": coal_jpy_mwh, "oil": oil_jpy_mwh, "nuclear": nuc_jpy_mwh}
 
 # -----------------------------
 # Random fleet generator
 # -----------------------------
-def generate_fleet(regions, rng, counts_cfg, caps_cfg, hr_cfg, min_output_cfg, ren_unit_caps):
+def generate_fleet(regions_list, rng_obj, counts_cfg, caps_cfg, hr_cfg, min_frac_cfg, ren_caps_cfg):
     """
     Generate a unit-level fleet with realistic ranges.
 
+    Parameters
+    ----------
+    regions_list : list[str]
+    rng_obj : numpy.random.Generator
+    counts_cfg : dict[(region,str)->int]
+        Number of units per fuel/tech for each region.
+    caps_cfg : dict[str->(float,float)]
+        Capacity range [MW/unit] for thermal/nuclear, e.g. {'lng':(200,900),...}
+    hr_cfg : dict[str->(float,float)]
+        Heat-rate range [GJ/MWh] per fuel for random draw (nuclear ignored).
+    min_frac_cfg : dict[str->(float,float)]
+        Minimum output fraction range per fuel (e.g., nuclear (0.6,0.9)).
+    ren_caps_cfg : dict[str->(float,float)]
+        Capacity range for renewables per unit.
+
     Returns
     -------
-    pandas.DataFrame columns:
-      ['unit_id','region','tech','fuel','cap_MW','hr_GJ_per_MWh','min_frac','is_renew','cf_key']
+    pandas.DataFrame
+        Columns: ['unit_id','region','tech','fuel','cap_MW','hr_GJ_per_MWh','min_frac','is_renew','cf_key']
     """
-    rows_local = []
-    uid_local = 0
+    rows = []
+    uid = 0
 
     # Thermal & Nuclear
-    for r_local in regions:
-        for fuel_local in ["lng", "coal", "oil", "nuclear"]:
-            n_units = counts_cfg[(r_local, fuel_local)]
-            cap_min, cap_max = caps_cfg[fuel_local]
-            hr_min, hr_max = hr_cfg[fuel_local]
-            min_frac_rng = min_output_cfg[fuel_local]
+    for r_loc in regions_list:
+        for fuel in ["lng", "coal", "oil", "nuclear"]:
+            n_units = int(counts_cfg[(r_loc, fuel)])
+            cap_lo, cap_hi = caps_cfg[fuel]
+            hr_lo, hr_hi = hr_cfg.get(fuel, (np.nan, np.nan))
+            min_lo, min_hi = min_frac_cfg[fuel]
             for _ in range(n_units):
-                cap = rng.uniform(cap_min, cap_max)
-                hr = rng.uniform(hr_min, hr_max) if fuel_local != "nuclear" else np.nan
-                min_frac = rng.uniform(min_frac_rng[0], min_frac_rng[1])
-                rows_local.append({
-                    "unit_id": f"U{uid_local}",
-                    "region": r_local,
-                    "tech": fuel_local,
-                    "fuel": fuel_local,
+                cap = rng_obj.uniform(cap_lo, cap_hi)
+                hr = rng_obj.uniform(hr_lo, hr_hi) if fuel != "nuclear" else np.nan
+                min_frac = rng_obj.uniform(min_lo, min_hi)
+                rows.append({
+                    "unit_id": f"U{uid}",
+                    "region": r_loc,
+                    "tech": fuel,
+                    "fuel": fuel,
                     "cap_MW": float(cap),
-                    "hr_GJ_per_MWh": (float(hr) if fuel_local != "nuclear" else np.nan),
+                    "hr_GJ_per_MWh": (float(hr) if fuel != "nuclear" else np.nan),
                     "min_frac": float(min_frac),
                     "is_renew": False,
                     "cf_key": None
                 })
-                uid_local += 1
-
-    # Renewables (as unit blocks with zero var cost, output <= CF*cap)
-    for r_local in regions:
-        for ren_key, cf_key in [("solar","solar"), ("onshore_wind","onshore_wind"),
-                                ("offshore_wind","offshore_wind"), ("river","river")]:
-            n_units = counts_cfg[(r_local, ren_key)]
-            cap_min, cap_max = ren_unit_caps[ren_key]
+                uid += 1
+
+    # Renewables
+    for r_loc in regions_list:
+        for ren_key, cf_key in [
+            ("solar", "solar"),
+            ("onshore_wind", "onshore_wind"),
+            ("offshore_wind", "offshore_wind"),
+            ("river", "river")
+        ]:
+            n_units = int(counts_cfg[(r_loc, ren_key)])
+            cap_lo, cap_hi = ren_caps_cfg[ren_key]
             for _ in range(n_units):
-                cap = rng.uniform(cap_min, cap_max)
-                rows_local.append({
-                    "unit_id": f"U{uid_local}",
-                    "region": r_local,
+                cap = rng_obj.uniform(cap_lo, cap_hi)
+                rows.append({
+                    "unit_id": f"U{uid}",
+                    "region": r_loc,
                     "tech": ren_key,
                     "fuel": None,
                     "cap_MW": float(cap),
@@ -118,220 +229,241 @@ def generate_fleet(regions, rng, counts_cfg, caps_cfg, hr_cfg, min_output_cfg, r
                     "is_renew": True,
                     "cf_key": f"{cf_key} hourly capacity factor"
                 })
-                uid_local += 1
+                uid += 1
+
+    return pd.DataFrame(rows)
 
-    df_units = pd.DataFrame(rows_local)
-    return df_units
 
 # -----------------------------
-# Simultaneous market clearing (energy + reserve)
+# Simultaneous market clearing
 # -----------------------------
-def clear_market(df_units, ts_df_slice, regions, demand_shares, voll_jpy_per_mwh,
-                 reserve_ratio, varcost_map, battery_eff=0.9):
+def clear_market(df_units, ts_df_slice, regions_list, demand_shares_map, voll_jpy_per_mwh,
+                 reserve_ratio, fuel_price_jpy_per_gj_map, nuclear_varcost_jpy_per_mwh,
+                 spill_penalty=1e-6):
     """
     Co-optimize energy and upward reserve simultaneously for each region and hour.
 
-    Decision variables:
-      g[u,t]         Generation (MW)
-      r[u,t]         Upward reserve (MW), eligible only for non-renew units
-      shed[r,t]      Load shedding (MW), penalized by VOLL
-    Constraints:
-      - Energy balance (per region, per hour): sum(g) + shed == demand
-      - Reserve requirement: sum(r) >= reserve_ratio * demand
-      - Unit limits:  min_frac*cap <= g <= cap (thermal/nuclear), g <= CF*cap (renew)
-                      0 <= r <= cap - g  (renew not eligible for r)
-    Objective:
-      Minimize sum(c_u * g + VOLL * shed)
+    Decision variables
+    ------------------
+    g[u,t]     : Generation (MW)
+    r[u,t]     : Upward reserve (MW), eligible only for non-renew units
+    shed[r,t]  : Load shedding (MW), penalized by VOLL
+    spill[r,t] : Over-generation dump (MW), tiny penalty to keep feasibility with must-run
+
+    Constraints
+    -----------
+    Energy balance (per region,t):
+        sum_u g[u,t] + shed[r,t] == demand[r,t] + spill[r,t]
+    Reserve requirement (per region,t):
+        sum_u r[u,t] >= reserve_ratio * demand[r,t]
+        Implemented as -sum r <= -req (so dual >= 0)
+    Unit limits:
+        Thermal/Nuclear: min_frac*cap <= g <= cap,  0 <= r <= cap - g
+        Renewables: 0 <= g <= CF*cap, r == 0
+
+    Objective
+    ---------
+    Minimize sum( var_cost_unit * g + VOLL * shed + spill_penalty * spill )
 
     Returns
     -------
-    dict with:
-      lmp_df: DataFrame [Time x regions] LMP (JPY/MWh)
-      res_price_df: DataFrame [Time x regions] Reserve price (JPY/MW)
-      dispatch_df: long table with g[u,t]
+    dict
+        {
+          'lmp_df': DataFrame [Time x regions] LMP (JPY/MWh),
+          'res_price_df': DataFrame [Time x regions] Reserve price (JPY/MW),
+          'dispatch_df': long DataFrame with unit-level g and r,
+        }
     """
     times = ts_df_slice.index
     T = len(times)
 
-    # Build regional demand
+    # Regional demand (relative scale from demand CF)
     base_cf = ts_df_slice["demand hourly capacity factor"].values
-    # Normalize to peak=1 and scale so sum(base)=sum(base)  => here we just use proportional shares
-    base_profile = base_cf / base_cf.max()
-    # Set national energy (MWh) equal to sum(base_profile) [MW] so demand is in MW consistently
-    # We scale to an arbitrary national peak of 1 MW-equivalent then allocate shares; absolute levels cancel in prices if varcost scale holds.
-    national_demand = base_profile  # MW proxy
-    demand = {r: national_demand * demand_shares[r] for r in regions}
-
-    # Build model
+    base_profile = base_cf / max(base_cf.max(), 1e-9)  # normalize peak=1
+    national_demand = base_profile  # MW proxy scale
+    demand = {r: national_demand * float(demand_shares_map[r]) for r in regions_list}
+
+    # Precompute per-unit variable costs (JPY/MWh) using unit heat-rates
+    vc_unit = {}
+    for u in df_units.index:
+        row_u = df_units.loc[u]
+        if row_u["is_renew"]:
+            vc_unit[u] = 0.0
+        else:
+            if row_u["fuel"] == "nuclear":
+                vc_unit[u] = float(nuclear_varcost_jpy_per_mwh)
+            else:
+                p_gj = float(fuel_price_jpy_per_gj_map[row_u["fuel"]])  # JPY/GJ
+                hr = float(row_u["hr_GJ_per_MWh"])
+                vc_unit[u] = p_gj * hr  # JPY/MWh
+
+    # Model
     mdl = pulp.LpProblem("CoOptim_Energy_Reserve", pulp.LpMinimize)
 
-    # Index maps
-    units_by_region = {r: df_units[df_units["region"] == r].index.tolist() for r in regions}
+    # Indices
+    units_by_region = {r: df_units[df_units["region"] == r].index.tolist() for r in regions_list}
 
     # Variables
-    g = pulp.LpVariable.dicts("g", ((u, t) for u in df_units.index for t in range(T)), lowBound=0, cat="Continuous")
-    r_up = pulp.LpVariable.dicts("r", ((u, t) for u in df_units.index for t in range(T)), lowBound=0, cat="Continuous")
-    shed = pulp.LpVariable.dicts("shed", ((r, t) for r in regions for t in range(T)), lowBound=0, cat="Continuous")
+    g = pulp.LpVariable.dicts("g", ((int(u), int(t)) for u in df_units.index for t in range(T)),
+                              lowBound=0, cat="Continuous")
+    r_up = pulp.LpVariable.dicts("r", ((int(u), int(t)) for u in df_units.index for t in range(T)),
+                                 lowBound=0, cat="Continuous")
+    shed = pulp.LpVariable.dicts("shed", ((r, int(t)) for r in regions_list for t in range(T)),
+                                 lowBound=0, cat="Continuous")
+    spill = pulp.LpVariable.dicts("spill", ((r, int(t)) for r in regions_list for t in range(T)),
+                                  lowBound=0, cat="Continuous")
 
     # Objective
-    cost_terms = []
+    obj_terms = []
     for u in df_units.index:
-        row = df_units.loc[u]
-        vc = 0.0
-        if not row["is_renew"]:
-            fuel = row["fuel"]
-            if fuel == "nuclear":
-                vc = varcost_map["nuclear"]
-            else:
-                vc = varcost_map[fuel]
+        c_u = float(vc_unit[u])
         for t in range(T):
-            cost_terms.append(vc * g[(u, t)])
-    # VOLL penalty
-    for r in regions:
+            obj_terms.append(c_u * g[(int(u), int(t))])
+    for r in regions_list:
         for t in range(T):
-            cost_terms.append(voll_jpy_per_mwh * shed[(r, t)])
-    mdl += pulp.lpSum(cost_terms)
+            obj_terms.append(voll_jpy_per_mwh * shed[(r, int(t))])
+            obj_terms.append(spill_penalty * spill[(r, int(t))])
+    mdl += pulp.lpSum(obj_terms)
 
     # Constraints
     # Unit limits
     for u in df_units.index:
-        row = df_units.loc[u]
-        cap = float(row["cap_MW"])
-        min_frac = float(row["min_frac"])
+        row_u = df_units.loc[u]
+        cap_u = float(row_u["cap_MW"])
+        min_frac_u = float(row_u["min_frac"])
         for t in range(T):
-            # Upper bound on generation
-            if row["is_renew"]:
-                cf_col = row["cf_key"]
+            if row_u["is_renew"]:
+                cf_col = row_u["cf_key"]
                 cf_val = float(ts_df_slice.iloc[t][cf_col])
-                mdl += g[(u, t)] <= cap * cf_val, f"RenCap_{u}_{t}"
-                mdl += r_up[(u, t)] == 0.0, f"RenNoReserve_{u}_{t}"
+                mdl += g[(int(u), int(t))] <= cap_u * cf_val, f"RenCap_{u}_{t}"
+                mdl += r_up[(int(u), int(t))] == 0.0, f"RenNoReserve_{u}_{t}"
             else:
-                mdl += g[(u, t)] <= cap, f"Cap_{u}_{t}"
-                mdl += g[(u, t)] >= min_frac * cap, f"MinOut_{u}_{t}"
-                # Reserve headroom
-                mdl += r_up[(u, t)] <= cap - g[(u, t)], f"ReserveHeadroom_{u}_{t}"
-
-    # Energy balance & Reserve requirement
-    lmp_names = {}         # energy balance names to read duals
-    res_names = {}         # reserve names to read duals (use <= form for positive dual)
-    for ridx, r in enumerate(regions):
+                mdl += g[(int(u), int(t))] <= cap_u, f"Cap_{u}_{t}"
+                mdl += g[(int(u), int(t))] >= min_frac_u * cap_u, f"MinOut_{u}_{t}"
+                mdl += r_up[(int(u), int(t))] <= cap_u - g[(int(u), int(t))], f"ReserveHeadroom_{u}_{t}"
+
+    # Energy balance & Reserve requirement (keep names to read duals)
+    energy_cons_names = {}
+    reserve_cons_names = {}
+    for r in regions_list:
         units_r = units_by_region[r]
         for t in range(T):
-            # Energy balance: sum g + shed == demand[r,t]
             cname = f"EnergyBal_{r}_{t}"
-            mdl += (pulp.lpSum([g[(u, t)] for u in units_r]) + shed[(r, t)] ==
-                    float(demand[r][t])), cname
-            lmp_names[(r, t)] = cname
+            mdl += (
+                pulp.lpSum([g[(int(u), int(t))] for u in units_r]) + shed[(r, int(t))]
+                == float(demand[r][t]) + spill[(r, int(t))]
+            ), cname
+            energy_cons_names[(r, t)] = cname
 
-            # Reserve: -sum r <= - req  (so dual >= 0 at optimum)
             req = reserve_ratio * float(demand[r][t])
             rname = f"ReserveReq_{r}_{t}"
-            mdl += (-pulp.lpSum([r_up[(u, t)] for u in units_r]) <= -req), rname
-            res_names[(r, t)] = rname
+            # -sum r <= -req  (dual >= 0 at optimum)
+            mdl += (-pulp.lpSum([r_up[(int(u), int(t))] for u in units_r]) <= -req), rname
+            reserve_cons_names[(r, t)] = rname
 
     # Solve
     solver = pulp.PULP_CBC_CMD(msg=False)
     mdl.solve(solver)
 
-    # Extract LMPs and reserve prices
-    lmp_mat = np.zeros((T, len(regions)))
-    res_mat = np.zeros((T, len(regions)))
-    for j, r in enumerate(regions):
+    # Extract prices
+    lmp_mat = np.zeros((T, len(regions_list)))
+    res_mat = np.zeros((T, len(regions_list)))
+    for j, r in enumerate(regions_list):
         for t in range(T):
-            lmp_mat[t, j] = mdl.constraints[lmp_names[(r, t)]].pi  # JPY/MWh
-            res_mat[t, j] = mdl.constraints[res_names[(r, t)]].pi  # JPY/MW
+            lmp_mat[t, j] = mdl.constraints[energy_cons_names[(r, t)]].pi      # JPY/MWh
+            res_mat[t, j] = mdl.constraints[reserve_cons_names[(r, t)]].pi     # JPY/MW
 
-    lmp_df = pd.DataFrame(lmp_mat, index=ts_df_slice.index, columns=regions)
-    res_price_df = pd.DataFrame(res_mat, index=ts_df_slice.index, columns=regions)
+    lmp_df = pd.DataFrame(lmp_mat, index=ts_df_slice.index, columns=regions_list)
+    res_price_df = pd.DataFrame(res_mat, index=ts_df_slice.index, columns=regions_list)
 
-    # Dispatch table
+    # Dispatch long table
     disp_rows = []
     for u in df_units.index:
-        row = df_units.loc[u]
+        row_u = df_units.loc[u]
         for t in range(T):
             disp_rows.append({
                 "Time": ts_df_slice.index[t],
-                "unit_id": row["unit_id"],
-                "region": row["region"],
-                "tech": row["tech"],
-                "g_MW": g[(u, t)].value(),
-                "r_up_MW": r_up[(u, t)].value()
+                "unit_id": row_u["unit_id"],
+                "region": row_u["region"],
+                "tech": row_u["tech"],
+                "g_MW": g[(int(u), int(t))].value(),
+                "r_up_MW": r_up[(int(u), int(t))].value()
             })
     dispatch_df = pd.DataFrame(disp_rows)
 
     return {"lmp_df": lmp_df, "res_price_df": res_price_df, "dispatch_df": dispatch_df}
 
+
 # -----------------------------
-# Streamlit UI
+# Streamlit App
 # -----------------------------
 st.set_page_config(page_title="Simultaneous Market (JP-10) — Random Fleet", layout="wide")
-st.title("同時市場シミュレーション（日本10エリア・乱数フリート）")
-
-# Regions
-REGIONS = ["Hokkaido","Tohoku","Tokyo","Chubu","Hokuriku","Kansai","Chugoku","Shikoku","Kyushu","Okinawa"]
+st.title("同時市場シミュレーション（日本10エリア・乱数フリート・完全版）")
 
-# Load time-series
+# Load time-series (robust to mixed formats)
 ts_df = load_timeseries()
 
+# Regions
+regions = ["Hokkaido", "Tohoku", "Tokyo", "Chubu", "Hokuriku",
+           "Kansai", "Chugoku", "Shikoku", "Kyushu", "Okinawa"]
+
 with st.sidebar:
     st.header("乱数と時間範囲")
-    seed = st.number_input("Random seed", value=42, step=1)
-    hours = st.slider("Hours to simulate", min_value=24, max_value=min(168, len(ts_df)), value=24, step=24)
-    start_idx = st.slider("Start index", min_value=0, max_value=max(0, len(ts_df)-hours), value=0, step=1)
-    ts_slice = ts_df.iloc[start_idx:start_idx+hours]
+    seed_val = st.number_input("Random seed", value=42, step=1)
+    max_hours = min(168, len(ts_df))
+    hours_val = st.slider("Hours to simulate", min_value=24, max_value=max_hours, value=min(24, max_hours), step=24)
+    start_idx = st.slider("Start index", min_value=0, max_value=max(0, len(ts_df) - hours_val), value=0, step=1)
+    ts_slice = ts_df.iloc[start_idx:start_idx + hours_val]
 
     st.header("需要配分（∑=1.0）")
-    default_share = pd.DataFrame({
-        "region": REGIONS,
-        "share": [0.03,0.09,0.32,0.14,0.03,0.17,0.07,0.03,0.11,0.01]
+    default_share_df = pd.DataFrame({
+        "region": regions,
+        "share": [0.03, 0.09, 0.32, 0.14, 0.03, 0.17, 0.07, 0.03, 0.11, 0.01]
     })
-    share_df = st.data_editor(default_share, num_rows="fixed", use_container_width=True)
-    ssum = float(share_df["share"].sum())
-    if abs(ssum - 1.0) > 1e-9:
-        st.warning(f"需要配分の合計が {ssum:.3f}、1.0 に正規化します。")
-    demand_shares = {row["region"]: float(row["share"])/ssum for _, row in share_df.iterrows()}
+    share_df = st.data_editor(default_share_df, num_rows="fixed", use_container_width=True)
+    share_sum = float(share_df["share"].sum())
+    if abs(share_sum - 1.0) > 1e-9:
+        st.warning(f"需要配分の合計が {share_sum:.3f}、1.0 に正規化します。")
+    demand_shares = {row["region"]: float(row["share"]) / share_sum for _, row in share_df.iterrows()}
 
-    st.header("燃料価格・為替（可変費）")
+    st.header("燃料価格・為替（可変費への影響）")
     usd_jpy = st.number_input("USD/JPY", value=148.21, step=0.5)
     lng_px = st.number_input("LNG (USD/MMBtu)", value=11.27, step=0.1)
     coal_px = st.number_input("Coal (USD/ton)", value=130.0, step=1.0)
     oil_px = st.number_input("Oil (USD/bbl)", value=80.0, step=1.0)
-    # Heat-rate (GJ/MWh) ranges (for random draw, UI below sets min/max)
-    st.caption("熱率の代表値：Gas CCGT≈6.6, Coal≈8.3, Oil≈9.2（レンジ内で乱数生成）")
-    hr_gas_min = st.number_input("HR_gas_min (GJ/MWh)", value=6.2, step=0.1)
-    hr_gas_max = st.number_input("HR_gas_max (GJ/MWh)", value=6.9, step=0.1)
-    hr_coal_min = st.number_input("HR_coal_min (GJ/MWh)", value=7.8, step=0.1)
-    hr_coal_max = st.number_input("HR_coal_max (GJ/MWh)", value=9.0, step=0.1)
-    hr_oil_min = st.number_input("HR_oil_min (GJ/MWh)", value=8.8, step=0.1)
-    hr_oil_max = st.number_input("HR_oil_max (GJ/MWh)", value=10.0, step=0.1)
-    nuc_var = st.number_input("原子力の可変費（JPY/MWh）", value=2300.0, step=100.0)
-
-    vc = var_costs_jpy_per_mwh(usd_jpy, lng_px, coal_px, oil_px,
-                               (hr_gas_min+hr_gas_max)/2, (hr_coal_min+hr_coal_max)/2, (hr_oil_min+hr_oil_max)/2,
-                               nuc_var)
-    st.caption("参考：中庸熱率での短期限界費用（JPY/MWh）")
-    st.write({k: round(v,1) for k,v in vc.items()})
+    price_gj = fuel_prices_jpy_per_gj(usd_jpy, lng_px, coal_px, oil_px)
+    st.caption("燃料価格（JPY/GJ）")
+    st.write({k: round(v, 1) for k, v in price_gj.items()})
+
+    st.header("熱率レンジ（GJ/MWh：乱数生成に使用）")
+    hr_gas_min = st.number_input("Gas CCGT min", value=6.2, step=0.1)
+    hr_gas_max = st.number_input("Gas CCGT max", value=6.9, step=0.1)
+    hr_coal_min = st.number_input("Coal min", value=7.8, step=0.1)
+    hr_coal_max = st.number_input("Coal max", value=9.0, step=0.1)
+    hr_oil_min = st.number_input("Oil min", value=8.8, step=0.1)
+    hr_oil_max = st.number_input("Oil max", value=10.0, step=0.1)
+    nuc_var_jpy_mwh = st.number_input("Nuclear variable cost (JPY/MWh)", value=2300.0, step=100.0)
 
     st.header("ユニット数（各エリア）")
     units_df = pd.DataFrame({
-        "region": REGIONS,
-        "lng_units": [6,10,25,10,4,20,8,4,10,2],
-        "coal_units":[3,6,10,6,3,10,5,2,7,1],
-        "oil_units": [2,3,6,3,2,5,3,2,3,1],
-        "nuc_units": [0,2,4,2,1,3,1,1,2,0],
-        "solar_units":[20,30,60,30,12,40,20,12,30,8],
-        "on_units":  [10,15,25,15,6,20,10,6,15,4],
-        "off_units": [2,3,6,3,1,4,2,1,3,0],
-        "river_units":[5,8,12,8,4,12,6,3,8,2]
+        "region": regions,
+        "lng_units":  [6, 10, 25, 10, 4, 20, 8, 4, 10, 2],
+        "coal_units": [3, 6, 10, 6, 3, 10, 5, 2, 7, 1],
+        "oil_units":  [2, 3, 6, 3, 2, 5, 3, 2, 3, 1],
+        "nuc_units":  [0, 2, 4, 2, 1, 3, 1, 1, 2, 0],
+        "solar_units":[20, 30, 60, 30, 12, 40, 20, 12, 30, 8],
+        "on_units":   [10, 15, 25, 15,  6, 20, 10,  6, 15, 4],
+        "off_units":  [2, 3, 6, 3, 1, 4, 2, 1, 3, 0],
+        "river_units":[5, 8, 12, 8, 4, 12, 6, 3, 8, 2]
     })
     units_df = st.data_editor(units_df, use_container_width=True)
 
     st.header("容量レンジ [MW/ユニット]")
     cap_bounds = {
-        "lng":   (200.0, 900.0),
-        "coal":  (300.0,1000.0),
-        "oil":   (100.0, 700.0),
-        "nuclear": (500.0,1400.0)
+        "lng": (200.0, 900.0),
+        "coal": (300.0, 1000.0),
+        "oil": (100.0, 700.0),
+        "nuclear": (500.0, 1400.0)
     }
     ren_bounds = {
         "solar": (10.0, 200.0),
@@ -340,7 +472,7 @@ with st.sidebar:
         "river": (10.0, 200.0)
     }
 
-    st.header("最低出力（比率レンジ）")
+    st.header("最低出力（比率レンジ, 乱数生成に使用）")
     minout_cfg = {
         "lng": (0.0, 0.2),
         "coal": (0.2, 0.6),
@@ -350,63 +482,75 @@ with st.sidebar:
 
     st.header("市場パラメータ")
     reserve_ratio = st.slider("一次予備率（需要比）", min_value=0.0, max_value=0.20, value=0.03, step=0.01)
-    voll = st.number_input("VOLL（JPY/MWh）", value=300000.0, step=10000.0)
+    voll_val = st.number_input("VOLL（JPY/MWh）", value=300000.0, step=10000.0)
 
-# Build counts config
+# Build counts config dict
 counts_cfg = {}
 for _, row in units_df.iterrows():
-    r = row["region"]
-    counts_cfg[(r,"lng")] = int(row["lng_units"])
-    counts_cfg[(r,"coal")] = int(row["coal_units"])
-    counts_cfg[(r,"oil")] = int(row["oil_units"])
-    counts_cfg[(r,"nuclear")] = int(row["nuc_units"])
-    counts_cfg[(r,"solar")] = int(row["solar_units"])
-    counts_cfg[(r,"onshore_wind")] = int(row["on_units"])
-    counts_cfg[(r,"offshore_wind")] = int(row["off_units"])
-    counts_cfg[(r,"river")] = int(row["river_units"])
-
-# Heat-rate and min-output configs
-hr_cfg = {"lng": (hr_gas_min, hr_gas_max), "coal": (hr_coal_min, hr_coal_max),
-          "oil": (hr_oil_min, hr_oil_max), "nuclear": (np.nan, np.nan)}
-min_output_cfg = minout_cfg
+    rname = row["region"]
+    counts_cfg[(rname, "lng")] = int(row["lng_units"])
+    counts_cfg[(rname, "coal")] = int(row["coal_units"])
+    counts_cfg[(rname, "oil")] = int(row["oil_units"])
+    counts_cfg[(rname, "nuclear")] = int(row["nuc_units"])
+    counts_cfg[(rname, "solar")] = int(row["solar_units"])
+    counts_cfg[(rname, "onshore_wind")] = int(row["on_units"])
+    counts_cfg[(rname, "offshore_wind")] = int(row["off_units"])
+    counts_cfg[(rname, "river")] = int(row["river_units"])
+
+# Heat-rate bounds map
+hr_bounds = {
+    "lng": (hr_gas_min, hr_gas_max),
+    "coal": (hr_coal_min, hr_coal_max),
+    "oil": (hr_oil_min, hr_oil_max),
+    "nuclear": (np.nan, np.nan)
+}
 
 if st.button("シミュレーション実行（同時市場クリアリング）"):
-    rng = np.random.default_rng(seed)
-    df_units = generate_fleet(REGIONS, rng, counts_cfg, cap_bounds, hr_cfg, min_output_cfg, ren_bounds)
+    rng = np.random.default_rng(int(seed_val))
+    fleet_df = generate_fleet(regions, rng, counts_cfg, cap_bounds, hr_bounds, minout_cfg, ren_bounds)
 
     st.subheader("生成フリート（ユニット一覧）")
-    st.dataframe(df_units)
+    st.dataframe(fleet_df)
+
+    res = clear_market(
+        df_units=fleet_df,
+        ts_df_slice=ts_slice,
+        regions_list=regions,
+        demand_shares_map=demand_shares,
+        voll_jpy_per_mwh=voll_val,
+        reserve_ratio=float(reserve_ratio),
+        fuel_price_jpy_per_gj_map=price_gj,
+        nuclear_varcost_jpy_per_mwh=nuc_var_jpy_mwh,
+        spill_penalty=1e-6
+    )
 
-    res = clear_market(df_units, ts_slice, REGIONS, demand_shares, voll,
-                       reserve_ratio, varcost_map=var_costs_jpy_per_mwh(
-                           usd_jpy, lng_px, coal_px, oil_px,
-                           (hr_gas_min+hr_gas_max)/2, (hr_coal_min+hr_coal_max)/2, (hr_oil_min+hr_oil_max)/2,
-                           nuc_var
-                       ))
-
-    # LMP & Reserve price plots
     st.subheader("LMP（JPY/MWh）")
-    st.plotly_chart(px.line(res["lmp_df"], x=res["lmp_df"].index, y=res["lmp_df"].columns), use_container_width=True)
+    st.plotly_chart(px.line(res["lmp_df"], x=res["lmp_df"].index, y=res["lmp_df"].columns,
+                            title="Area LMP (JPY/MWh)"), use_container_width=True)
 
     st.subheader("予備力価格（JPY/MW）")
-    st.plotly_chart(px.line(res["res_price_df"], x=res["res_price_df"].index, y=res["res_price_df"].columns), use_container_width=True)
-
-    # Simple national dispatch stack (sum by tech)
-    disp = res["dispatch_df"].merge(df_units[["unit_id","tech"]], on="unit_id")
-    stack = disp.groupby(["Time","tech"], as_index=False)["g_MW"].sum()
-    fig = go.Figure()
-    for tech in ["solar","onshore_wind","offshore_wind","river","nuclear","coal","lng","oil"]:
-        if tech in stack["tech"].unique():
-            sub = stack[stack["tech"]==tech]
-            fig.add_trace(go.Scatter(x=sub["Time"], y=sub["g_MW"], mode="lines", stackgroup="one", name=tech))
-    fig.update_layout(title="全国発電スタック（合算）", yaxis_title="MW")
-    st.plotly_chart(fig, use_container_width=True)
+    st.plotly_chart(px.line(res["res_price_df"], x=res["res_price_df"].index, y=res["res_price_df"].columns,
+                            title="Area Reserve Price (JPY/MW)"), use_container_width=True)
+
+    # National dispatch stack (sum by tech)
+    disp = res["dispatch_df"]  # already includes tech
+    stack = disp.groupby(["Time", "tech"], as_index=False)["g_MW"].sum()
+    fig_stack = go.Figure()
+    for tech_name in ["solar", "onshore_wind", "offshore_wind", "river", "nuclear", "coal", "lng", "oil"]:
+        if tech_name in stack["tech"].unique():
+            sub = stack[stack["tech"] == tech_name]
+            fig_stack.add_trace(go.Scatter(x=sub["Time"], y=sub["g_MW"], mode="lines",
+                                           stackgroup="one", name=tech_name))
+    fig_stack.update_layout(title="全国発電スタック（合算）", yaxis_title="MW")
+    st.plotly_chart(fig_stack, use_container_width=True)
 
     # Downloads
-    csv_buf = StringIO()
-    df_units.to_csv(csv_buf, index=False, encoding="utf-8")
-    st.download_button("ユニット一覧CSVダウンロード", data=csv_buf.getvalue(), file_name="fleet_units.csv", mime="text/csv")
-
-    csv_buf2 = StringIO()
-    res["dispatch_df"].to_csv(csv_buf2, index=False, encoding="utf-8")
-    st.download_button("ディスパッチ結果CSVダウンロード", data=csv_buf2.getvalue(), file_name="dispatch.csv", mime="text/csv")
+    csv_buf_units = StringIO()
+    fleet_df.to_csv(csv_buf_units, index=False, encoding="utf-8")
+    st.download_button("ユニット一覧CSVダウンロード", data=csv_buf_units.getvalue(),
+                       file_name="fleet_units.csv", mime="text/csv")
+
+    csv_buf_disp = StringIO()
+    res["dispatch_df"].to_csv(csv_buf_disp, index=False, encoding="utf-8")
+    st.download_button("ディスパッチ結果CSVダウンロード", data=csv_buf_disp.getvalue(),
+                       file_name="dispatch.csv", mime="text/csv")