File size: 4,426 Bytes
c711202 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | """
Calibration plot: nominal vs empirical coverage for concentration and source fields.
Reads scalability CSV results (which include coverage columns) and plots calibration curves.
Can overlay multiple runs (e.g., default vs calibrated output scale).
Usage:
julia --project=../.. plot_calibration.jl results/scalability/default.csv results/scalability/calibrated.csv
julia --project=../.. plot_calibration.jl results/scalability/scalability_results.csv
"""
using CSV, DataFrames
using CairoMakie
using Statistics
using Printf
function coverage_at_level(c_mean, c_std, c_true, z)
Statistics.mean(abs.(c_true .- c_mean) .< z .* c_std)
end
"""
Extract (nominal, empirical) coverage pairs from a DataFrame row
at many confidence levels.
"""
function calibration_curve_from_row(row)
# We only have 50/90/95 in the CSV. Return those.
nominal = [0.50, 0.90, 0.95]
empirical_c = [row.cov_c_50, row.cov_c_90, row.cov_c_95]
empirical_s = [row.cov_s_50, row.cov_s_90, row.cov_s_95]
return nominal, empirical_c, empirical_s
end
"""
Average calibration curve across all grid sizes (excluding very coarse).
Automatically detects which coverage levels are in the CSV.
"""
function average_calibration(df; min_N=11)
df_filt = filter(r -> r.N >= min_N, df)
# Detect coverage columns: cov_c_XX and cov_s_XX
all_cols = names(df_filt)
cov_c_cols = sort([c for c in all_cols if startswith(c, "cov_c_")])
cov_s_cols = sort([c for c in all_cols if startswith(c, "cov_s_")])
# Extract nominal levels from column names
nominal = [parse(Int, split(c, "_")[end]) / 100.0 for c in cov_c_cols]
avg_c = [Statistics.mean(df_filt[!, col]) for col in cov_c_cols]
avg_s = [Statistics.mean(df_filt[!, col]) for col in cov_s_cols]
std_c = [Statistics.std(df_filt[!, col]) for col in cov_c_cols]
std_s = [Statistics.std(df_filt[!, col]) for col in cov_s_cols]
return (; nominal, avg_c, avg_s, std_c, std_s)
end
function plot_calibration(csv_paths::Vector{String}, labels::Vector{String};
filename::Union{Nothing,String}=nothing, min_N::Int=11)
fig = Figure(size=(800, 380), fontsize=12)
colors = [:royalblue, :crimson, :seagreen, :darkorange]
panels = [(:avg_c, :std_c, "Concentration"), (:avg_s, :std_s, "Source")]
for (col_idx, (avg_key, std_key, title)) in enumerate(panels)
ax = Axis(fig[1, col_idx];
xlabel="Nominal coverage",
ylabel="Empirical coverage",
title=title,
aspect=1,
)
xlims!(ax, 0.0, 1.0)
ylims!(ax, 0.0, 1.0)
# Diagonal (perfect calibration)
lines!(ax, [0, 1], [0, 1]; color=:gray60, linestyle=:dash, linewidth=1,
label="Perfect")
for (i, (path, label)) in enumerate(zip(csv_paths, labels))
df = CSV.read(path, DataFrame)
cal = average_calibration(df; min_N=min_N)
avg = getfield(cal, avg_key)
std_vals = getfield(cal, std_key)
scatterlines!(ax, cal.nominal, avg;
color=colors[i], linewidth=2, markersize=8, label=label)
# Error bars (±1 std across grid sizes)
errorbars!(ax, cal.nominal, avg, std_vals;
color=colors[i], whiskerwidth=8, linewidth=1)
end
axislegend(ax; position=:lt)
end
if !isnothing(filename)
mkpath(dirname(filename))
save(filename, fig, px_per_unit=3)
println("Saved: $filename")
end
return fig
end
# CLI
if abspath(PROGRAM_FILE) == @__FILE__
if length(ARGS) < 1
println("Usage: julia plot_calibration.jl <csv1> [csv2] [--labels label1,label2] [-o output.pdf]")
exit(1)
end
# Parse args: CSV files, optional --labels, optional -o
local csv_paths = String[]
local labels = String[]
local output = joinpath(@__DIR__, "results", "scalability", "calibration.pdf")
local idx = 1
while idx <= length(ARGS)
if ARGS[idx] == "--labels"
idx += 1
labels = String.(split(ARGS[idx], ","))
elseif ARGS[idx] == "-o"
idx += 1
output = ARGS[idx]
else
push!(csv_paths, ARGS[idx])
end
idx += 1
end
if isempty(labels)
labels = ["Run $j" for j in 1:length(csv_paths)]
end
plot_calibration(csv_paths, labels; filename=output)
end
|