repro-committed-q-learning-reactive-rl / run_qcommit_official.jl
ProCreations's picture
Publish exact native committed Q-learning reproduction
98bc8ca verified
Raw
History Blame Contribute Delete
1.25 kB
#!/usr/bin/env julia
# Execute the authors' exact qcorridor.jl implementation for Figure 3.
# Usage: julia run_qcommit_official.jl AUTHORS_QCORRIDOR_JL OUTPUT_CSV
if length(ARGS) != 2
error("expected AUTHORS_QCORRIDOR_JL OUTPUT_CSV")
end
include(abspath(ARGS[1]))
const T = 1000
const SEEDS = 0:999
const LENGTHS = (5, 10, 20, 50, 100, 200)
const ALPHA0 = 0.1
const ALPHAT = 0.01
const EPS0 = 0.1
const EPST = 0.01
function curve(k::Int, committed::Bool)
counts = zeros(Int, T)
for seed in SEEDS
_, qs = qcorridor(
T, k, committed, seed, ALPHA0, ALPHAT, EPS0, EPST, 0
)
for t in 1:T
# The optimal reactive policy selects right in both features.
counts[t] += (qs[t, 1, 2] > qs[t, 1, 1] &&
qs[t, 2, 2] > qs[t, 2, 1])
end
end
return counts
end
open(abspath(ARGS[2]), "w") do io
println(io, "mode,k,t,optimal_count,total_seeds")
for committed in (true, false)
mode = committed ? "committed" : "regular"
for k in LENGTHS
counts = curve(k, committed)
for t in 1:T
println(io, mode, ",", k, ",", t, ",", counts[t], ",", length(SEEDS))
end
end
end
end