File size: 1,526 Bytes
98bc8ca | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | using Random
function qcorridor(T, k, com, seed, alpha0, alphaT, eps0, epsT, N)
# Initialize algorithm paramters
rng = Xoshiro(seed + 1000*k)
pi = [0.5, 0.5]
# Compute decreasing alpha, eps
function c(t, c0, cT)
if c0 == cT return c0 end
cb = cT * T / (c0 - cT)
ca = c0 * cb
return ca / (cb + t)
end
# Initialize Q-table and start episode
q = 0.1 * randn(rng, 2, 2)
ts = N > 1 ? floor.(10 .^ range(0, log10(T), N)) :
N == 0 ? range(1, T) : [T]
qs = N != 1 ? zeros(size(ts, 1), 2, 2) : q
x = 0
z = 0
u = rand(rng) < pi[z + 1]
j = 1
for t in 1:T
# Sample transition
x += 2*u - 1
z_ = x > 0
r = x == 0 ? 0 : x == k + 1 ? k : -1
# Update Q-values
q[z + 1, u + 1] += c(t, alpha0, alphaT) * (r +
(0 <= x <= k ? maximum(@view q[z_ + 1, :]) : 0) - q[z + 1, u + 1])
if eps0 > 0
eps = c(t, eps0, epsT)
pi[1] = q[1, 2] > q[1, 1] ? 1 - eps : eps
pi[2] = q[2, 2] > q[2, 1] ? 1 - eps : eps
end
# Log Q-table (in logarithmic intervals)
while j <= size(ts, 1) && t == ts[j]
qs[j, :, :] = q
j += 1
end
if !(0 <= x <= k) # Start new episode
x = 0
z = 0
u = rand(rng) < pi[z + 1]
elseif !com || z_ != z # Sample new action
z = z_
u = rand(rng) < pi[z_ + 1]
end
end
return ts, qs
end
|