source/q-commit/qcorridor.jl

using Random

function qcorridor(T, k, com, seed, alpha0, alphaT, eps0, epsT, N)
    # Initialize algorithm paramters
    rng = Xoshiro(seed + 1000*k)
    pi = [0.5, 0.5]

    # Compute decreasing alpha, eps
    function c(t, c0, cT)
        if c0 == cT return c0 end
        cb = cT * T / (c0 - cT)
        ca = c0 * cb
        return ca / (cb + t)
    end

    # Initialize Q-table and start episode
    q = 0.1 * randn(rng, 2, 2)
    ts = N > 1 ? floor.(10 .^ range(0, log10(T), N)) :
        N == 0 ? range(1, T) : [T]
    qs = N != 1 ? zeros(size(ts, 1), 2, 2) : q
    x = 0
    z = 0
    u = rand(rng) < pi[z + 1]
    j = 1

    for t in 1:T
        # Sample transition
        x += 2*u - 1
        z_ = x > 0
        r = x == 0 ? 0 : x == k + 1 ? k : -1

        # Update Q-values
        q[z + 1, u + 1] += c(t, alpha0, alphaT) * (r +
            (0 <= x <= k ? maximum(@view q[z_ + 1, :]) : 0) - q[z + 1, u + 1])

        if eps0 > 0
            eps = c(t, eps0, epsT)
            pi[1] = q[1, 2] > q[1, 1] ? 1 - eps : eps
            pi[2] = q[2, 2] > q[2, 1] ? 1 - eps : eps
        end

        # Log Q-table (in logarithmic intervals)
        while j <= size(ts, 1) && t == ts[j]
            qs[j, :, :] = q
            j += 1
        end

        if !(0 <= x <= k)  # Start new episode
            x = 0
            z = 0
            u = rand(rng) < pi[z + 1]
        elseif !com || z_ != z  # Sample new action
            z = z_
            u = rand(rng) < pi[z_ + 1]
        end
    end
    return ts, qs
end