util/lua/compute_stats.lua

-- Compute network status.
local transform = require 'torchnet.transform'

local stats = {}

local function get_layer(model, layer_names)
    local res = {}

    -- local inv_table = {}
    -- for _, l in pairs(layer_names) do inv_table[l] = true end

    for i = 1, #model.modules do
        local m = model.modules[i]

        if layer_names[torch.typename(m)] then 
            table.insert(res, m) 
        elseif m.modules then
            local prev = get_layer(m, layer_names)
            for _, mm in ipairs(prev) do
                table.insert(res, mm)
            end
        end
    end
    return res
end

function stats.get_relus(model)
    local layer_names = { ["nn.ReLU"] = true, ["cudnn.ReLU"] = true }
    return get_layer(model, layer_names)
end

function stats.get_fcs(model)
    local fc_layers = {}
    for i = 1, #model.modules do
        local m = model.modules[i]
        if torch.typename(m) == 'nn.Linear' then
            local prev_layer = (i == 1 and 'input' or model.modules[i-1])
            table.insert(fc_layers, {prev_layer, model.modules[i] })
        end
    end
    return fc_layers
end

local function permute_stats(stats) 
    for j = 1, #stats do
        local perm = {}
        for rr = 3, stats[j]:nDimension() do table.insert(perm, rr) end
        table.insert(perm, 1); table.insert(perm, 2)

        stats[j] = stats[j]:permute(unpack(perm))
    end
end

function stats.create_cell(dims)
    local counter = {}
    for i = 1, #dims do
        counter[i] = 1
    end

    local done = false
    local t = {}
    while not done do
        -- Based on current 
        local currt = t
        for i = 1, #dims do
            if not currt[counter[i]] then currt[counter[i]] = {} end
            currt = currt[counter[i]]
        end

        -- Advance
        done = true
        for i = #dims, 1, -1 do
            counter[i] = counter[i] + 1
            if counter[i] <= dims[i] then 
                done = false
                break 
            end
            counter[i] = 1
        end
    end

    return t
end

local function evaluate_through(model, all_data, all_labels, nBatch, func)
    model:evaluate()
    local nTrain = all_data:size(1)
    nBatch = nBatch or 128
    local accuracy = 0.0

    -- Statistics
    for i = 1, nTrain, nBatch do
        local data = all_data[{{i, i + nBatch - 1}, {}}]
        local res = model:forward(data:cuda())
        local gt_labels = all_labels:sub(i, i + nBatch - 1):float()

        func(data, gt_labels, res)

        best_score, best_idx = torch.max(res, 2)
        accuracy = accuracy + best_idx:float():eq(gt_labels):sum()
    end
    return accuracy / nTrain
end

function stats.torchnet_evaluator(model, dataset, collector, maxload)
    local n = 0
    local perm = transform.randperm(dataset:size())
    local accuracy = 0.0

    local crit = collector.needbackprop and nn.ClassNLLCriterion():cuda()
    if collector.needbackprop then model:training() else model:evaluate() end

    if collector.starter then collector.starter(model) end

    for sample in dataset:iterator{perm = perm} do
        local input_cuda = sample.input:cuda()
        local res = model:forward(input_cuda)

        if collector.needbackprop then
            -- Also backprop with ground truth data
            local target_cuda = torch.squeeze(sample.target):cuda()

            crit:forward(res, target_cuda)
            model:zeroGradParameters()
            crit:backward(res, target_cuda)
            model:backward(input_cuda, crit.gradInput)
        end

        if collector.collector then collector.collector(input_cuda, sample.target) end

        best_score, best_idx = torch.max(res, 2)
        accuracy = accuracy + best_idx:double():eq(sample.target):sum()
        n = n + input_cuda:size(1)
        if n >= maxload then break end
    end

    if collector.finalizer then collector.finalizer(model) end

    return accuracy / n, collector.returner()
end

function stats.node_stats_collector(nClass)
    -- Check how many ReLU layers are there.
    -- Statistics
    -- layer, node id -> K by 2 tensor.
    local node_stats = {}
    local relu_layers = {}

    local collector = function (batch_input, batch_target)
        -- Get layer statistics
        local n_batch = batch_input:size(1)
        for j = 1, #relu_layers do
            local output = relu_layers[j].output:clone()
            local dim = torch.totable(output[1]:size())
            if not node_stats[j] then
                node_stats[j] = torch.zeros(nClass, 2, unpack(dim))            
            end
                
            local high = output:ge(1e-4):double()
            local low = output:lt(1e-4):double()
                    
            for k = 1, n_batch do
                local t = batch_target[k]
                if type(t) ~= 'number' then t = torch.squeeze(t) end
                local s = node_stats[j][t]
                assert(s, string.format("stats.node_stats_collector out of bound. j = %d, batch_target[%d] = %d", j, k, t))
                s[1]:add(low[k])
                s[2]:add(high[k])
            end       
        end
    end

    -- local accuracy = evaluate_through(model, all_data, all_labels, nBatch, collector)
    -- return node_stats, relu_layers, accuracy
    return { 
        returner = function () return node_stats, relu_layers end, 
        starter = function (model) relu_layers = stats.get_relus(model) end, 
        collector = collector, 
        finalizer = function () permute_stats(node_stats) end 
    }
end

function stats.node_grad_corr_collector(nClass)
    local node_stats = {}
    local relu_layers = {}

    local collector = function (batch_input, batch_target)
        -- Get layer statistics
        local n_batch = batch_input:size(1)
        for j = 1, #relu_layers do
            local gradInput = relu_layers[j].gradInput:clone()

            local dim = torch.totable(gradInput[1]:size())
            if not node_stats[j] then
                node_stats[j] = torch.zeros(nClass, 2, unpack(dim))            
            end
                
            local pos = gradInput:gt(1e-4):double()
            local neg = gradInput:lt(-1e-4):double()
                    
            for k = 1, n_batch do
                local t = batch_target[k]
                if type(t) ~= 'number' then t = torch.squeeze(t) end
                local s = node_stats[j][t]
                assert(s, string.format("stats.node_grad_corr_collector out of bound. j = %d, batch_target[%d] = %d", j, k, t))
                s[1]:add(neg[k])
                s[2]:add(pos[k])
            end
        end
    end

    -- local accuracy = evaluate_through(model, all_data, all_labels, nBatch, collector)
    -- return node_stats, relu_layers, accuracy
    return { 
        returner = function () return node_stats, relu_layers end, 
        starter = function (model) relu_layers = stats.get_relus(model) end, 
        collector = collector, 
        needbackprop = true,
        finalizer = function () permute_stats(node_stats) end 
    }
end

--[[
local s = node_stats[j][l][t]
if bConv then
    local gv = gradInput[k][l]:view(-1)
    for kk = 1, gv:size(1) do
        if gv[kk] < -1e-4 then
            table.insert(s[1], img_counter)
        elseif gv[kk] > 1e-4 then
            table.insert(s[2], img_counter)
        end
    end
else
    local gv = gradInput[k][l]
    if gv < -1e-4 then
        table.insert(s[1], img_counter)
    elseif gv > 1e-4 then
        table.insert(s[2], img_counter)
    end
end
]]--

function stats.node_resp_image_collector(nImage, collection_type)
    local node_stats = {}
    local relu_layers = {}
    local batch_inputs = {}
    local batch_targets = {}

    local collector = function (batch_input, batch_target)
        -- Get layer statistics
        local n_batch = batch_input:size(1)
        local baseaddr = #batch_inputs * batch_input:size(1)

        for j = 1, #relu_layers do
            local output = relu_layers[j].output
            local outputAgg
            if output[1]:nDimension() == 3 then
                if collection_type == 'sum' then
                    outputAgg = output:sum(3):sum(4)
                elseif collection_type == 'max' then
                    local agg1, _ = output:max(3)
                    outputAgg, _ = agg1:max(4)
                else
                    error(string.format("collection_type = %s is not defined.", collection_type))
                end
            else
                outputAgg = output:clone()
            end

            -- Check if the 
            local nChannel = output[1]:size(1)
            if not node_stats[j] then
                node_stats[j] = torch.FloatTensor(nImage, nChannel):zero()
            end

            assert(baseaddr + 1 <= nImage, string.format("baseaddr + 1 = %d is out of bound (%d)", baseaddr + 1, nImage))
            assert(baseaddr + n_batch <= nImage, string.format("baseaddr + n_batch = %d is out of bound (%d)", baseaddr + n_batch, nImage))

            node_stats[j]:sub(baseaddr + 1, baseaddr + n_batch):copy(outputAgg)
        end
        table.insert(batch_inputs, batch_input:float())
        table.insert(batch_targets, batch_target:float())
    end

    -- local accuracy = evaluate_through(model, all_data, all_labels, nBatch, collector)
    -- return node_stats, relu_layers, accuracy
    return { 
        returner = function () return node_stats, relu_layers, batch_inputs, batch_targets end, 
        starter = function (model) relu_layers = stats.get_relus(model) end, 
        collector = collector, 
        finalizer = function () 
            for i = 1, #node_stats do 
                node_stats[i] = node_stats[i]:transpose(1, 2)
            end
        end
    }
end

function stats.node_grad_image_collector(nImage)
    local node_stats = {}
    local relu_layers = {}
    local batch_inputs = {}
    local batch_targets = {}

    local collector = function (batch_input, batch_target)
        -- Get layer statistics
        local n_batch = batch_input:size(1)
        local baseaddr = #batch_inputs * batch_input:size(1)

        for j = 1, #relu_layers do
            local gradInput = relu_layers[j].gradInput
            local gradInputAgg
            if gradInput[1]:nDimension() == 3 then
                gradInputAgg = gradInput:sum(3):sum(4)
            else
                gradInputAgg = gradInput:clone()
            end

            -- Check if the 
            local nChannel = gradInput[1]:size(1)
            if not node_stats[j] then
                node_stats[j] = torch.FloatTensor(nImage, nChannel):zero()
            end

            assert(baseaddr + 1 <= nImage, string.format("baseaddr + 1 = %d is out of bound (%d)", baseaddr + 1, nImage))
            assert(baseaddr + n_batch <= nImage, string.format("baseaddr + n_batch = %d is out of bound (%d)", baseaddr + n_batch, nImage))

            node_stats[j]:sub(baseaddr + 1, baseaddr + n_batch):copy(gradInputAgg)
        end
        table.insert(batch_inputs, batch_input:float())
        table.insert(batch_targets, batch_target:float())
    end

    -- local accuracy = evaluate_through(model, all_data, all_labels, nBatch, collector)
    -- return node_stats, relu_layers, accuracy
    return { 
        returner = function () return node_stats, relu_layers, batch_inputs, batch_targets end, 
        starter = function (model) relu_layers = stats.get_relus(model) end, 
        collector = collector, 
        needbackprop = true,
        finalizer = function () 
            for i = 1, #node_stats do 
                node_stats[i] = node_stats[i]:transpose(1, 2)
            end
        end
    }
end

function stats.weight_stats(model, all_data, all_labels, nBatch, nClass)
    local fc_layers = stats.get_fcs(model)

    -- weight stats
    local weight_stats = {}
    for j = 1, #fc_layers do weight_stats[j] = {} end

    local collector = function (batch_input, batch_target)  
        -- Get weight stats
        local nbatch = batch_input:size(1)
        for j = 1, #fc_layers do
            -- Get input and output, and compute their cross statistics
            local prev_input = fc_layers[j][1] == 'input' and batch_input or fc_layers[j][1].output                
            local curr = fc_layers[j][2]
            
            if not weight_stats[j].stats then
                weight_stats[j].stats = torch.zeros(nClass, 4, curr.weight:size(1), curr.weight:size(2))
            end

            local high_i = prev_input:ge(1e-4):double()
            local low_i = prev_input:lt(1e-4):double()
            local high_o = curr.output:ge(1e-4):double()
            local low_o = curr.output:lt(1e-4):double()
            
            for k = 1, nbatch do
                local s = weight_stats[j].stats[batch_target[k]]
                s[1]:add(torch.ger(low_o[k], low_i[k]))
                s[2]:add(torch.ger(low_o[k], high_i[k]))
                s[3]:add(torch.ger(high_o[k], low_i[k]))
                s[4]:add(torch.ger(high_o[k], high_i[k]))
            end
        end
    end

    local accuracy = evaluate_through(model, all_data, all_labels, nBatch, collector)
    for j = 1, #fc_layers do
        weight_stats[j].stats = weight_stats[j].stats:permute(3, 4, 2, 1)
    end
    return weight_stats, fc_layers, accuracy
end

return stats