util/lua/nnutils.lua

require 'torch'
require 'nn'
require 'cunn'
require 'cudnn'
require 'fbnn'
require 'fbcunn'
require 'fbcode.deeplearning.experimental.yuandong.layers.custom_layers'

local pl = require 'pl.import_into'()
local tnt = require 'torchnet'
local bistro = require 'bistro'
local cjson = require 'cjson'

-- Some global variables. 

local conv_layer, relu_layer, maxpool_layer

if cudnn then
    conv_layer = cudnn.SpatialConvolution
    relu_layer = cudnn.ReLU
    maxpool_layer = cudnn.SpatialMaxPooling
else 
    conv_layer = nn.SpatialConvolutionMM
    relu_layer = nn.ReLU
    maxpool_layer = nn.SpatialMaxPooling
end

-- Some local utility that make networks.
-- make_network, it take layer specification + inputdim as input, return the actual layer and output dim.
local function conv_layer_size(inputsize, kw, dw, pw)
    dw = dw or 1
    pw = pw or 0
    return math.floor((inputsize + pw * 2 - kw) / dw) + 1
end

local function spatial_layer_size(layer, inputdim)
    layer.nip = layer.nip or inputdim[2]
    layer.nop = layer.nop or layer.nip

    -- print(pl.pretty.write(inputdim))
    assert(#inputdim == 4, 'Spatial_layer_size: Input dim must be 4 dimensions')
    assert(layer.nip == inputdim[2], string.format('Spatial_layer_size: the number of input channel [%d] is not the same as specified [%d]!', inputdim[2], layer.nip))
    assert(layer.nop, 'Spatial_layer_size: layer.nop is null!')

    layer.pw = layer.pw or 0
    layer.dw = layer.dw or 1

    layer.kh = layer.kh or layer.kw
    layer.ph = layer.ph or layer.pw
    layer.dh = layer.dh or layer.dw

    local outputw = conv_layer_size(inputdim[4], layer.kw, layer.dw, layer.pw)
    local outputh = conv_layer_size(inputdim[3], layer.kh, layer.dh, layer.ph)

    return {inputdim[1], layer.nop, outputh, outputw}
end

local function spec_expand(dim, spec, inputdim)
    local res = {}
    local total = 0
    for i = 1, #spec do
        if type(spec[i]) == 'table' then
            this_res, this_total = spec_expand(dim, spec[i], inputdim)
        else
            this_res = pl.tablex.deepcopy(inputdim)
            this_res[dim] = spec[i]
            this_total = spec[i]
        end
        table.insert(res, this_res)
        total = total + this_total
    end
    return res, total
end

local function make_network(layer, inputdim)
    -- if layer.showinputdim then
        print("++++++++++++++++++++++++++++")
        print("Layer spec = " .. pl.pretty.write(layer, '', false))
        print("Input dim = " .. pl.pretty.write(inputdim, '', false))
        print("----------------------------")
    -- end

    if not layer.type or layer.type == 'seq' then
        local seq = nn.Sequential()

        -- Maka a sequential network
        local curr_inputdim = inputdim
        local ll
        for _, l in ipairs(layer) do
            ll, curr_inputdim = make_network(l, curr_inputdim)
            seq:add(ll)
        end
        return seq, curr_inputdim

    elseif layer.type == 'parallel' then
        local para = nn.ParallelTable()

        local ll
        local outputdim = {}
        for idx, l in ipairs(layer) do
            ll, outputdim[idx] = make_network(l, inputdim[idx])
            para:add(ll)
        end
        return para, outputdim
    elseif layer.type == 'join' then
        assert(type(inputdim) == 'table', 'MakeNetwork::Join: inputdim must be a table!')

        local outputdim
        for idx, dim in ipairs(inputdim) do
            if not outputdim then 
                outputdim = dim
            else
                for dim_idx, d_size in ipairs(dim) do
                    if dim_idx == layer.dim then
                        outputdim[dim_idx] = outputdim[dim_idx] + d_size
                    else
                        assert(outputdim[dim_idx] == d_size, 'Join table, dimension ' .. dim_idx .. ' disagree!')
                    end
                end
            end
        end
        return nn.JoinTable(layer.dim), outputdim

    elseif layer.type == 'conv' then
        layer.nip = layer.nip or inputdim[2]
        layer.dw = layer.dw or 1

        layer.kh = layer.kh or layer.kw
        layer.dh = layer.dh or layer.dw

        layer.pw = layer.pw or math.floor(layer.kw / 2)
        layer.ph = layer.ph or math.floor(layer.kh / 2)

        assert(layer.nip == inputdim[2], string.format('MakeNetwork::Conv: #input channel [%d] must match with specification [%d]!', inputdim[2], layer.nip));        
        assert(layer.kw)
        assert(layer.kh)
        assert(layer.pw)
        assert(layer.ph)
        assert(layer.dw)
        assert(layer.dh)
        assert(layer.nip)
        assert(layer.nop)

        local conv_layer = conv_layer(layer.nip, layer.nop, layer.kw, layer.kh, layer.dw, layer.dh, layer.pw, layer.ph)
        return conv_layer, spatial_layer_size(layer, inputdim)

    elseif layer.type == 'relu' then
        return relu_layer(), inputdim

    elseif layer.type == 'bn' then
        assert(#inputdim == 2, 'Error! Input to BatchNormalization must be 2-dimensional.')
        return nn.BatchNormalization(inputdim[2]), inputdim

    elseif layer.type == 'spatialbn' then
        assert(#inputdim == 4, 'Error! Input to SpatialBatchNormalization must be 4-dimensional.')
        return nn.SpatialBatchNormalization(inputdim[2]), inputdim

    elseif layer.type == 'thres' then
        return nn.Threshold(0, 1e-6), inputdim

    elseif layer.type == 'maxp' then
        assert(layer.kw and layer.dw, "MakeNetwork:MaxP: kw and dw should not be nil")

        layer.kh = layer.kh or layer.kw
        layer.dh = layer.dh or layer.dw

        return maxpool_layer(layer.kw, layer.kh, layer.dw, layer.dh), spatial_layer_size(layer, inputdim)

    elseif layer.type == 'maxp1' then
        assert(layer.kw and layer.dw, "MakeNetwork:MaxP1: kw and dw should not be nil")
        local outputdim = { inputdim[1], conv_layer_size(inputdim[2], layer.kw, layer.dw), inputdim[3] }
        return nn.TemporalMaxPooling(layer.kw, layer.dw), outputdim

    elseif layer.type == 'reshape' then
        if layer.dir == '4-2' then
            layer.wi = layer.wi or inputdim[4]
            layer.nip = layer.nip or inputdim[2]
            layer.nop = layer.nop or inputdim[2]*inputdim[3]*inputdim[4]
        elseif layer.dir == '3-2' then
            -- For temporal 1D network, inputdim[2] is the length and inputdim[3] is the number of channels.
            layer.wi = layer.wi or inputdim[2]
            layer.nip = layer.nip or inputdim[3]
            layer.nop = layer.nop or inputdim[2]*inputdim[3]
        end

        if layer.wi then
            -- Reshape from image to vector.
            if layer.dir == '4-2' then
               layer.hi = layer.hi or inputdim[3]

               assert(#inputdim == 4, 'MakeNetwork::Reshape4-2: Input dim must be 4 dimensions')
               local outputsize = { inputdim[1], layer.nip*layer.wi*layer.hi }
               assert(outputsize[2] == inputdim[2]*inputdim[3]*inputdim[4], 'MakeNetwork::Reshape4-2: Input dim must match with specified dimensions')
               assert(outputsize[2] > 0, 
                 string.format("MakeNetwork::Reshape4-2: outputsize[2] = %d, (nip, wi, hi) = (%d, %d, %d)", 
                      outputsize[2], layer.nip, layer.wi, layer.hi))
               return nn.View(outputsize[2]), outputsize
            elseif layer.dir == '3-2' then
               assert(#inputdim == 3, 'MakeNetwork::Reshape3-2: Input dim must be 3 dimensions')
               local outputsize = { inputdim[1], layer.nip*layer.wi }
               assert(outputsize[2] == inputdim[2]*inputdim[3], 'MakeNetwork::Reshape3-2: Input dim must match with specified dimensions')
               assert(outputsize[2] > 0, 
                 string.format("MakeNetwork::Reshape3-2: outputsize[2] = %d, (nip, wi) = (%d, %d)", 
                      outputsize[2], layer.nip, layer.wi))
               return nn.View(outputsize[2]), outputsize
            end
        elseif layer.wo then
            -- Reshape from vector to image.
            assert(#inputdim == 2, 'MakeNetwork::Reshape2-4: Input dim must be 2 dimensions')
            layer.nip = layer.nip or inputdim[2]
            layer.ho = layer.ho or layer.wo
            layer.nop = layer.nop or inputdim[2] / (layer.wo * layer.ho)

            local outputsize = { inputdim[1], layer.nop, layer.ho, layer.wo }
            assert(outputsize[2]*outputsize[3]*outputsize[4] == inputdim[2], 'MakeNetwork::Reshape2-4: Input dim must match with specified dimensions')
            return nn.View(layer.nop, layer.ho, layer.wo), outputsize
        end
    elseif layer.type == 'fc' then
        assert(#inputdim == 2, 'MakeNetwork::FC: Input dim must be 2 dimensions')
        layer.nip = layer.nip or inputdim[2]
        assert(layer.nip == inputdim[2], string.format('MakeNetwork::FC: the number of input channel [%d] is not the same as specified [%d]!', inputdim[2], layer.nip))
        return nn.Linear(layer.nip, layer.nop), { inputdim[1], layer.nop }

    elseif layer.type == 'conv1' then
        -- inputdim[1] : batchsize
        -- inputdim[2] : input length
        -- inputdim[3] : nip
        assert(layer.kw, 'MakeNetwork:Conv1: kw must be specified')
        assert(#inputdim == 3, 'MakeNetwork:Conv1: Input dim must be 3 dimensions')
        assert(layer.nop, 'MakeNetwork:Conv1: nop must be specified')
        -- Note that for temporal convolutional, 
        layer.nip = layer.nip or inputdim[3]
        layer.dw = layer.dw or 1

        assert(layer.nip == inputdim[3], string.format('MakeNetwork::Conv1: the number of input channels [%d] is not the same as specified [%d]!', inputdim[3], layer.nip))
        local outputdim = {inputdim[1], conv_layer_size(inputdim[2], layer.kw, layer.dw), layer.nop}
        return nn.TemporalConvolution(layer.nip, layer.nop, layer.kw, layer.dw), outputdim

    elseif layer.type == 'usample' then
        assert(#inputdim == 4, 'MakeNetwork::USample: Input dim must be 4 dimensions')
        layer.wi = layer.wi or inputdim[3]
        assert(layer.wi == inputdim[3], string.format('MakeNetwork::USample: Input height [%d] much match with specification [%d].', inputdim[3], layer.wi))
        assert(layer.wi == inputdim[4], string.format('MakeNetwork::USample: Input width [%d] much match with specification [%d].', inputdim[4], layer.wi))
        return nn.SpatialUpSamplingNearest(layer.wo / layer.wi), { inputdim[1], inputdim[2], layer.wo, layer.wo }
    elseif layer.type == 'recursive-split' then
        -- Check if the size are the same.
        -- print("InputDim:")
        -- print(pl.pretty.write(inputdim))

        outputdim, total_use = spec_expand(layer.dim, layer.spec, inputdim)
        assert(total_use == inputdim[layer.dim], string.format("MakeNetwork::RecursiveSplitTable: Total usage specified by layer.spec [%d] is not the same as the inputdim[%d] (which is %d)", total_use, layer.dim, inputdim[layer.dim]))

        return nn.RecursiveSplitTable(layer.dim - 1, #inputdim - 1, layer.spec), outputdim
    elseif layer.type == 'addtable' then
        assert(type(inputdim) == 'table', "MakeNetwork::addtable: Inputdim must be a table.")
        assert(inputdim[1], "MakeNetwork::addtable: Inputdim must not be empty.")

        for i = 2, #inputdim do
            assert(#inputdim[i] == #inputdim[1], string.format("MakeNetwork::addtable: Each entry of input dim must be of the same length, yet #input[%d] = %d while #inputdim[1] = %d", i, #inputdim[i], #inputdim[1]))
            for j = 1, #inputdim[i] do
                assert(inputdim[i][j] == inputdim[1][j], string.format("MakeNetwork::addtable: Each entry of inputdim must be of same size. Now inputdim[%d][%d] = %d while inputdim[1][%d] = %d", i, j, inputdim[i][j], j, inputdim[1][j]))
            end
        end

        return nn.CAddTable(), inputdim[1]
    elseif layer.type == 'dropout' then
        return nn.Dropout(layer.ratio), inputdim
    elseif layer.type == 'logsoftmax' then
        return nn.LogSoftMax(), inputdim
    else
        error("Unknown layer type " .. layer.type);
    end
end

local function merge_tables(tbls)
    if #tbls == 0 then return {} end
    local res = tbls[1]
    for i = 2,#tbls do
        for j = 1, #tbls[i] do
            table.insert(res, tbls[i][j])
        end
    end

    return res
end

local nnutils = {
    make_network = make_network,
    spatial_layer_size = spatial_layer_size,
    merge_tables = merge_tables
}

-- Debugging
local g_nn_dbg = false

function nnutils.dbg_set() 
    g_nn_dbg = true
end

function nnutils.dbg_clear() 
    g_nn_dbg = false
end

function nnutils.dprint(s, ...)
    if g_nn_dbg then
       local p = {...}
       if #p == 0 then print(s) 
       else print(string.format(s, unpack(p))) 
       end
    end
end

-- local debug_mapping = {}
-- function nnutils.dbg_set_mapping(key, value)
--     debug_mapping[key] = value
-- end

-- function nnutils.

-- Get to know whether we are in a bistro run or in a local run, by checking the name of local directory..
function nnutils.in_bistro()
    local cwd = io.popen('pwd'):read("*all")
    local bistro_prefix = '/gfsai-bistro'
    return string.sub(cwd, 1, #bistro_prefix) == bistro_prefix
end

function nnutils.json_stats(t)
   return 'json_stats: ' ..  cjson.encode(t)
end

function nnutils.deepcopy(obj)
  file = torch.MemoryFile() -- creates a file in memory
  file:writeObject(obj) -- writes the object into file
  file:seek(1) -- comes back at the beginning of the file
  return file:readObject() -- gets a clone of object
end

function nnutils.add_if_nonexist(t1, t2)
   for k, v in pairs(t2) do
       if not t1[k] then t1[k] = v end
   end
   return t1
end

function nnutils.get_first_available(t, keys)
  for _, k in ipairs(keys) do
     if t[k] then return k, t[k] end
  end
  return nil, nil
end

---------- Layer-wise operation -----------------

function nnutils.pick_layers(model, name)
    local layers = {}
    for i = 1, #model.modules do
        local m = model.modules[i]
        if torch.typename(m):match(name) then
           table.insert(layers, m)
        end
    end
    return layers
end

function nnutils.operate_layers(model, name, func)
    for i = 1, #model.modules do
        local m = model.modules[i]
        if torch.typename(m):match(name) then 
            func(m)
        end
    end    
end

function nnutils.add_regular_hooks(rack)
    -- Trainer has the following hooks:
    --   start, start-epoch, sample, forward, backward, update, "end-epoch", "end"
    -- Tester has the following hooks:
    --   start, start-epoch, sample, forward, end-epoch, end

    -- hook collectgarbage and synchronize (timing purposes)
    rack:addHook('forward', function() collectgarbage() end)
    if rack.hooks.update then
        rack:addHook('update', function() cutorch.synchronize() end)
    end

    local tntexp = require 'fbcode.deeplearning.experimental.yuandong.torchnet.init'

    -- time one iteration
    tntexp.TimeMeter{ rack = rack, label = "time", perbatch = true }
end

function nnutils.set_output_json(log, trainer, config)
    local epoch = 1
    local entries_to_log = {'trainloss', 'testloss', 'train top@1', 'train top@5', 'test top@1', 'test top@5' }

    trainer:addHook(
       'end-epoch',
       function()
          -- Save a few things to json
          local perf_table = { epoch = epoch }
          for _, entry in ipairs(entries_to_log) do
            if log.key2idx[entry] then
                perf_table[entry] = log:get(entry)
            end
          end

          -- Entries for current time stamp
          perf_table.timestamp = os.clock()

          -- Special entry for whetlab (must be the last one)
          perf_table.neg_loss = -log:get("testloss")

          bistro.log(pl.tablex.merge(perf_table, config, true))
          epoch = epoch + 1
       end
    )    
end

function nnutils.add_save_on_trainer(log, net, trainer, saveto)
    -- customized model save
    log:column('saved')
    local netsav = net:clone('weight', 'bias', 'running_mean', 'running_std')
    local minerr = math.huge
    trainer:addHook(
       'end-epoch',
       function()
          local valid_col = nnutils.get_first_available(log.key2idx, { 'testloss', 'trainloss' } ) 
          local z = log:get(valid_col)
          if z and z < minerr then
             if pl.path.isdir(saveto) then
                savefile = string.format('%s/model.bin', saveto)
             else 
                savefile = saveto 
             end
             local f = torch.DiskFile(savefile, 'w')
             f:binary()
             f:writeObject(netsav)
             f:close()
             minerr = z
             log:set('saved', '*')
          else
             log:set('saved', '')
          end
       end
    )
end

function nnutils.add_logging(trainer, log)
    trainer:addHook(
       'end-epoch',
       function()
          log:print{}
          log:print{stdout = true, labels = true, separator = ' | '}
       end
    )
end

function nnutils.reload_if(config, model_name, config_name)
    -- reload?
    if config.evalOnly and config.reload == '' then
       error("evalOnly only works if there is a model to be reloaded!")
    end

    local net
    if config.reload ~='' then
       require 'nn'
       require 'cutorch'
       require 'cunn'
       require 'cudnn'
       print(string.format('| reloading experiment %s', config.reload))
       local f = torch.DiskFile(string.format('%s/%s', config.reload, model_name))
       f:binary()
       net = f:readObject()
       f:close()

       if config_name then
           local oldconfig_file = string.format('%s/%s', config.reload, config_name)
           if pl.path.exists(oldconfig_file) then
              local oldconfig = torch.load(oldconfig_file)
              oldconfig.subdir = nil
              tnt.utils.table.merge(oldconfig, config)
              oldconfig.reload = nil
              oldconfig.save = config.reload
              config = oldconfig
            end
       end
    end
    return net, config
end

--------------------------- Parse text to word table index ----------------------------

function nnutils.parse_to_idx(s, isfilename, word2index, per_char)
    local word_indices = {}
    -- Check the words 
    local sep = per_char and "." or "[^%s]+"

    local content 
    if isfilename then
       local f = torch.DiskFile(s)
       content = f:readString('*a') -- NOTE: this reads the whole file at once
       print(string.format("size of content = %d", string.len(content)))
       f:close()
    else 
       content = s
    end

    for token in string.gmatch(content, sep) do
        local index = word2index[token]
        if index ~= nil then 
          table.insert(word_indices, index) 
        end
    end
    print(string.format("Number of tokens = %d", #word_indices))

    return word_indices
end

function nnutils.split_seq_into_batch(data, batch_size, seq_length)
    -- Cut them into batches. 
    local len = data:size(1)
    local xdata, ydata

    if len % (batch_size * seq_length) ~= 0 then
       print('cutting off end of data so that the batches/sequences divide evenly')
       xdata = data:sub(1, batch_size * seq_length * math.floor(len / (batch_size * seq_length)))
    else 
       xdata = data
    end

    local ydata = xdata:clone()
    ydata:sub(1,-2):copy(xdata:sub(2,-1))
    ydata[-1] = xdata[1]

    local x_batches = xdata:view(batch_size, -1):split(seq_length, 2)  -- #rows = #batches
    local y_batches = ydata:view(batch_size, -1):split(seq_length, 2)  -- #rows = #batches
    assert(#x_batches == #y_batches)

    return x_batches, y_batches
end

---------------------------　Deal with configurations ----------------------------------
function nnutils.get_config(default_config)
  local config = bistro.get_params(nnutils.add_if_nonexist(default_config, 
      { lr = 0.05, gpu = 1, seed = 1111, nthread = 8, evalOnly = false, 
        train_batch = 128, test_batch = 128, max_epoch = 100, reload = '', save = './'}))

  -- reload?
  if config.evalOnly and config.reload == '' then
     error("evalOnly only works if there is a model to be reloaded!")
  end

  local net
  if config.reload ~= '' then
     require 'nn'
     require 'cutorch'
     require 'cunn'
     require 'cudnn'
     print(string.format('| reloading experiment %s', config.reload))
     if pl.path.isdir(config.reload) then
        loadfilename = string.format('%s/model.bin', config.reload)
     else 
        loadfilename = config.reload
     end
     local f = torch.DiskFile(loadfilename)
     f:binary()
     net = f:readObject()
     f:close()

     local oldconfig_file = string.format('%s/config.bin', config.reload)
     if pl.path.exists(oldconfig_file) then
        local oldconfig = torch.load(oldconfig_file)
        oldconfig.subdir = nil
        tnt.utils.table.merge(oldconfig, config)
        oldconfig.reload = nil
        oldconfig.save = config.reload
        config = oldconfig
     end

  end

  -- execute lua code in command line with config as environmentß
  -- tnt.utils.sys.cmdline(arg, config)
  print(pl.pretty.write(config))

  return config, net
end

--------------------------　Simple Torchnet framework ----------------------------------
function nnutils.torchnet_custom_merge()
  local transform = require 'torchnet.transform'
  local utils = require 'torchnet.utils'
  return transform.tableapply(
    function (field)
      if type(field) == 'table' and field[1] then
        if type(field[1]) == 'number' then
          return torch.Tensor(field)
        elseif torch.typename(field[1]) and torch.typename(field[1]):match('Tensor') then
          return utils.table.mergetensor(field)
        end
      end
      return field
    end)
end

local function old_wrap_dataset(dataset_closure, nthread, nbatch)
  local tntexp = require('fbcode.deeplearning.experimental.yuandong.torchnet.init')
  if nthread == 0 then
    -- return tnt.CudaDataset(tnt.BatchDataset{ dataset = dataset_closure(), batchsize = nbatch })
    return tntexp.CudaDataset(dataset_closure())
  else 
    -- local dataset_closure_local = dataset_closure 
    -- local batch_closure = function () 
    --     return tnt.BatchDataset{ dataset = dataset_closure_local(), batchsize = nbatch } 
    -- end
    return tntexp.CudaDataset(tntexp.ParallelDataset{
        nthread = nthread,
        closure = dataset_closure
    })
  end
end

function nnutils.run_old_torchnet(train, test, net, crit_type, config)
    local tntexp = require('fbcode.deeplearning.experimental.yuandong.torchnet.init')

    net = net:cuda()

    local crit
    if crit_type == "classification" then
       crit = nn.ClassNLLCriterion()
    elseif crit_type == "reconstruction" then
       crit = nn.MSECriterion()
    end
    crit = crit:cuda()

    config.logger_filename = config.logger_filename or string.format('%s/log', config.save)
    config.model_filename = config.model_filename or string.format("%s/model.bin", config.save)

    local log = tntexp.Logger{ filename = config.logger_filename }

    local trainer = tntexp.SGDTrainer(log)
    nnutils.add_regular_hooks(trainer, crit)
    -- check the average criterion value
    tntexp.AverageValueMeter{ rack = trainer, eval = function() return crit.output end, label = "trainloss" }
    if crit_type == "classification" then
        tntexp.ClassErrorMeter{ rack = trainer, eval = function() return net.output end, topk = {5,1}, label = "train" }  
    end


    -- log
    -- trainer:addHook(
    --    'sample',
    --    function(sample)
    --       print(sample)
    --    end
    -- )

    -- tester
    if test then 
       local tester = tntexp.SGDTester(log)
       nnutils.add_regular_hooks(tester, crit)

       tntexp.AverageValueMeter{ rack = tester, eval = function() return crit.output end, label = "testloss" }
       if crit_type == "classification" then
          tntexp.ClassErrorMeter{ rack = tester, eval = function() return net.output end, topk = {5,1}, label = "test" }  
       end

       -- we hook it to the trainer
       tester:test{
           network = net,
           dataset = old_wrap_dataset(test, config.nthread, config.test_batch), 
           rack = trainer
       }
    end

    -- customized model save
    nnutils.add_save_on_trainer(log, net, trainer, config.model_filename)

    -- log
    -- Note that this has to be put in the last, otherwise since the statistics are not fully collected, it will error.
    trainer:addHook(
       'end-epoch',
       function()
          log:print{}
          log:print{stdout = true, labels = true, separator = ' | '}
       end
    )

    -- go
    log:header{}

    trainer:train{
       network = net,
       criterion = crit,
       dataset = old_wrap_dataset(train, config.nthread, config.train_batch), 
       lr = config.lr,
       maxepoch = config.max_epoch
    }
    return log
end


local function wrap_dataset(dataset_closure, nthread)
  local tnt = require('torchnet')
  if nthread == 0 then
    -- return tnt.CudaDataset(tnt.BatchDataset{ dataset = dataset_closure(), batchsize = nbatch })
    return tnt.DatasetSampler(dataset_closure())
  else 
    -- local dataset_closure_local = dataset_closure 
    -- local batch_closure = function () 
    --     return tnt.BatchDataset{ dataset = dataset_closure_local(), batchsize = nbatch } 
    -- end
    local dataset = tnt.ParallelDatasetSampler{
        nthread = nthread,
        closure = dataset_closure
    }

    return dataset
  end
end

function nnutils.run_torchnet(train, test, net, crit_type, config)
    local tnt = require('torchnet')

    nnutils.dprint("Put network to cuda")
    net = net:cuda()

    local crit
    if crit_type == "classification" then
       crit = nn.ClassNLLCriterion()
    elseif crit_type == "reconstruction" then
       crit = nn.MSECriterion()
    end

    nnutils.dprint("Put crit to cuda")
    crit = crit:cuda()

    config.logger_filename = config.logger_filename or string.format('%s/log', config.save)
    config.model_filename = config.model_filename or string.format("%s/model.bin", config.save)

    local log = tnt.Logger{ filename = config.logger_filename }

    local engine = tnt.SGDEngine()

    -- time one iteration
    local timer = tnt.TimeMeter{ per = true }

    -- check the average criterion value
    local trainloss = tnt.AverageValueMeter()
    local testloss = tnt.AverageValueMeter()
    local trainerr = tnt.ClassErrorMeter{ topk = {5,1} }
    local testerr = tnt.ClassErrorMeter{ topk = {5,1} }
    local saved = false

    local log_terms = {
        timer = function () return timer:value()*1000 end,
        trainloss = function () return trainloss:value() end, 
        -- testloss = function () return testloss:value() end,
        trainerr1 = function () return trainerr:value(1) end, 
        testerr1 = function () return testerr:value(1) end,
        trainerr5 = function () return trainerr:value(5) end, 
        testerr5 = function () return testerr:value(5) end, 
        saved = function () return saved and '*' or ' ' end
    }

    -- customized model save
    -- we save a stateless model
    local netsav = net:clone('weight', 'bias')
    local minerr = math.huge

    local train_wrapper = wrap_dataset(train, config.nthread)
    local test_wrapper = wrap_dataset(test, config.nthread)

    -- print(train_wrapper)
    -- print("Net type = ")

    -- print(torch.typename(net))
    -- print("Crit type = ")

    -- print(torch.typename(crit))
    -- print("TrainWrapper type = ")
    -- print(torch.typename(train_wrapper))

    -- print("Lr type = ")
    -- print(type(config.lr))

    -- print("max_epoch type = ")

    -- print(type(config.max_epoch))

    -- local class = require 'class'
    -- local env = require 'argcheck.env'

    -- print(env.istype(net, 'nn.Module'))
    -- print(env.istype(crit, 'nn.Criterion'))
    -- print(env.istype(train_wrapper, 'tnt.DatasetSampler'))
    -- print(env.istype(config.lr, 'number'))
    -- print(class.type(train_wrapper))

    for event, state in
       engine:train{ network = net, criterion = crit, sampler = train_wrapper, 
                     lr = config.lr, maxepoch = config.max_epoch } do
        if event == 'start-epoch' then
--             print("nnutils.run_torchnet: In start-epoch!")
             trainloss:reset()
             trainerr:reset()
             timer:reset()
             timer:resume()
        elseif event == 'update' then
--             print("nnutils.run_torchnet: In update!")

             trainloss:add(state.criterion.output)
             trainerr:add(state.network.output, state.sample.target)
             cutorch.synchronize()
             collectgarbage()
             timer:inc()
        elseif event == 'end-epoch' then
--             print("nnutils.run_torchnet: In end-epoch!")

             timer:stop()

             -- test
             for event, state in engine:test{ network = net, sampler = test_wrapper } do
                if event == 'start-epoch' then
                   testerr:reset()
                   -- testloss:reset()
                elseif event == 'forward' then
                   collectgarbage()
                   -- testloss:add(state.criterion.output)
                   testerr:add(state.network.output, state.sample.target)
                end
             end

             -- save if better than ever
             local z = testerr:value(1)
             if z < minerr then
                local f = torch.DiskFile(config.model_filename, 'w')
                f:binary()
                f:writeObject(netsav)
                f:close()
                minerr = z
                saved = true
             else 
                saved = false
             end
             -- spit out log
             local messages = { string.format(" epoch: %d", state.epoch) }
             for k, v in pairs(log_terms) do
                 local value = v()
                 if type(value) == 'number' then 
                    if value == math.ceil(value) then
                      value = string.format("%d", value)
                    else
                      value = string.format("%.2f", value) 
                    end
                 end
                 table.insert(messages, string.format("%s: %s", k, value))
             end
             log:print(table.concat(messages, " | "))
          end
    end
    return log_terms
    -- return log
end

--------------------------- Remove batch normalization ---------------------------------

local function merge_layer(bn, linear)
   if bn == nil or linear == nil then return end

   local bn_matched = (torch.type(bn) == "nn.SpatialBatchNormalization" or torch.type(bn) == "nn.BatchNormalization")
   local linear_matched = (torch.type(linear) == "cudnn.SpatialConvolution" or torch.type(linear) == "nn.Linear")

   assert(not bn_matched or linear_matched, "Find BatchNormalization layer but linear layer is missing!")
   if (not bn_matched) or (not linear_matched) then return end

   --[[ 
    linear.weight = channelo * channeli * kh * kw
    linear.bias = channelo
    bn.weight = channelo
    bn.bias = channelo
    bn.running_mean = channelo
    bn.running_std = channelo
   --]]
   -- Note that running_std is the inverse of std.
   local device_id = bn.running_mean:getDevice()
   cutorch.withDevice(device_id, 
       function () 
           local scale = bn.running_std:clone()
           local shift = bn.running_mean:clone()

           scale:cmul(bn.weight)
           shift:cmul(scale):mul(-1.0):add(bn.bias)

           for i = 1, linear.weight:size(1) do
               linear.weight[i]:mul(scale[i])
           end
           linear.bias:cmul(scale):add(shift)
   end)

   return linear
end

local function recursive_merge_layer(model)
    local prev_mod
    for i = 1, #model.modules do
        local mod = model.modules[i]
        if mod.modules then
          recursive_merge_layer(mod)
        else
          merge_layer(mod, prev_mod)
        end
        prev_mod = mod
    end
end

local function rebuild_layers_except(old_model, except_mod_names)
    if old_model.modules then
        local new_model = old_model:clone()
        new_model.modules = {}

        for i = 1, #old_model.modules do
            local layer = rebuild_layers_except(old_model.modules[i], except_mod_names)
            if layer ~= nil then new_model:add(layer) end
        end
        return new_model
    else
        if pl.tablex.find(except_mod_names, torch.type(old_model)) == nil then 
            return old_model 
        end
    end
end

function nnutils.remove_batchnorm(model)
    -- Actual convert the model.
    recursive_merge_layer(model)
    -- Remove batch normalization layers by rebuild the model
    return rebuild_layers_except(model, { "nn.BatchNormalization", "nn.SpatialBatchNormalization" })
end

---------------------------- Remove all data parallel parallel ---------------------

local function remove_data_parallel(model)
  local res = {}
  for i, m in ipairs(model.modules) do
    if torch.typename(m) == 'nn.DataParallel' then
      table.insert(res, remove_data_parallel(m.modules[1]))
    else
      table.insert(res, m:clone())
    end
  end

  local new_model = model:clone()
  new_model.modules = res
  return new_model
end

nnutils.remove_data_parallel = remove_data_parallel

---------------------------- Convert between different permutations of classes ---------------------

local function load_list(f)
  if type(f) ~= 'string' then return f end

  local ext = pl.path.extension(f) 
  if ext == '.t7' then 
    return torch.load(f)
  elseif ext == '.lst' then
    return (require 'torchnet.utils.sys').loadlist(f, true)
  end
end

function nnutils.classconverter(sourcefile, targetfile, name_converter)
  local src = load_list(sourcefile)
  local dst = load_list(targetfile)

  local dst_inv = {}
  for i, v in ipairs(dst) do
    dst_inv[v] = i
  end

  return function (srcidx) 
    if type(srcidx) == 'number' then
      return dst_inv[name_converter(src[srcidx])] 
    elseif type(srcidx) == 'table' then
      local res = {}
      for _, i in ipairs(srcidx) do
         table.insert(res, dst_inv[name_converter(src[i])])
      end
      return res
    elseif torch.typename(srcidx) == 'torch.DoubleTensor' then
      local res = torch.DoubleTensor(srcidx:size())
      for i = 1, res:nElement() do
        res[i] = dst_inv[name_converter(src[i])]
      end
      return res
    end
  end
end

------------------ Compute the prediction ------------------
--- Top one accuracy

function nnutils.predict_compare(model, s)
  model:evaluate()
  local data_cuda = s.input:cuda()
  local res = model:forward(data_cuda)
  local max_value, max_indices = torch.max(res:float(), 2)

  local accuracy = s.target:long():eq(max_indices:long()):sum() / s.input:size(1)
  return max_indices, accuracy
end

-- Predict the top k error
-- s.input is the data, s.target is the label, topk = { 1, 3, 5} e.g.
function nnutils.predict_top(model, s, topk)
  model:evaluate()
  local data_cuda = s.input:cuda()
  local output = model:forward(data_cuda)

  local sum = {}
  local maxk = 0
  for _,k in ipairs(topk) do
    sum[k] = 0
    maxk = math.max(maxk, k)
  end
  local _, pred = output:double():sort(2, true)
  local no = output:size(1)
  for i=1,no do
    local predi = pred[i]
    local targi = s.target[i]
    local minik = math.huge
    for k=1,maxk do
       if predi[k] == targi then
          minik = k
          break
       end
    end
    for _,k in ipairs(topk) do
       if minik > k then
          sum[k] = sum[k]+1
       end
    end
  end

  for _, k in ipairs(topk) do
      sum[k] = sum[k] / no
  end

  return sum
end

------------------------------ Misc ---------------------------------------
-- Check layers with given name in net1 and net2, and skip any difference caused by bn sign.
function nnutils.compare_network_skip_bn_upto_sign(net1, net2, layername)
   local i = 1
   local j = 1

   print(string.format("#net1.modules = %d", #net1.modules))
   print(string.format("#net2.modules = %d", #net2.modules))

   while true do
     print("i = ", i)
     print("j = ", j)

     while i <= #net1.modules do
        if torch.type(net1.modules[i]) ~= layername then i = i + 1 else break end
     end
     if i > #net1.modules then break end

     while j <= #net2.modules do
        if torch.type(net2.modules[j]) ~= layername then j = j + 1 else break end
     end
     if j > #net2.modules then break end

     -- Compare their parameters
     local w1 = net1.modules[i]:parameters()
     local w2 = net2.modules[j]:parameters()
     if #w1 ~= #w2 then 
        print("compare_network_skip_bn_upto_sign: Dimension mismatch.")
        print(string.format("net1.modules[%d] = ", i))
        print(#w1)
        print(string.format("net2.modules[%d] = ", j))
        print(#w2)
        error("")
     end

     for k = 1, #w1 do
       local s1 = w1[k]:storage()
       local s2 = w2[k]:storage()

       for l = 1, s1:size() do
          if math.abs(math.abs(s1[l]) - math.abs(s2[l])) > 1e-4 then
            error(string.format("Weight net1.modules[%d][%d][%d] (= %f) is different from net2.modules[%d][%d][%d] (= %f)", i, k, l, s1[l], j, k, l, s2[l]))
          end
       end
     end

     i = i + 1
     j = j + 1
   end
end

return nnutils