torch_snippets.md

Torch Notes

1. How to use boolean variables in cmd:

cmd:option('--sample_patch', false, 'generate patch or image?')

So default is false, when one wants to turn that option on: th myfile.lua --sample_path

2. Writing network architecture in option: For instance, if we have opt.arch = 'o-19,c-5-1-2-32,c-1-1-0-1024', then the structure of this network is:

   nn.Sequential {
     [input -> (1) -> (2) -> (3) -> output]
     (1): nn.Sequential {
       [input -> (1) -> (2) -> output]
       (1): nn.OneHot
       (2): nn.Transpose
     }
     (2): nn.Sequential {
       [input -> (1) -> (2) -> (3) -> output]
       (1): nn.SpatialConvolution(3 -> 32, 5x5, 1,1, 2,2)
       (2): nn.SpatialBatchNormalization
       (3): nn.ReLU
     }
     (3): nn.Sequential {
       [input -> (1) -> (2) -> (3) -> output]
       (1): nn.SpatialConvolution(32 -> 1024, 1x1)
       (2): nn.SpatialBatchNormalization
       (3): nn.ReLU
     }
   }
   
   

   require 'nn'
   require 'dpnn'
   
   local M = {}
   
   local function build_conv_block(nInputPlane, nOutputPlane, kW, kH, dW, dH, padW, padH)
     --[[
     This function build a convolution block: Convolution -> BatchNormalization -> ReLU
     Sample Input: c-5-1-2-32, this will build a convolutional layer with kernel size 5,
                   stride 1, padding 2 and output plane 32.
     ]]
     local conv_block = nn.Sequential()
   
     conv_block:add(nn.SpatialConvolution(nInputPlane, nOutputPlane, kW, kH, dW, dH, padW, padH)) --receptive field: 5x5
     conv_block:add(nn.SpatialBatchNormalization(nOutputPlane))
     conv_block:add(nn.ReLU(true)) -- 32x128x128
   
     return conv_block
   end
   
   local function build_onehot_block(outputSize)
     --[[
     This function build a one-hot block
     Sample Input: o-19, this will build a one-hot layer with a outputSize of 19
     ]]
     local onehot_block = nn.Sequential()
   
     onehot_block:add(nn.OneHot(outputSize)) --receptive field: 5x5
     onehot_block:add(nn.Transpose({2,4},{4,3}))
   
   
     return onehot_block
   end
   
   function M.build_model(opt)
     local arch = opt.arch:split(',')
     local model = nn.Sequential()
     local prev_dim = opt.input_dim -- input image dimension
     local next_dim, layer
   
     for i,v in ipairs(arch) do
       local params = string.split(v,'-')
       if params[1] == 'c' then -- convolutional layer
         local f = tonumber(params[2]) -- filter size
         local d = tonumber(params[3]) -- stride
         local p = tonumber(params[4]) -- padding size
         next_dim = tonumber(params[5]) -- nOutputPlane
         layer = build_conv_block(prev_dim, next_dim, f, f, d, d, p, p)
       elseif params[1] == 'o' then -- one hot layer
         local outputSize = tonumber(params[2]) -- outputSize
         layer = build_onehot_block(outputSize)
         next_dim = prev_dim
       end
   
       model:add(layer)
       prev_dim = next_dim
     end
   
     return model
   end
   
   return M

3. Implement own layer

   require 'torch'
   require 'nn'
       
   local LayerName, parent = torch.class('nn.LayerName', 'nn.Module')
   
   function LayerName:__init()
     parent.__init(self)
     -- initialization code
   end
   
   function LayerName:updateOutput(input)
     -- forward pass
   end
   
   function LayerName:updateGradInput(input, gradOutput)
     -- backward pass
   end  
   
   function LayerName:setMode(mode)
     -- other functions
   end

4. Set up GPU

local M = {}
function M.setup_gpu(gpu, backend, use_cudnn)
  local dtype = 'torch.FloatTensor'
  if gpu >= 0 then
    if backend == 'cuda' then
      require 'cutorch'
      require 'cunn'
      cutorch.setDevice(gpu + 1)
      dtype = 'torch.CudaTensor'
      if use_cudnn then
        require 'cudnn'
        cudnn.benchmark = true
      end
    elseif backend == 'opencl' then
      require 'cltorch'
      require 'clnn'
      cltorch.setDevice(gpu + 1)
      dtype = torch.Tensor():cl():type()
      use_cudnn = false
    end
  else
    use_cudnn = false
  end
  return dtype, use_cudnn
end
return M

local dtype, use_cudnn = setup_gpu(opt.gpu, opt.backend, opt.use_cudnn)
if use_cudnn then cudnn.convert(model, cudnn) end