apaszke

Weight format proposal

Certain backends have aditional requirements for the weights, and it would be good if we could somehow transform the weights into this format once, and reuse it accross many iterations.

Currently we have two modules that could already benefit from these changes:

• cuDNN RNN could format the weights into a single contiguous block of memory
• BatchNorm could maintain fp32 running averages to stabilize fp16 training

apaszke

# Usage:
# LD_DEBUG=bindings <program> >bindings_dump 2>&1
# python print_bindings.py bindings_dump

import sys
import os
from collections import defaultdict

class colors:
    red = '\033[91m'

apaszke

from graphviz import Digraph
import torch
from torch.autograd import Variable, Function

def iter_graph(root, callback):
    queue = [root]
    seen = set()
    while queue:
        fn = queue.pop()
        if fn in seen:

apaszke

from graphviz import Digraph
import torch
from torch.autograd import Variable, Function

def iter_graph(root, callback):
    queue = [root]
    seen = set()
    while queue:
        fn = queue.pop()
        if fn in seen:

apaszke

From baaec5d9f9e0e32bbd7d089d99698e1d83966f5d Mon Sep 17 00:00:00 2001                          
From: Adam Paszke <adam.paszke@gmail.com>                                                       
Date: Thu, 11 May 2017 07:36:09 -0700                                                           
Subject: [PATCH 1/2] Disable fused RNN kernels                                                  
                                                                                                
---                                                                                             
 test/test_nn.py                               | 24 ++++++++++++++----------                    
 torch/nn/_functions/rnn.py                    |  7 ++++---                                     
 torch/nn/_functions/thnn/rnnFusedPointwise.py |  2 ++                                          
 3 files changed, 20 insertions(+), 13 deletions(-)                                             

apaszke

From 62adfff8c43c3dcd8a175fc7072f3d76d40b0a10 Mon Sep 17 00:00:00 2001
From: Adam Paszke <adam.paszke@gmail.com>
Date: Wed, 5 Apr 2017 14:17:49 -0700
Subject: [PATCH] tmp

---
 torch/autograd/function.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/torch/autograd/function.py b/torch/autograd/function.py

apaszke

#
# Copyright (c) Alex J. Champandard, 2017.
#

import PIL.Image

import torch
from torch import nn, optim
import torch.nn.functional as F
from torch.autograd import Variable

apaszke

#include <stdio.h>

void print() {
    printf("called!\n");
}

typedef void (*fn)();

struct callvec {
    fn function;

apaszke

require 'nn'
require 'nngraph'

local CleanGModule, parent = torch.class('nn.CleanGModule','nn.gModule')

local function recursiveFree(input)
      if torch.isTensor(input) then
         input:set()
      elseif torch.type(input) == 'table' then
         for i=1,#input do