SINGA cifar10 CNN Model

README.md

This is a shared example model of SINGA. Install SINGA and run this model with command:

singa -m train

file.cfg

[data]
data_file=https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz

network.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================

import sys
import os

from singa import layer
from singa import metric
from singa import loss
from singa import net as ffnet

def add_layer_group(net, name, nb_filers, sample_shape=None):
    net.add(layer.Conv2D(name + '_1', nb_filers, 3, 1, pad=1,
                         input_sample_shape=sample_shape))
    net.add(layer.Activation(name + '_1'))
    net.add(layer.Conv2D(name + '_2', nb_filers, 3, 1, pad=1))
    net.add(layer.Activation(name + '_3'))
    net.add(layer.MaxPooling2D(name, 2, 2, pad=0))


def create_vgg():
    net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
    add_layer_group(net, 'conv1', 64, (3, 32, 32))
    add_layer_group(net, 'conv2', 128)
    add_layer_group(net, 'conv3', 256)
    add_layer_group(net, 'conv4', 512)
    add_layer_group(net, 'conv5', 512)
    net.add(layer.Flatten('flat'))
    net.add(layer.Dense('ip1', 512))
    net.add(layer.Activation('relu_ip1'))
    net.add(layer.Dropout('drop1'))
    net.add(layer.Dense('ip2', 10))
    return net


def create_alexnet():
    net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
    W0 = {'init': 'gaussian', 'mean': 0, 'std': 0.0001}
    W1 = {'init': 'gaussian', 'mean': 0, 'std': 0.01}
    W2 = {'init': 'gaussian', 'mean': 0, 'std': 0.01, 'decay_mult': 250}
    b = {'init': 'constant', 'value': 0, 'lt_mult': 2}

    net.add(layer.Conv2D('conv1', 32, 5, 1, W_specs=W0.copy(), b_specs=b.copy(),
                         pad=2, input_sample_shape=(3, 32, 32)))
    net.add(layer.MaxPooling2D('pool1', 3, 2, pad=1))
    net.add(layer.Activation('relu1'))
    net.add(layer.LRN(name='lrn1'))
    net.add(layer.Conv2D('conv2', 32, 5, 1, W_specs=W1.copy(), b_specs=b.copy(),
                         pad=2))
    net.add(layer.Activation('relu2'))
    net.add(layer.MaxPooling2D('pool2', 3, 2,  pad=1))
    net.add(layer.LRN('lrn2'))
    net.add(layer.Conv2D('conv3', 64, 5, 1, W_specs=W1.copy(), b_specs=b.copy(),
                         pad=2))
    net.add(layer.Activation('relu3'))
    net.add(layer.MaxPooling2D('pool3', 3, 2, pad=1))
    net.add(layer.Flatten('flat'))
    net.add(layer.Dense('dense', 10, W_specs=W2.copy(), b_specs=b.copy()))
    return net

def create():
    return create_alexnet()

process.py

#!/usr/bin/env python
#/************************************************************
#*
#* Licensed to the Apache Software Foundation (ASF) under one
#* or more contributor license agreements.  See the NOTICE file
#* distributed with this work for additional information
#* regarding copyright ownership.  The ASF licenses this file
#* to you under the Apache License, Version 2.0 (the
#* "License"); you may not use this file except in compliance
#* with the License.  You may obtain a copy of the License at
#*
#*   http://www.apache.org/licenses/LICENSE-2.0
#*
#* Unless required by applicable law or agreed to in writing,
#* software distributed under the License is distributed on an
#* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#* KIND, either express or implied.  See the License for the
#* specific language governing permissions and limitations
#* under the License.
#*
#*************************************************************/

#**************
#*sudo apt-get install libjpeg-dev
#*sudo pip install 

from PIL import Image
import sys, glob, os, random, shutil, time
import numpy as np

def do_resize(img,small_size):
  size = img.size
  if size[0]<size[1]:
    new_size = ( small_size, int(small_size*size[1]/size[0]) )
  else:
    new_size = ( int(small_size*size[0]/size[1]), small_size )
  new_img=img.resize(new_size)
  #print "resize to %d,%d" % new_size
  return new_img

def do_crop(img,crop,position):
  if img.size[0] < crop[0]:
    raise Exception('img size[0] %d is smaller than crop[0]: %d' % (img[0],crop[0]))
  if img.size[1] < crop[1]:
    raise Exception('img size[1] %d is smaller than crop[1]: %d' % (img[1],crop[1]))
  if position == 'left_top':
    left=0
    upper=0
  if position == 'left_bottom':
    left=0
    upper=img.size[1]-crop[1]
  if position == 'right_top':
    left=img.size[0]-crop[0]
    upper=0
  if position == 'right_bottom':
    left=img.size[0]-crop[0]
    upper=img.size[1]-crop[1]
  if position == 'center':
    left=(img.size[0]-crop[0])/2
    upper=(img.size[1]-crop[1])/2

  box =(left,upper,left+crop[0],upper+crop[1])
  new_img = img.crop(box)
  #print "crop to box %d,%d,%d,%d" % box
  return new_img

def do_flip(img):
  new_img = img.transpose(Image.FLIP_LEFT_RIGHT)
  return new_img

def load_img(path, grayscale=False):
  from PIL import Image
  img = Image.open(path)
  if grayscale:
      img = img.convert('L')
  else:  # Ensure 3 channel even when loaded image is grayscale
      img = img.convert('RGB')
  return img
  
def process_img(
            img,
            small_size,
            size,
            is_aug
                     ):
    im = load_img(img)
    im = do_resize(im,small_size)
    dataArray = []

    if is_aug:
        positions=["left_top","left_bottom","right_top","right_bottom","center"]
    else:
        positions=["center"]
    for position in positions:
        newIm=do_crop(im,size,position)
        assert newIm.size==size
        pix = np.array(newIm.convert("RGB"))
        dataArray.append(pix.transpose(2,0,1))
        if is_aug:
            newIm=do_flip(newIm)
            pix = np.array(newIm.convert("RGB"))
            dataArray.append(pix.transpose(2,0,1))

    return dataArray

def unpickle(file):
    import cPickle
    fo = open(file, 'rb')
    dict = cPickle.load(fo)
    fo.close()
    return dict

serve.py

from singa import tensor
import os
import numpy as np
import data_ as d
import process

data_path="data_"
topk=5

class Service():

    def __init__(self, model,device):
        self.model=model
        if hasattr(d,"mean_file"):
            self.mean = np.load(d.mean_file)
        else:
            self.mean = np.load(os.path.join(data_path,"train.mean.npy"))
        self.device = device

    def serve(self,request):
        image = request.files['image']
        if not image:
            return "error, no image file found!"
        if not allowed_file(image.filename):
            return "error, only jpg image is allowed."
        try:
            #process images
            images=process.process_img(image,36,(32,32),True)
            images=np.array(images[0:10]).astype(np.float32)

            #normalize
            images -= self.mean

            x = tensor.from_numpy(images.astype(np.float32))
            x.to_device(self.device)
            y = self.model.predict(x)
            y.to_host()
            y = tensor.to_numpy(y)
            prob = np.average(y, 0)
            #sort and reverse 
            labels = np.flipud(np.argsort(prob))

            response =""
            for i in range(topk):
                response += "%d:%s\n" % (labels[i],prob[labels[i]])
            return response
        except Exception as e:
            print e
            return "sorry, system error."

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1] in ["jpg","JPG","JPEG","jpeg"]

train.py

from singa import tensor, device, optimizer
from singa import utils, initializer, metric
from singa.proto import core_pb2
import os, sys
import cPickle
import numpy as np
import data_ as d
import process

data_path="data_"
data_folder="cifar-10-batches-py"

class Trainer():

    def __init__(self,model,device):
        self.model=model
        self.device = device
        self.opt = optimizer.SGD(momentum=0.9, weight_decay=0.0005)

    def initialize(self):
        print 'Start intialization............'
        self.model.to_device(self.device)
        for (p, specs) in zip(self.model.param_values(), self.model.param_specs()):
            filler = specs.filler
            if filler.type == 'gaussian':
                initializer.gaussian(p, filler.mean, filler.std)
            elif filler.type == 'xavier':
                initializer.xavier(p)
                p *= 0.5  # 0.5 if use glorot, which would have val acc to 83
            else:
                p.set_value(0)
            self.opt.register(p, specs)
            print specs.name, filler.type, p.l1()
        print 'End intialization............'

    def data_prepare(self):

        data_dir = os.path.join(data_path,data_folder)

        self.train_x, self.train_y = load_train_data(data_dir)
        self.test_x, self.test_y = load_test_data(data_dir)
        self.mean = np.average(self.train_x, axis=0)
        self.train_x -= self.mean
        self.test_x -= self.mean

        mean_path = os.path.join(data_path,'train.mean')
        np.save(mean_path, self.mean)
        print 'saved mean to %s' % mean_path
       

    def train(self, num_epoch=140, batch_size=50):
      
        if not os.path.exists(os.path.join(data_path,data_folder)):
            untar_data(d.data_file,data_path)

        self.data_prepare()

        print 'training shape', self.train_x.shape, self.train_y.shape
        print 'validation shape', self.test_x.shape, self.test_y.shape

        tx = tensor.Tensor((batch_size, 3, 32, 32), self.device)
        ty = tensor.Tensor((batch_size,), self.device,core_pb2.kInt)

        num_train_batch = self.train_x.shape[0] / batch_size
        num_test_batch = self.test_x.shape[0] / (batch_size)

        accuracy = metric.Accuracy()
        idx = np.arange(self.train_x.shape[0], dtype=np.int32)
        for epoch in range(num_epoch):
            np.random.shuffle(idx)
            loss, acc = 0.0, 0.0
            print 'Epoch %d' % epoch
            for b in range(num_train_batch):
                x = self.train_x[idx[b * batch_size: (b + 1) * batch_size]] 
                y = self.train_y[idx[b * batch_size: (b + 1) * batch_size]]
                tx.copy_from_numpy(x)
                ty.copy_from_numpy(y)
                grads, (l, a) = self.model.train(tx, ty)
                loss += l
                acc += a
                for (s, p, g) in zip(self.model.param_specs(), self.model.param_values(), grads):
                    self.opt.apply_with_lr(epoch, get_lr(epoch), g, p, str(s.name))
                info = 'training loss = %f, training accuracy = %f' % (l, a)

                # update progress bar
                utils.update_progress(b * 1.0 / num_train_batch, info)
            print ""
            info = 'training loss = %f, training accuracy = %f' \
                % (loss / num_train_batch, acc / num_train_batch)
            print info
            loss,acc = 0.0, 0.0
            for b in range(num_test_batch):
                x = self.test_x[b * batch_size: (b + 1) * batch_size]
                y = self.test_y[b * batch_size: (b + 1) * batch_size]
                tx.copy_from_numpy(x)
                ty.copy_from_numpy(y)
                l, a = self.model.evaluate(tx, ty)
                loss += l
                acc += a
    
            print 'testing loss = %f, accuracy = %f' % (loss / num_test_batch, acc / num_test_batch)
            if epoch > 0 and epoch % 10 == 0:
                self.model.save('model_%d.bin' % epoch)
        self.model.save('model.bin')
        return

def get_lr(epoch):
    if epoch < 360:
        return 0.0008
    elif epoch < 540:
        return 0.0001
    else:
        return 0.00001

def load_dataset(filepath):
    print 'Loading data file %s' % filepath
    with open(filepath, 'rb') as fd:
        cifar10 = cPickle.load(fd)
    image = cifar10['data'].astype(dtype=np.uint8)
    image = image.reshape((-1, 3, 32, 32))
    label = np.asarray(cifar10['labels'], dtype=np.uint8)
    label = label.reshape(label.size, 1)
    return image, label


def load_train_data(dir_path, num_batches=5):
    labels = []
    batchsize = 10000
    images = np.empty((num_batches * batchsize, 3, 32, 32), dtype=np.uint8)
    for did in range(1, num_batches + 1):
        fname_train_data = dir_path + "/data_batch_{}".format(did)
        image, label = load_dataset(fname_train_data)
        images[(did - 1) * batchsize:did * batchsize] = image
        labels.extend(label)
    images = np.array(images, dtype=np.float32)
    labels = np.array(labels, dtype=np.int32)
    return images, labels


def load_test_data(dir_path):
    images, labels = load_dataset(dir_path + "/test_batch")
    return np.array(images,  dtype=np.float32), np.array(labels, dtype=np.int32)

def untar_data(file_path,dest):
    print 'untar data ..................'
    tar_file = file_path
    import tarfile
    tar = tarfile.open(tar_file)
    tar.extractall(dest)
    tar.close()