DeepDream loop

loop.ipynd

from os import listdir
from os.path import isfile, join
from cStringIO import StringIO
import numpy as np
import scipy.ndimage as nd
import PIL.Image
from IPython.display import clear_output, Image, display
from google.protobuf import text_format
import caffe

# uncomment if you compiled caffe with GPU support
#caffe.set_mode_gpu()
#caffe.set_device(0)

def showarray(a, fmt='jpeg'):
    a = np.uint8(np.clip(a, 0, 255))
    f = StringIO()
    PIL.Image.fromarray(a).save(f, fmt)
    display(Image(data=f.getvalue()))
    
model_path = '../caffe/models/bvlc_googlenet/' # substitute your path here
net_fn   = model_path + 'deploy.prototxt'
param_fn = model_path + 'bvlc_googlenet.caffemodel'

# Patching model to be able to compute gradients.
# Note that you can also manually add "force_backward: true" line to "deploy.prototxt".
model = caffe.io.caffe_pb2.NetParameter()
text_format.Merge(open(net_fn).read(), model)
model.force_backward = True
open('tmp.prototxt', 'w').write(str(model))

net = caffe.Classifier('tmp.prototxt', param_fn
                       ,mean = np.float32([104.0, 116.0, 122.0]) # ImageNet mean, training set dependent
                       ,channel_swap = (2,1,0)
                      ) # the reference model has channels in BGR order instead of RGB

# a couple of utility functions for converting to and from Caffe's input image layout
def preprocess(net, img):
    return np.float32(np.rollaxis(img, 2)[::-1]) - net.transformer.mean['data']
def deprocess(net, img):
    return np.dstack((img + net.transformer.mean['data'])[::-1])

def make_step(net, step_size=1.5, end='inception_4c/output', jitter=32, clip=True):
    '''Basic gradient ascent step.'''

    src = net.blobs['data'] # input image is stored in Net's 'data' blob
    dst = net.blobs[end]

    ox, oy = np.random.randint(-jitter, jitter+1, 2)
    src.data[0] = np.roll(np.roll(src.data[0], ox, -1), oy, -2) # apply jitter shift
            
    net.forward(end=end)
    dst.diff[:] = dst.data  # specify the optimization objective
    net.backward(start=end)
    g = src.diff[0]
    # apply normalized ascent step to the input image
    src.data[:] += step_size/np.abs(g).mean() * g

    src.data[0] = np.roll(np.roll(src.data[0], -ox, -1), -oy, -2) # unshift image
            
    if clip:
        bias = net.transformer.mean['data']
        src.data[:] = np.clip(src.data, -bias, 255-bias)    
        
def deepdream(net, base_img, iter_n=10, octave_n=4, octave_scale=1.4, end='inception_4c/output', clip=True, **step_params):
    # prepare base images for all octaves
    octaves = [preprocess(net, base_img)]
    for i in xrange(octave_n-1):
        octaves.append(nd.zoom(octaves[-1], (1, 1.0/octave_scale,1.0/octave_scale), order=2))
    
    src = net.blobs['data']
    detail = np.zeros_like(octaves[-1]) # allocate image for network-produced details
    for octave, octave_base in enumerate(octaves[::-1]):
        h, w = octave_base.shape[-2:]
        if octave > 0:
            # upscale details from the previous octave
            h1, w1 = detail.shape[-2:]
            detail = nd.zoom(detail, (1, 1.0*h/h1,1.0*w/w1), order=2)

        src.reshape(1,3,h,w) # resize the network's input image size
        src.data[0] = octave_base+detail
        for i in xrange(iter_n):
            make_step(net, end=end, clip=clip, **step_params)
            
        # extract details produced on the current octave
        detail = src.data[0]-octave_base
    # returning the resulting image
    return deprocess(net, src.data[0])

#===================================================================================#

# using two passes improves the quality of the loop but needs twice as long
two_passes = True

# how much of the previous frame should be used in the next frame
# experiment with different values between 0.1 and 0.9
factor = 0.2

# layer of the network to use
layer = 'inception_4d/output'

# number of octaves per frame
octave_n = 4

# number of iterations per octave
iter_n = 8

# folder containing the frames
in_dir = "rabbit/"

# folder where the output frames are going to be saved
out_dir = "rabbit/out/"

# get all files in in_dir
inputs = sorted([f for f in listdir(in_dir) if isfile(join(in_dir,f))])
input_len = len(inputs)

first_frame = True
iteration_count = input_len
if two_passes:
    iteration_count *= 2

for it in xrange(iteration_count):
    image = np.float32(PIL.Image.open(in_dir + inputs[it % input_len]))
    
    if first_frame:
        first_frame = False
    else:
        # apply part of the previous update to the current frame
        image = image + diff*factor
    
    res = deepdream(net, image, jitter=0, iter_n=iter_n, octave_n=octave_n, end=layer)
    
    # store the difference to use it as bias for the next frame
    diff = res - image
    
    # save result
    if (not two_passes) or (it >= input_len):
        PIL.Image.fromarray(np.uint8(res)).save(out_dir + ("frame%04d.jpg"%(it%input_len)), quality=95) 
    
    print it+1, "/", iteration_count
    showarray(res)
    clear_output(wait=True)    

Is the d of "loop.ipynd" a typo?