BIGBALLON

"""
modfied from MorvanZhou' code!

Know more, visit my Python tutorial page: https://morvanzhou.github.io/tutorials/
My Youtube Channel: https://www.youtube.com/user/MorvanZhou
More about Reinforcement learning: https://morvanzhou.github.io/tutorials/machine-learning/reinforcement-learning/

Dependencies:
tensorflow: 1.1.0
matplotlib

BIGBALLON

dl2017@mtk:~$ strace mpirun -np 2     -H 192.168.2.243:1,192.168.3.246:1     -bind-to none -map-by slot     -x NCCL_DEBUG=INFO -x LD_LIBRARY_PATH     python3 keras_mnist_advanced.py 
execve("/usr/local/bin/mpirun", ["mpirun", "-np", "2", "-H", "192.168.2.243:1,192.168.3.246:1", "-bind-to", "none", "-map-by", "slot", "-x", "NCCL_DEBUG=INFO", "-x", "LD_LIBRARY_PATH", "python3", "keras_mnist_advanced.py"], [/* 33 vars */]) = 0
brk(NULL)                               = 0x176c000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fda47570000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/usr/local/cuda-8.0/lib64/tls/x86_64/libopen-rte.so.40", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/local/cuda-8.0/lib64/tls/x86_64", 0x7fff4ff675c0) = -1 ENOENT (No such file or directory)
open("/usr/local/cuda-8.0/lib64/tls/libopen-rte.so.40", O_RDONLY|O_CLO

BIGBALLON

(dp) dl2017@mtk:~/Desktop/horovod/examples$ strace mpirun --prefix /usr/local \
>     -np 2 \
>     -H 192.168.2.243:1,192.168.3.246:1 \
>     -bind-to none -map-by slot \
>     -x NCCL_DEBUG=INFO -x LD_LIBRARY_PATH \
>     python3 keras_mnist_advanced.py
execve("/usr/local/bin/mpirun", ["mpirun", "--prefix", "/usr/local", "-np", "2", "-H", "192.168.2.243:1,192.168.3.246:1", "-bind-to", "none", "-map-by", "slot", "-x", "NCCL_DEBUG=INFO", "-x", "LD_LIBRARY_PATH", "python3", ...], [/* 36 vars */]) = 0
brk(NULL)                               = 0x19c7000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0c9c190000

BIGBALLON

(dp) dl2017@mtk:~/Desktop/horovod/examples$ strace -f -e 'trace=!poll'     mpirun -np 2     -H 192.168.2.243:1,192.168.3.246:1     -bind-to none -map-by slot     -x NCCL_DEBUG=INFO -x LD_LIBRARY_PATH     python3 keras_mnist_advanced.pyexecve("/usr/local/bin/mpirun", ["mpirun", "-np", "2", "-H", "192.168.2.243:1,192.168.3.246:1", "-bind-to", "none", "-map-by", "slot", "-x", "NCCL_DEBUG=INFO", "-x", "LD_LIBRARY_PATH", "python3", "keras_mnist_advanced.py"], [/* 36 vars */]) = 0
brk(NULL)                               = 0x10ec000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f32b0cc6000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/usr/local/cuda-8.0/lib64/tls/x86_64/libopen-rte.so.40", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/local/cuda-8.0/lib64/tls/x86_64", 0x7ffdaecc9af0) = -1 ENOENT (No such file or directory)
open("/usr/local/c

BIGBALLON

from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf

def main(_):
  mnist = input_data.read_data_sets("./data", one_hot=True)
  x = tf.placeholder(tf.float32, [None, 784])
  W = tf.Variable(tf.truncated_normal(shape=[784, 10], stddev=0.1))
  b = tf.Variable(tf.constant(0.1, shape=[10]))
  y = tf.matmul(x, W) + b
  y_ = tf.placeholder(tf.float32, [None, 10])

BIGBALLON

"""
Lasagne implementation of SGDR on WRNs from "SGDR: Stochastic Gradient Descent with Restarts" (http://arxiv.org/abs/XXXX)
This code is based on Lasagne Recipes available at
https://github.com/Lasagne/Recipes/blob/master/papers/deep_residual_learning/Deep_Residual_Learning_CIFAR-10.py
and on WRNs implementation by Florian Muellerklein available at
https://gist.github.com/FlorianMuellerklein/3d9ba175038a3f2e7de3794fa303f1ee

"""

from __future__ import print_function

BIGBALLON

import keras
import numpy as np
import math
from keras.datasets import fashion_mnist
from keras.preprocessing.image import ImageDataGenerator
from keras.layers.normalization import BatchNormalization
from keras.layers import Conv2D, Dense, Input, add, Activation, Flatten, AveragePooling2D
from keras.callbacks import LearningRateScheduler, TensorBoard
from keras.regularizers import l2
from keras import optimizers

BIGBALLON

# *-* coding: UTF-8 *-*
__author__ = 'BG'
import urllib2
import os
import re

class ZOLPIC:
    def __init__(self):
        if not os.path.exists('./PIC'):
            os.mkdir(r'./PIC')

BIGBALLON

# 1
python3 train.py -b 128 -e 200 -d cifar10 -lr_m tanh_epoch -net lenet -depth 5 -width 1 -tanh_begin -4 -tanh_end 4 -log ./cifar10_tb4_te4_lenet_1
python3 train.py -b 128 -e 200 -d cifar10 -lr_m tanh_epoch -net lenet -depth 5 -width 1 -tanh_begin -3.5 -tanh_end 2.5 -log ./cifar10_tb35_te25_lenet_1
python3 train.py -b 128 -e 200 -d cifar10 -lr_m tanh_epoch -net lenet -depth 5 -width 1 -tanh_begin -4 -tanh_end 2.5 -log ./cifar10_tb4_te25_lenet_1
python3 train.py -b 128 -e 200 -d cifar10 -lr_m tanh_epoch -net lenet -depth 5 -width 1 -tanh_begin -2.5 -tanh_end 2.5 -log ./cifar10_tb25_te25_lenet_1
#
python3 train.py -b 128 -e 200 -d fashion_mnist -lr_m tanh_epoch -net lenet -depth 5 -width 1 -tanh_begin -4 -tanh_end 4 -log ./fashion_mnist_tb4_te4_lenet_1
python3 train.py -b 128 -e 200 -d fashion_mnist -lr_m tanh_epoch -net lenet -depth 5 -width 1 -tanh_begin -3.5 -tanh_end 2.5 -log ./fashion_mnist_tb35_te25_lenet_1
python3 train.py -b 128 -e 200 -d fashion_mnist -lr_m tanh_epoch -net lenet -depth 5 -width 1 -ta

BIGBALLON

[global]
floatX = float32
device=cuda
optimizer=fast_run

[blas]
ldflags = -L/usr/local/lib -lopenblas

[nvcc]
fastmath = True