enijkamp

%tensorflow_version 1.x
from tensorflow.python.client import device_lib
print(device_lib.list_local_devices())

enijkamp

import math
import pickle
import torch
import torch.nn.functional as F


def gelu(x):
    '''Gaussian Error Linear Unit - a smooth version of RELU'''
    cdf = 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
    return x * cdf

enijkamp

2019-03-12 02:49:54,679 : set device 0 out of 2 cuda devices
2019-03-12 02:49:54,680 : Namespace(checkpoint=None, data_path='./data', dataset='CIFAR10', device=0, e_increasing_layer_size=False, e_layer_size=1000, e_number_of_hidden_layers=3, epochs=100000000000000000000, gamma=1.0, initial_weights=None, lamBda=0.0, lr_init=0.0001, model='ResNet32', momentum=0.0, num_workers=4, resume_checkpoint=False, save_checkpoint=False, seed=2092, train_log_freq=40, w_batch_size=10, weight_sharing=False, wg_number_of_hidden_layers=2, with_adam=True, with_lr_plateau_schedule=False, with_norm=2, with_residual=True, x_batch_size=256, z_batch_size=20, z_dim=500, z_std=1.0)
2019-03-12 02:50:01,291 : {'weight_sharing': False, 'input_noise_size': 500, 'with_bias': True, 'wg_number_of_hidden_layers': 2, 'wg_hidden_layer_size_formula': <function train.<locals>.wg_hidden_layer_size_formula at 0x7f337b6072f0>, 'with_batchnorm': True, 'e_layer_size': 1000, 'with_residual': True, 'with_layernorm': False, 'e_increasing_layer_size': Fal

enijkamp

Traceback (most recent call last):
  File "/home/enijkamp/enijkamp@g.ucla.edu/research_students/yu_wgen/cifar10_resnet/train_wgen_resnet_multigpu.py", line 709, in <module>
    train(args_override, output_dir, setup_logging('main', output_dir, console=True), return_dict)
  File "/home/enijkamp/enijkamp@g.ucla.edu/research_students/yu_wgen/cifar10_resnet/train_wgen_resnet_multigpu.py", line 503, in train
    w = hypernet(z)
  File "/home/enijkamp/enijkamp@g.ucla.edu/research_students/yu_wgen/venv/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/enijkamp/enijkamp@g.ucla.edu/research_students/yu_wgen/venv/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 144, in forward
    return self.gather(outputs, self.output_device)
  File "/home/enijkamp/enijkamp@g.ucla.edu/research_students/yu_wgen/venv/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 156, in gather

enijkamp

from itertools import cycle

import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from matplotlib.figure import Figure

import numpy as np

import imageio

enijkamp

2019-01-22 21:26:43,521 : >>  epoch 1210 : average_loss=  2.2262, average_accuracy=42.246%,  mean_weight_std=0.000451, mean_normalized_std= 0.0224  mean_dist=  183.1
2019-01-22 21:26:43,523 : >>  epoch 1210 : median_loss=  2.2259, median_accuracy=42.230%,  median_weight_std=0.0003338, median_normalized_std= 0.0034  median_dist=  183.1
2019-01-22 21:26:43,523 : >>  epoch 1210 : ensemble accuracy: 42.25%

enijkamp

2019-01-20 23:27:25,717 : set device 1 out of 4 cuda devices
2019-01-20 23:27:25,717 : Namespace(checkpoint=None, data_path='./data', dataset='CIFAR10', device=1, e_increasing_layer_size=False, e_layer_size=200, e_number_of_hidden_layers=3, epochs=100000000000000000000, gamma=1.0, initial_weights=None, lamBda=0.0, lr_init=0.0001, model='LeNet', momentum=0.0, num_workers=4, resume_checkpoint=False, save_checkpoint=True, seed=2092, train_log_freq=1, w_batch_size=10, weight_sharing=False, wg_number_of_hidden_layers=2, with_adam=True, with_lr_plateau_schedule=False, with_norm=2, with_residual=False, x_batch_size=512, z_batch_size=10, z_dim=100, z_std=1.0)

enijkamp

2019-01-20 02:22:59,985 : >>  epoch 20   : dist pair-wise (target)
2019-01-20 02:22:59,987 : 
[[ 0.    11.072 10.757 11.233 11.483 10.696 11.791 11.471 12.791 11.896]
 [11.072  0.     9.685 10.94  10.753 10.934 10.456 11.523 11.475 11.205]
 [10.757  9.685  0.    10.439 10.51  11.268 10.206 10.785 11.371 10.588]
 [11.233 10.94  10.439  0.    10.959 11.421 11.273 11.323 11.763 11.446]
 [11.483 10.753 10.51  10.959  0.    11.945 10.611 11.584 10.717 10.703]
 [10.696 10.934 11.268 11.421 11.945  0.    12.167 11.694 12.71  12.403]
 [11.791 10.456 10.206 11.273 10.611 12.167  0.    11.947 10.735 10.685]
 [11.471 11.523 10.785 11.323 11.584 11.694 11.947  0.    12.685 12.144]

enijkamp

2019-01-20 01:54:49,042 : epoch 210  , step 385   loss0=    0.0000 loss1=   29.9530 loss2=    0.0000 lr=  0.001000
2019-01-20 01:54:49,132 : epoch 210  , step 386   loss0=    0.0000 loss1=   30.0526 loss2=    0.0000 lr=  0.001000
2019-01-20 01:54:49,243 : epoch 210  , step 387   loss0=    0.0000 loss1=   30.1210 loss2=    0.0000 lr=  0.001000
2019-01-20 01:54:49,327 : epoch 210  , step 388   loss0=    0.0000 loss1=   30.0791 loss2=    0.0000 lr=  0.001000
2019-01-20 01:54:49,421 : epoch 210  , step 389   loss0=    0.0000 loss1=   29.9636 loss2=    0.0000 lr=  0.001000
2019-01-20 01:54:49,514 : epoch 210  , step 390   loss0=    0.0000 loss1=   29.8071 loss2=    0.0000 lr=  0.001000
2019-01-20 01:55:04,683 : epoch=210 n=10 acc=0.7752 (target)
2019-01-20 01:55:23,738 : >>  epoch 210  : average_loss=  0.7859, average_accuracy=73.131%,  mean_weight_std=0.09486, mean_normalized_std=11.0705  mean_dist=  36.55
2019-01-20 01:55:23,744 : >>  epoch 210  : median_loss=  0.7838, median_accuracy=73.045%,  median_weight_std

enijkamp

loss1=    0.1201 lr=  0.000100
2019-01-14 07:50:29,196 : epoch 153  , step 277  : loss1=    0.1204 lr=  0.000100
2019-01-14 07:50:29,358 : epoch 153  , step 278  : loss1=    0.1205 lr=  0.000100
2019-01-14 07:50:29,607 : epoch 153  , step 279  : loss1=    0.1201 lr=  0.000100
2019-01-14 07:50:29,776 : epoch 153  , step 280  : loss1=    0.1208 lr=  0.000100
2019-01-14 07:50:29,898 : epoch 153  , step 281  : loss1=    0.1199 lr=  0.000100
2019-01-14 07:50:30,070 : epoch 153  , step 282  : loss1=    0.1207 lr=  0.000100
2019-01-14 07:50:30,184 : epoch 153  , step 283  : loss1=    0.1203 lr=  0.000100
2019-01-14 07:50:30,315 : epoch 153  , step 284  : loss1=    0.1203 lr=  0.000100
2019-01-14 07:50:30,460 : epoch 153  , step 285  : loss1=    0.1208 lr=  0.000100