Skip to content

Instantly share code, notes, and snippets.

@stsievert
Last active April 14, 2019 23:12
Show Gist options
  • Save stsievert/981ee42e70ccf81090aba37c3afe2bc0 to your computer and use it in GitHub Desktop.
Save stsievert/981ee42e70ccf81090aba37c3afe2bc0 to your computer and use it in GitHub Desktop.
Hyperparameter comparisons (with successive halving, hyperband, stop on plateau and passive random sampling)
Display the source blob
Display the rendered blob
Raw
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
import skorch.utils
from skorch import NeuralNetRegressor
import torch.nn as nn
import torch
import skorch
def _initialize(method, layer, gain=1):
weight = layer.weight.data
# _before = weight.data.clone()
kwargs = {'gain': gain} if 'xavier' in str(method) else {}
method(weight.data, **kwargs)
# assert torch.all(weight.data != _before)
class Autoencoder(nn.Module):
def __init__(self, activation='ReLU', init='xavier_uniform_',
**kwargs):
super().__init__()
self.activation = activation
self.init = init
self._iters = 0
init_method = getattr(torch.nn.init, init)
act_layer = getattr(nn, activation)
act_kwargs = {'inplace': True} if self.activation != 'PReLU' else {}
gain = 1
if self.activation in ['LeakyReLU', 'ReLU']:
name = 'leaky_relu' if self.activation == 'LeakyReLU' else 'relu'
gain = torch.nn.init.calculate_gain(name)
inter_dim = 28 * 28 // 4
latent_dim = inter_dim // 4
layers = [
nn.Linear(28 * 28, inter_dim),
act_layer(**act_kwargs),
nn.Linear(inter_dim, latent_dim),
act_layer(**act_kwargs)
]
for layer in layers:
if hasattr(layer, 'weight') and layer.weight.data.dim() > 1:
_initialize(init_method, layer, gain=gain)
self.encoder = nn.Sequential(*layers)
layers = [
nn.Linear(latent_dim, inter_dim),
act_layer(**act_kwargs),
nn.Linear(inter_dim, 28 * 28),
nn.Sigmoid()
]
layers = [
nn.Linear(latent_dim, 28 * 28),
nn.Sigmoid()
]
for layer in layers:
if hasattr(layer, 'weight') and layer.weight.data.dim() > 1:
_initialize(init_method, layer, gain=gain)
self.decoder = nn.Sequential(*layers)
def forward(self, x):
self._iters += 1
shape = x.size()
x = x.view(x.shape[0], -1)
x = self.encoder(x)
x = self.decoder(x)
return x.view(shape)
class NegLossScore(NeuralNetRegressor):
steps = 0
def partial_fit(self, *args, **kwargs):
super().partial_fit(*args, **kwargs)
self.steps += 1
def score(self, X, y):
X = skorch.utils.to_tensor(X, device=self.device)
y = skorch.utils.to_tensor(y, device=self.device)
self.initialize_criterion()
y_hat = self.predict(X)
y_hat = skorch.utils.to_tensor(y_hat, device=self.device)
loss = super().get_loss(y_hat, y, X=X, training=False).item()
print(f'steps = {self.steps}, loss = {loss}')
return -1 * loss
def initialize(self, *args, **kwargs):
super().initialize(*args, **kwargs)
self.callbacks_ = []
from keras.datasets import mnist
import numpy as np
import skimage.util
import random
import skimage.filters
import skimage
import scipy.signal
def noise_img(x):
noises = [
{"mode": "s&p", "amount": np.random.uniform(0.1, 0.1)},
{"mode": "gaussian", "var": np.random.uniform(0.10, 0.15)},
]
# noise = random.choice(noises)
noise = noises[1]
return skimage.util.random_noise(x, **noise)
def train_formatting(img):
img = img.reshape(28, 28).astype("float32")
return img.flat[:]
def blur_img(img):
assert img.ndim == 1
n = int(np.sqrt(img.shape[0]))
img = img.reshape(n, n)
h = np.zeros((n, n))
angle = np.random.uniform(-5, 5)
w = random.choice(range(1, 3))
h[n // 2, n // 2 - w : n // 2 + w] = 1
h = skimage.transform.rotate(h, angle)
h /= h.sum()
y = scipy.signal.convolve(img, h, mode="same")
return y.flat[:]
def dataset(n=None):
(x_train, _), (x_test, _) = mnist.load_data()
x = np.concatenate((x_train, x_test))
if n:
x = x[:n]
else:
n = int(70e3)
x = x.astype("float32") / 255.
x = np.reshape(x, (len(x), 28 * 28))
y = np.apply_along_axis(train_formatting, 1, x)
clean = y.copy()
noisy = y.copy()
# order = [noise_img, blur_img]
# order = [blur_img]
order = [noise_img]
random.shuffle(order)
for fn in order:
noisy = np.apply_along_axis(fn, 1, noisy)
noisy = noisy.astype("float32")
clean = clean.astype("float32")
# noisy = noisy.reshape(-1, 1, 28, 28).astype("float32")
# clean = clean.reshape(-1, 1, 28, 28).astype("float32")
return noisy, clean
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment