Skip to content

Instantly share code, notes, and snippets.

@stsievert
Last active September 10, 2018 18:52
Show Gist options
  • Save stsievert/33bdc47b52ffc085dddd75e9b719cc07 to your computer and use it in GitHub Desktop.
Save stsievert/33bdc47b52ffc085dddd75e9b719cc07 to your computer and use it in GitHub Desktop.
Testing patience for hyperparam search
Display the source blob
Display the rendered blob
Raw
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
import skorch.utils
from skorch import NeuralNetRegressor
import torch.nn as nn
import torch
import skorch
def _initialize(method, layer, gain=1):
weight = layer.weight.data
# _before = weight.data.clone()
kwargs = {'gain': gain} if 'xavier' in str(method) else {}
method(weight.data, **kwargs)
# assert torch.all(weight.data != _before)
class Autoencoder(nn.Module):
def __init__(self, activation='ReLU', init='xavier_uniform_',
**kwargs):
super().__init__()
self.activation = activation
self.init = init
self._iters = 0
init_method = getattr(torch.nn.init, init)
act_layer = getattr(nn, activation)
act_kwargs = {'inplace': True} if self.activation != 'PReLU' else {}
gain = 1
if self.activation in ['LeakyReLU', 'ReLU']:
name = 'leaky_relu' if self.activation == 'LeakyReLU' else 'relu'
gain = torch.nn.init.calculate_gain(name)
inter_dim = 28 * 28 // 4
latent_dim = inter_dim // 4
layers = [
nn.Linear(28 * 28, inter_dim),
act_layer(**act_kwargs),
nn.Linear(inter_dim, latent_dim),
act_layer(**act_kwargs)
]
for layer in layers:
if hasattr(layer, 'weight') and layer.weight.data.dim() > 1:
_initialize(init_method, layer, gain=gain)
self.encoder = nn.Sequential(*layers)
layers = [
nn.Linear(latent_dim, inter_dim),
act_layer(**act_kwargs),
nn.Linear(inter_dim, 28 * 28),
nn.Sigmoid()
]
layers = [
nn.Linear(latent_dim, 28 * 28),
nn.Sigmoid()
]
for layer in layers:
if hasattr(layer, 'weight') and layer.weight.data.dim() > 1:
_initialize(init_method, layer, gain=gain)
self.decoder = nn.Sequential(*layers)
def forward(self, x):
self._iters += 1
shape = x.size()
x = x.view(x.shape[0], -1)
x = self.encoder(x)
x = self.decoder(x)
return x.view(shape)
class NegLossScore(NeuralNetRegressor):
steps = 0
def partial_fit(self, *args, **kwargs):
super().partial_fit(*args, **kwargs)
self.steps += 1
def score(self, X, y):
X = skorch.utils.to_tensor(X, device=self.device)
y = skorch.utils.to_tensor(y, device=self.device)
self.initialize_criterion()
y_hat = self.predict(X)
y_hat = skorch.utils.to_tensor(y_hat, device=self.device)
loss = super().get_loss(y_hat, y, X=X, training=False).item()
print(f'steps = {self.steps}, loss = {loss}')
return -1 * loss
def initialize(self, *args, **kwargs):
super().initialize(*args, **kwargs)
self.callbacks_ = []
from keras.datasets import mnist
import numpy as np
import skimage.util
import random
import skimage.filters
import skimage
import scipy.signal
def noise_img(x):
noises = [
{"mode": "s&p", "amount": np.random.uniform(0.1, 0.1)},
{"mode": "gaussian", "var": np.random.uniform(0.10, 0.15)},
]
# noise = random.choice(noises)
noise = noises[1]
return skimage.util.random_noise(x, **noise)
def train_formatting(img):
img = img.reshape(28, 28).astype("float32")
return img.flat[:]
def blur_img(img):
assert img.ndim == 1
n = int(np.sqrt(img.shape[0]))
img = img.reshape(n, n)
h = np.zeros((n, n))
angle = np.random.uniform(-5, 5)
w = random.choice(range(1, 3))
h[n // 2, n // 2 - w : n // 2 + w] = 1
h = skimage.transform.rotate(h, angle)
h /= h.sum()
y = scipy.signal.convolve(img, h, mode="same")
return y.flat[:]
def dataset(n=None):
(x_train, _), (x_test, _) = mnist.load_data()
x = np.concatenate((x_train, x_test))
if n:
x = x[:n]
else:
n = int(70e3)
x = x.astype("float32") / 255.
x = np.reshape(x, (len(x), 28 * 28))
y = np.apply_along_axis(train_formatting, 1, x)
clean = y.copy()
noisy = y.copy()
# order = [noise_img, blur_img]
# order = [blur_img]
order = [noise_img]
random.shuffle(order)
for fn in order:
noisy = np.apply_along_axis(fn, 1, noisy)
noisy = noisy.astype("float32")
clean = clean.astype("float32")
# noisy = noisy.reshape(-1, 1, 28, 28).astype("float32")
# clean = clean.reshape(-1, 1, 28, 28).astype("float32")
return noisy, clean
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment