Skip to content

Instantly share code, notes, and snippets.

@codekansas
Last active January 16, 2018 10:56
Show Gist options
  • Save codekansas/6eb99e2d7a2ca3ea174f4ca11b62fe20 to your computer and use it in GitHub Desktop.
Save codekansas/6eb99e2d7a2ca3ea174f4ca11b62fe20 to your computer and use it in GitHub Desktop.
Adding noise to gradients as a regularizer
from keras.optimizers import SGD, Adagrad, RMSprop, Adadelta, Adam, Adamax
import keras.backend as K
DEFAULT_NOISE = 0.05
def ballpark_gradient(gradient, noise):
return [g * K.random_normal(shape=K.shape(g), mean=1.0, std=noise) for g in gradient]
class BallparkSGD(SGD):
def __init__(self, noise=DEFAULT_NOISE, **kwargs):
self.noise = noise
super(BallparkSGD, self).__init__(**kwargs)
def get_gradients(self, loss, params):
grads = super(BallparkSGD, self).get_gradients(loss, params)
return ballpark_gradient(grads, self.noise)
class BallparkRMSprop(RMSprop):
def __init__(self, noise=DEFAULT_NOISE, **kwargs):
self.noise = noise
super(BallparkRMSprop, self).__init__(**kwargs)
def get_gradients(self, loss, params):
grads = super(BallparkRMSprop, self).get_gradients(loss, params)
return ballpark_gradient(grads, self.noise)
class BallparkAdagrad(Adagrad):
def __init__(self, noise=DEFAULT_NOISE, **kwargs):
self.noise = noise
super(BallparkAdagrad, self).__init__(**kwargs)
def get_gradients(self, loss, params):
grads = super(BallparkAdagrad, self).get_gradients(loss, params)
return ballpark_gradient(grads, self.noise)
class BallparkAdadelta(Adadelta):
def __init__(self, noise=DEFAULT_NOISE, **kwargs):
self.noise = noise
super(BallparkAdadelta, self).__init__(**kwargs)
def get_gradients(self, loss, params):
grads = super(BallparkAdadelta, self).get_gradients(loss, params)
return ballpark_gradient(grads, self.noise)
class BallparkAdam(Adam):
def __init__(self, noise=DEFAULT_NOISE, **kwargs):
self.noise = noise
super(BallparkAdam, self).__init__(**kwargs)
def get_gradients(self, loss, params):
grads = super(BallparkAdam, self).get_gradients(loss, params)
return ballpark_gradient(grads, self.noise)
class BallparkAdamax(Adamax):
def __init__(self, noise=DEFAULT_NOISE, **kwargs):
self.noise = noise
super(BallparkAdamax, self).__init__(**kwargs)
def get_gradients(self, loss, params):
grads = super(BallparkAdamax, self).get_gradients(loss, params)
return ballpark_gradient(grads, self.noise)
@YoelShoshan
Copy link

YoelShoshan commented Jan 16, 2018

Interesting!
Wondering - any special reason that you are using mean=1.0 ?
why don't you want the noise to be symmetrical (same chance for negative values) ?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment