Skip to content

Instantly share code, notes, and snippets.

@Algomancer
Created June 16, 2017 14:10
Show Gist options
  • Save Algomancer/f2304ff3b623b775757b4ed9634b27c1 to your computer and use it in GitHub Desktop.
Save Algomancer/f2304ff3b623b775757b4ed9634b27c1 to your computer and use it in GitHub Desktop.
from torch.autograd import Variable
from torch.nn import functional as F
class SELU(nn.Module):
def __init__(self):
super(SELU, self).__init__()
self.alpha = 1.6732632423543772848170429916717
self.scale = 1.0507009873554804934193349852946
def forward(self, x):
a = self.scale * F.relu(x)
b = self.scale * self.alpha * (F.elu(-1 * F.relu(-1 * x)))
return a + b
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment