cynthia/mish.py

## mish.py
# mish_original(), (c) 2019 Diganta Misra
# https://github.com/digantamisra98/Mish/blob/master/LICENSE
#
# Other functions MIT licensed, (c) 2019 Sangwhan Moon

# In [53]: %timeit mish_original(x)
# 30.2 µs ± 184 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)

# In [54]: %timeit mish1(x)
# 194 µs ± 1.68 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)

# In [55]: %timeit mish1s(x)
# 12.7 µs ± 36.5 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)

# In [56]: %timeit mish2(x)
# 149 µs ± 182 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)

# In [57]: %timeit mish2s(x)
# 12.4 µs ± 35.6 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)

def mish_original(x):
    return x * torch.tanh(torch.nn.functional.softplus(x))

def mish1(x):
    y = torch.exp(-x)
    return x * (1 + 2 * y) / (1 + 2 * y + 2 * (y ** 2))

@torch.jit.script
def mish1s(x):
    y = torch.exp(-x)
    return x * (1 + 2 * y) / (1 + 2 * y + 2 * (y ** 2))

def mish2(x):
    y = torch.exp(-x)
    r = 1 + 2 * y
    return x * r / (r + 2 * (y ** 2))

@torch.jit.script
def mish2s(x):
    y = torch.exp(-x)
    r = 1 + 2 * y
    return x * r / (r + 2 * (y ** 2))
	# mish_original(), (c) 2019 Diganta Misra
	# https://github.com/digantamisra98/Mish/blob/master/LICENSE
	#
	# Other functions MIT licensed, (c) 2019 Sangwhan Moon

	# In [53]: %timeit mish_original(x)
	# 30.2 µs ± 184 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)

	# In [54]: %timeit mish1(x)
	# 194 µs ± 1.68 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)

	# In [55]: %timeit mish1s(x)
	# 12.7 µs ± 36.5 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)

	# In [56]: %timeit mish2(x)
	# 149 µs ± 182 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)

	# In [57]: %timeit mish2s(x)
	# 12.4 µs ± 35.6 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)

	def mish_original(x):
	return x * torch.tanh(torch.nn.functional.softplus(x))

	def mish1(x):
	y = torch.exp(-x)
	return x * (1 + 2 * y) / (1 + 2 * y + 2 * (y ** 2))

	@torch.jit.script
	def mish1s(x):
	y = torch.exp(-x)
	return x * (1 + 2 * y) / (1 + 2 * y + 2 * (y ** 2))

	def mish2(x):
	y = torch.exp(-x)
	r = 1 + 2 * y
	return x * r / (r + 2 * (y ** 2))

	@torch.jit.script
	def mish2s(x):
	y = torch.exp(-x)
	r = 1 + 2 * y
	return x * r / (r + 2 * (y ** 2))