Chapter 3 - Activation functions

C3_act.py

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(context='paper',
        style='ticks',
        font_scale=1,
        rc={'figure.figsize': (8, 5),
            'figure.dpi': 100,  # need fixed
            'xtick.direction': 'in',
            'ytick.direction': 'in',
            'axes.linewidth': '1.3',
            'axes.labelsize': 12,
            'xtick.labelsize': 12,
            'ytick.labelsize': 12,
            'savefig.dpi': 300,
            'legend.fontsize': 10,
            # 'text.usetex':True,
             })# Identity func
Identity = lambda x: x
# Sigmoid func
Sigmoid = lambda x: 1/(1+np.exp(-x))
# Tanh func
Tanh = lambda x: np.tanh(x) # or 2/(1+np.exp(-2*x)) - 1
# ReLU
ReLU = lambda x: np.array([i if i>=0 else 0 for i in x ])
# Leaky ReLU
LReLU = lambda x, alpha: np.array([i if i>=0 else i*alpha for i in x ])
# ELU
ELU = lambda x, alpha: np.array([i if i>=0 else alpha*(np.exp(i) -1) for i in x])

x = np.linspace(-5,5,100)
plt.figure(figsize=(8,5), dpi=100)
plt.plot(x, Identity(x), 'k-', alpha=0.8, label='Identity',linewidth=3)
plt.plot(x, Sigmoid(x), label='Sigmoid',linewidth=3)
plt.plot(x, Tanh(x), label='TanH',linewidth=3)
plt.plot(x, ReLU(x), label='ReLU', linewidth=3)
plt.plot(x, LReLU(x, 0.1), label=r'LReLU ($\alpha=0.1$)',linewidth=3)
plt.plot(x, ELU(x,1), label=r'ELU ($\alpha=1$)',linewidth=3)
plt.plot([-6,6],[0,0],'k--', alpha=0.5)
plt.plot([-6,6],[1,1],'k:', alpha=0.5)
plt.plot([0,0],[-6,6],'k--', alpha=0.5)
plt.ylim(-2,2)
plt.xlim(-4,2)
plt.xlabel('x')
plt.ylabel('f(x)')
plt.legend()
plt.savefig('C3_imags/C3_activation.png', dpi=300, bbox_inches='tight')