Skip to content

Instantly share code, notes, and snippets.

@huseinzol05

huseinzol05/iris-mlp-softmax-crossentropy-numpy.ipynb

Last active October 4, 2018 20:31
Show Gist options
  • Save huseinzol05/b1b0cfe87a669cdc0cc66f0f661dc86a to your computer and use it in GitHub Desktop.
Save huseinzol05/b1b0cfe87a669cdc0cc66f0f661dc86a to your computer and use it in GitHub Desktop.
Download ZIP
implement 3 hidden layer feed-forward with softmax and cross-entropy for Iris dataset
Display the source blob
Display the rendered blob
Raw
iris-mlp-softmax-crossentropy-numpy.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
" \"This module will be removed in 0.20.\", DeprecationWarning)\n"
]
}
],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from sklearn.preprocessing import MinMaxScaler\n",
"from sklearn.preprocessing import LabelEncoder\n",
"from sklearn.cross_validation import train_test_split"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Id</th>\n",
" <th>SepalLengthCm</th>\n",
" <th>SepalWidthCm</th>\n",
" <th>PetalLengthCm</th>\n",
" <th>PetalWidthCm</th>\n",
" <th>Species</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>5.1</td>\n",
" <td>3.5</td>\n",
" <td>1.4</td>\n",
" <td>0.2</td>\n",
" <td>Iris-setosa</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>4.9</td>\n",
" <td>3.0</td>\n",
" <td>1.4</td>\n",
" <td>0.2</td>\n",
" <td>Iris-setosa</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>3</td>\n",
" <td>4.7</td>\n",
" <td>3.2</td>\n",
" <td>1.3</td>\n",
" <td>0.2</td>\n",
" <td>Iris-setosa</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4</td>\n",
" <td>4.6</td>\n",
" <td>3.1</td>\n",
" <td>1.5</td>\n",
" <td>0.2</td>\n",
" <td>Iris-setosa</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>5</td>\n",
" <td>5.0</td>\n",
" <td>3.6</td>\n",
" <td>1.4</td>\n",
" <td>0.2</td>\n",
" <td>Iris-setosa</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Id SepalLengthCm SepalWidthCm PetalLengthCm PetalWidthCm Species\n",
"0 1 5.1 3.5 1.4 0.2 Iris-setosa\n",
"1 2 4.9 3.0 1.4 0.2 Iris-setosa\n",
"2 3 4.7 3.2 1.3 0.2 Iris-setosa\n",
"3 4 4.6 3.1 1.5 0.2 Iris-setosa\n",
"4 5 5.0 3.6 1.4 0.2 Iris-setosa"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df = pd.read_csv('Iris.csv')\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"# normalized between (0, 1) to prevent gradient vanishing caused by out sigmoid\n",
"X = MinMaxScaler().fit_transform(df.iloc[:,1:-1].values)\n",
"Y = LabelEncoder().fit_transform(df.iloc[:, -1])\n",
"onehot = np.zeros((Y.shape[0], np.unique(Y).shape[0]))\n",
"onehot[range(Y.shape[0]),Y] = 1.0\n",
"X_train, X_test, Y_train, Y_test = train_test_split(df.iloc[:,1:-1].values, onehot,test_size=0.2)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# reference: https://en.wikipedia.org/wiki/Activation_function\n",
"def sigmoid(X, grad=False):\n",
" if grad:\n",
" return sigmoid(X) * (1 - sigmoid(X))\n",
" else:\n",
" return 1 / (1 + np.exp(-X))\n",
" \n",
"def softmax(X, grad=False):\n",
" if grad:\n",
" p = softmax(X)\n",
" return p * (1-p)\n",
" else:\n",
" e_x = np.exp(X - np.max(X, axis=-1, keepdims=True))\n",
" return e_x / np.sum(e_x, axis=-1, keepdims=True)\n",
"\n",
"# reference: https://en.wikipedia.org/wiki/Cross_entropy\n",
"# change from log2 to log10 or loge doesnt hurt much, it is still an equation, minima is our target\n",
"def cross_entropy(X, Y, grad=False):\n",
" if grad:\n",
" X = np.clip(X, 1e-15, 1 - 1e-15)\n",
" return -(Y / X) + (1 - Y) / (1 - X)\n",
" else:\n",
" X = np.clip(X, 1e-15, 1 - 1e-15)\n",
" return -Y * np.log(X) - (1 - Y) * np.log(1 - X)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"W1 = np.random.randn(X_train.shape[1], 50) / np.sqrt(df.shape[1])\n",
"b1 = np.zeros((50))\n",
"W2 = np.random.randn(50, 100) / np.sqrt(50)\n",
"b2 = np.zeros((100))\n",
"W3 = np.random.randn(100, np.unique(Y).shape[0]) / np.sqrt(100)\n",
"b3 = np.zeros((np.unique(Y).shape[0]))\n",
"\n",
"EPOCH = 50\n",
"LEARNING_RATE = 0.01"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"epoch 0, accuracy 0.350000, cost 0.656911\n",
"epoch 1, accuracy 0.341667, cost 1.834073\n",
"epoch 2, accuracy 0.308333, cost 3.114981\n",
"epoch 3, accuracy 0.350000, cost 3.377698\n",
"epoch 4, accuracy 0.308333, cost 0.633299\n",
"epoch 5, accuracy 0.691667, cost 0.622091\n",
"epoch 6, accuracy 0.566667, cost 0.618965\n",
"epoch 7, accuracy 0.458333, cost 0.616203\n",
"epoch 8, accuracy 0.500000, cost 0.613087\n",
"epoch 9, accuracy 0.566667, cost 0.609445\n",
"epoch 10, accuracy 0.608333, cost 0.605143\n",
"epoch 11, accuracy 0.675000, cost 0.600024\n",
"epoch 12, accuracy 0.683333, cost 0.593896\n",
"epoch 13, accuracy 0.683333, cost 0.586508\n",
"epoch 14, accuracy 0.691667, cost 0.577539\n",
"epoch 15, accuracy 0.691667, cost 0.566599\n",
"epoch 16, accuracy 0.691667, cost 0.553297\n",
"epoch 17, accuracy 0.691667, cost 0.537389\n",
"epoch 18, accuracy 0.691667, cost 0.518886\n",
"epoch 19, accuracy 0.691667, cost 0.498092\n",
"epoch 20, accuracy 0.691667, cost 0.475773\n",
"epoch 21, accuracy 0.691667, cost 0.453268\n",
"epoch 22, accuracy 0.691667, cost 0.431952\n",
"epoch 23, accuracy 0.691667, cost 0.412542\n",
"epoch 24, accuracy 0.700000, cost 0.395272\n",
"epoch 25, accuracy 0.700000, cost 0.380061\n",
"epoch 26, accuracy 0.725000, cost 0.366821\n",
"epoch 27, accuracy 0.725000, cost 0.355286\n",
"epoch 28, accuracy 0.725000, cost 0.345315\n",
"epoch 29, accuracy 0.733333, cost 0.336567\n",
"epoch 30, accuracy 0.750000, cost 0.328933\n",
"epoch 31, accuracy 0.783333, cost 0.322089\n",
"epoch 32, accuracy 0.808333, cost 0.315966\n",
"epoch 33, accuracy 0.825000, cost 0.310315\n",
"epoch 34, accuracy 0.841667, cost 0.305078\n",
"epoch 35, accuracy 0.841667, cost 0.300091\n",
"epoch 36, accuracy 0.850000, cost 0.295296\n",
"epoch 37, accuracy 0.866667, cost 0.290590\n",
"epoch 38, accuracy 0.875000, cost 0.285924\n",
"epoch 39, accuracy 0.891667, cost 0.281233\n",
"epoch 40, accuracy 0.908333, cost 0.276485\n",
"epoch 41, accuracy 0.925000, cost 0.271640\n",
"epoch 42, accuracy 0.933333, cost 0.266691\n",
"epoch 43, accuracy 0.941667, cost 0.261614\n",
"epoch 44, accuracy 0.950000, cost 0.256435\n",
"epoch 45, accuracy 0.958333, cost 0.251126\n",
"epoch 46, accuracy 0.958333, cost 0.245783\n",
"epoch 47, accuracy 0.958333, cost 0.240328\n",
"epoch 48, accuracy 0.933333, cost 0.235533\n",
"epoch 49, accuracy 0.958333, cost 0.234271\n"
]
}
],
"source": [
"# a(x, w) = x.dot(w)\n",
"# z(x) = sigmoid(x)\n",
"# y_hat(x) = softmax(x)\n",
"# E(y_hat, y) = cross_entropy(y_hat, y)\n",
"\n",
"# feed-forward\n",
"# a1(x, w1) -> z1(a1) -> a2(z1, w2) -> z2(a2) -> a3(z2, w3) -> y_hat(a3) -> E(y_hat, y)\n",
"\n",
"# back-propagation\n",
"# a1(x, w1) <- dz1(a1) <- da2(z1, w2) <- dz2(a2) <- da3(z2, w3) <- dy_hat(a3) <- dE(y_hat, y)\n",
"\n",
"for i in range(EPOCH):\n",
" a1 = np.dot(X_train, W1) + b1\n",
" z1 = sigmoid(a1)\n",
" a2 = np.dot(z1, W2) + b2\n",
" z2 = sigmoid(a2)\n",
" a3 = np.dot(z2, W3) + b3\n",
" y_hat = softmax(a3)\n",
" accuracy = np.mean(np.argmax(y_hat,axis = 1) == np.argmax(Y_train,axis = 1))\n",
" cost = np.mean(cross_entropy(y_hat,Y_train))\n",
" dy_hat = cross_entropy(y_hat,Y_train, grad=True)\n",
" da3 = softmax(a3, True) * dy_hat\n",
" dW3 = z2.T.dot(da3)\n",
" db3 = np.sum(da3, axis=0)\n",
" dz2 = da3.dot(W3.T)\n",
" da2 = sigmoid(a2, True) * dz2\n",
" dW2 = z1.T.dot(da2)\n",
" db2 = np.sum(da2, axis=0)\n",
" dz1 = da2.dot(W2.T)\n",
" da1 = sigmoid(a1, True) * dz1\n",
" dW1 = X_train.T.dot(da1)\n",
" db1 = np.sum(da1, axis=0)\n",
" W3 += -LEARNING_RATE * dW3\n",
" b3 += -LEARNING_RATE * db3\n",
" W2 += -LEARNING_RATE * dW2\n",
" b2 += -LEARNING_RATE * db2\n",
" W1 += -LEARNING_RATE * dW1\n",
" b1 += -LEARNING_RATE * db1\n",
" print('epoch %d, accuracy %f, cost %f'%(i, accuracy, cost))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment