Created
May 20, 2019 20:53
-
-
Save charlesreid1/3ea4561566f6e9277c29ad242cee9811 to your computer and use it in GitHub Desktop.
example_hyperas.ipynb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"nbformat": 4, | |
"nbformat_minor": 0, | |
"metadata": { | |
"colab": { | |
"name": "example_hyperas.ipynb", | |
"version": "0.3.2", | |
"provenance": [], | |
"include_colab_link": true | |
}, | |
"kernelspec": { | |
"name": "python3", | |
"display_name": "Python 3" | |
} | |
}, | |
"cells": [ | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"id": "view-in-github", | |
"colab_type": "text" | |
}, | |
"source": [ | |
"<a href=\"https://colab.research.google.com/gist/charlesreid1/3ea4561566f6e9277c29ad242cee9811/example_hyperas.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "tOG1pjCA9XKU", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"!pip install --upgrade jupyter-console &> /dev/null" | |
], | |
"execution_count": 0, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "lKbEk3su9au8", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"!pip install --upgrade hyperas &> /dev/null" | |
], | |
"execution_count": 0, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "tGBuNeOL_Y2F", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 52 | |
}, | |
"outputId": "b43fa103-1d6d-40ac-92c5-019e33d2abd2" | |
}, | |
"source": [ | |
"from google.colab import drive\n", | |
"drive.mount('/gdrive')\n", | |
"%ls /gdrive" | |
], | |
"execution_count": 33, | |
"outputs": [ | |
{ | |
"output_type": "stream", | |
"text": [ | |
"Drive already mounted at /gdrive; to attempt to forcibly remount, call drive.mount(\"/gdrive\", force_remount=True).\n", | |
"\u001b[0m\u001b[01;34m'My Drive'\u001b[0m/\n" | |
], | |
"name": "stdout" | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "sP0w6p9y_jwI", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 312 | |
}, | |
"outputId": "07324a32-6212-41ee-c8ec-46dd6ba97eaa" | |
}, | |
"source": [ | |
"!ls '/gdrive/My Drive/Colab Notebooks'" | |
], | |
"execution_count": 34, | |
"outputs": [ | |
{ | |
"output_type": "stream", | |
"text": [ | |
"'Copy of hyperparameter_cnn1d_dna_transcription.ipynb'\n", | |
" data-exploration_dna_transcription.ipynb\n", | |
" example_hyperas.ipynb\n", | |
" hyperparameter_cnn1d_dna_transcription.ipynb\n", | |
" keras_cnn1d_dna_transcription_powerx.ipynb\n", | |
" keras_cnn1d_dna_transcription-round2.ipynb\n", | |
" keras-mnist.ipynb\n", | |
" keras-sklearn_cnn1d_dna_transcription.ipynb\n", | |
" keras-sklearn_cnn1d_dna_transcription_powerx.ipynb\n", | |
" keras-sklearn_cnn1d_dna_transcription_quantx.ipynb\n", | |
" old_deepchem_ch6_improved.ipynb\n", | |
" old_deepchem-ch6.ipynb\n", | |
" old_deepchem-chp3.ipynb\n", | |
" old_keras-ch6-improved.ipynb\n", | |
" old_keras-ch6.ipynb\n", | |
" old_Untitled0.ipynb\n", | |
" Untitled0.ipynb\n" | |
], | |
"name": "stdout" | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "DbaqFwkt9n73", | |
"colab_type": "code", | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 3076 | |
}, | |
"outputId": "16eec283-d7ac-4aee-8167-89efb637755a" | |
}, | |
"source": [ | |
"import numpy as np\n", | |
"import os\n", | |
"\n", | |
"from hyperopt import Trials, STATUS_OK, tpe\n", | |
"from keras.datasets import mnist\n", | |
"from keras.layers.core import Dense, Dropout, Activation\n", | |
"from keras.models import Sequential\n", | |
"from keras.utils import np_utils\n", | |
"\n", | |
"from hyperas import optim\n", | |
"from hyperas.distributions import choice, uniform\n", | |
"\n", | |
"\n", | |
"def data():\n", | |
" \"\"\"\n", | |
" Data providing function:\n", | |
"\n", | |
" This function is separated from create_model() so that hyperopt\n", | |
" won't reload data for each evaluation run.\n", | |
" \"\"\"\n", | |
" (x_train, y_train), (x_test, y_test) = mnist.load_data()\n", | |
" x_train = x_train.reshape(60000, 784)\n", | |
" x_test = x_test.reshape(10000, 784)\n", | |
" x_train = x_train.astype('float32')\n", | |
" x_test = x_test.astype('float32')\n", | |
" x_train /= 255\n", | |
" x_test /= 255\n", | |
" nb_classes = 10\n", | |
" y_train = np_utils.to_categorical(y_train, nb_classes)\n", | |
" y_test = np_utils.to_categorical(y_test, nb_classes)\n", | |
" return x_train, y_train, x_test, y_test\n", | |
"\n", | |
"\n", | |
"def create_model(x_train, y_train, x_test, y_test):\n", | |
" \"\"\"\n", | |
" Model providing function:\n", | |
"\n", | |
" Create Keras model with double curly brackets dropped-in as needed.\n", | |
" Return value has to be a valid python dictionary with two customary keys:\n", | |
" - loss: Specify a numeric evaluation metric to be minimized\n", | |
" - status: Just use STATUS_OK and see hyperopt documentation if not feasible\n", | |
" The last one is optional, though recommended, namely:\n", | |
" - model: specify the model just created so that we can later use it again.\n", | |
" \"\"\"\n", | |
" model = Sequential()\n", | |
" model.add(Dense(512, input_shape=(784,)))\n", | |
" model.add(Activation('relu'))\n", | |
" model.add(Dropout({{uniform(0, 1)}}))\n", | |
" model.add(Dense({{choice([256, 512, 1024])}}))\n", | |
" model.add(Activation({{choice(['relu', 'sigmoid'])}}))\n", | |
" model.add(Dropout({{uniform(0, 1)}}))\n", | |
"\n", | |
" # If we choose 'four', add an additional fourth layer\n", | |
" if {{choice(['three', 'four'])}} == 'four':\n", | |
" model.add(Dense(100))\n", | |
"\n", | |
" # We can also choose between complete sets of layers\n", | |
"\n", | |
" model.add({{choice([Dropout(0.5), Activation('linear')])}})\n", | |
" model.add(Activation('relu'))\n", | |
"\n", | |
" model.add(Dense(10))\n", | |
" model.add(Activation('softmax'))\n", | |
"\n", | |
" model.compile(loss='categorical_crossentropy', metrics=['accuracy'],\n", | |
" optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})\n", | |
"\n", | |
" result = model.fit(x_train, y_train,\n", | |
" batch_size={{choice([64, 128])}},\n", | |
" epochs=2,\n", | |
" verbose=2,\n", | |
" validation_split=0.1)\n", | |
" #get the highest validation accuracy of the training epochs\n", | |
" validation_acc = np.amax(result.history['val_acc']) \n", | |
" print('Best validation acc of epoch:', validation_acc)\n", | |
" return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}\n", | |
"\n", | |
"if __name__ == '__main__':\n", | |
" best_run, best_model = optim.minimize(model=create_model,\n", | |
" data=data,\n", | |
" algo=tpe.suggest,\n", | |
" max_evals=5,\n", | |
" trials=Trials(),\n", | |
" notebook_name=os.path.join('..','gdrive','My Drive','Colab Notebooks','example_hyperas'))\n", | |
" X_train, Y_train, X_test, Y_test = data()\n", | |
" print(\"Evalutation of best performing model:\")\n", | |
" print(best_model.evaluate(X_test, Y_test))\n", | |
" print(\"Best performing model chosen hyper-parameters:\")\n", | |
" print(best_run)" | |
], | |
"execution_count": 0, | |
"outputs": [ | |
{ | |
"output_type": "stream", | |
"text": [ | |
">>> Imports:\n", | |
"#coding=utf-8\n", | |
"\n", | |
"try:\n", | |
" from google.colab import drive\n", | |
"except:\n", | |
" pass\n", | |
"\n", | |
"try:\n", | |
" import numpy as np\n", | |
"except:\n", | |
" pass\n", | |
"\n", | |
"try:\n", | |
" import os\n", | |
"except:\n", | |
" pass\n", | |
"\n", | |
"try:\n", | |
" from hyperopt import Trials, STATUS_OK, tpe\n", | |
"except:\n", | |
" pass\n", | |
"\n", | |
"try:\n", | |
" from keras.datasets import mnist\n", | |
"except:\n", | |
" pass\n", | |
"\n", | |
"try:\n", | |
" from keras.layers.core import Dense, Dropout, Activation\n", | |
"except:\n", | |
" pass\n", | |
"\n", | |
"try:\n", | |
" from keras.models import Sequential\n", | |
"except:\n", | |
" pass\n", | |
"\n", | |
"try:\n", | |
" from keras.utils import np_utils\n", | |
"except:\n", | |
" pass\n", | |
"\n", | |
"try:\n", | |
" from hyperas import optim\n", | |
"except:\n", | |
" pass\n", | |
"\n", | |
"try:\n", | |
" from hyperas.distributions import choice, uniform\n", | |
"except:\n", | |
" pass\n", | |
"\n", | |
">>> Hyperas search space:\n", | |
"\n", | |
"def get_space():\n", | |
" return {\n", | |
" 'Dropout': hp.uniform('Dropout', 0, 1),\n", | |
" 'Dense': hp.choice('Dense', [256, 512, 1024]),\n", | |
" 'Activation': hp.choice('Activation', ['relu', 'sigmoid']),\n", | |
" 'Dropout_1': hp.uniform('Dropout_1', 0, 1),\n", | |
" 'Dropout_2': hp.choice('Dropout_2', ['three', 'four']),\n", | |
" 'add': hp.choice('add', [Dropout(0.5), Activation('linear')]),\n", | |
" 'optimizer': hp.choice('optimizer', ['rmsprop', 'adam', 'sgd']),\n", | |
" 'batch_size': hp.choice('batch_size', [64, 128]),\n", | |
" }\n", | |
"\n", | |
">>> Data\n", | |
" 1: \n", | |
" 2: \"\"\"\n", | |
" 3: Data providing function:\n", | |
" 4: \n", | |
" 5: This function is separated from create_model() so that hyperopt\n", | |
" 6: won't reload data for each evaluation run.\n", | |
" 7: \"\"\"\n", | |
" 8: (x_train, y_train), (x_test, y_test) = mnist.load_data()\n", | |
" 9: x_train = x_train.reshape(60000, 784)\n", | |
" 10: x_test = x_test.reshape(10000, 784)\n", | |
" 11: x_train = x_train.astype('float32')\n", | |
" 12: x_test = x_test.astype('float32')\n", | |
" 13: x_train /= 255\n", | |
" 14: x_test /= 255\n", | |
" 15: nb_classes = 10\n", | |
" 16: y_train = np_utils.to_categorical(y_train, nb_classes)\n", | |
" 17: y_test = np_utils.to_categorical(y_test, nb_classes)\n", | |
" 18: \n", | |
" 19: \n", | |
" 20: \n", | |
">>> Resulting replaced keras model:\n", | |
"\n", | |
" 1: def keras_fmin_fnct(space):\n", | |
" 2: \n", | |
" 3: \"\"\"\n", | |
" 4: Model providing function:\n", | |
" 5: \n", | |
" 6: Create Keras model with double curly brackets dropped-in as needed.\n", | |
" 7: Return value has to be a valid python dictionary with two customary keys:\n", | |
" 8: - loss: Specify a numeric evaluation metric to be minimized\n", | |
" 9: - status: Just use STATUS_OK and see hyperopt documentation if not feasible\n", | |
" 10: The last one is optional, though recommended, namely:\n", | |
" 11: - model: specify the model just created so that we can later use it again.\n", | |
" 12: \"\"\"\n", | |
" 13: model = Sequential()\n", | |
" 14: model.add(Dense(512, input_shape=(784,)))\n", | |
" 15: model.add(Activation('relu'))\n", | |
" 16: model.add(Dropout(space['Dropout']))\n", | |
" 17: model.add(Dense(space['Dense']))\n", | |
" 18: model.add(Activation(space['Activation']))\n", | |
" 19: model.add(Dropout(space['Dropout_1']))\n", | |
" 20: \n", | |
" 21: # If we choose 'four', add an additional fourth layer\n", | |
" 22: if space['Dropout_2'] == 'four':\n", | |
" 23: model.add(Dense(100))\n", | |
" 24: \n", | |
" 25: # We can also choose between complete sets of layers\n", | |
" 26: \n", | |
" 27: model.add(space['add'])\n", | |
" 28: model.add(Activation('relu'))\n", | |
" 29: \n", | |
" 30: model.add(Dense(10))\n", | |
" 31: model.add(Activation('softmax'))\n", | |
" 32: \n", | |
" 33: model.compile(loss='categorical_crossentropy', metrics=['accuracy'],\n", | |
" 34: optimizer=space['optimizer'])\n", | |
" 35: \n", | |
" 36: result = model.fit(x_train, y_train,\n", | |
" 37: batch_size=space['batch_size'],\n", | |
" 38: epochs=2,\n", | |
" 39: verbose=2,\n", | |
" 40: validation_split=0.1)\n", | |
" 41: #get the highest validation accuracy of the training epochs\n", | |
" 42: validation_acc = np.amax(result.history['val_acc']) \n", | |
" 43: print('Best validation acc of epoch:', validation_acc)\n", | |
" 44: return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}\n", | |
" 45: \n", | |
"Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n", | |
"11493376/11490434 [==============================] - 1s 0us/step\n", | |
" 0%| | 0/5 [00:00<?, ?it/s, best loss: ?]WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", | |
"Instructions for updating:\n", | |
"Colocations handled automatically by placer.\n", | |
"WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", | |
"Instructions for updating:\n", | |
"Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n", | |
"WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", | |
"Instructions for updating:\n", | |
"Use tf.cast instead.\n", | |
"Train on 54000 samples, validate on 6000 samples\n", | |
"Epoch 1/2\n", | |
" - 10s - loss: 1.7256 - acc: 0.4249 - val_loss: 0.8002 - val_acc: 0.8493\n", | |
"\n", | |
"Epoch 2/2\n", | |
" - 10s - loss: 0.9461 - acc: 0.6979 - val_loss: 0.4520 - val_acc: 0.8923\n", | |
"\n", | |
"Best validation acc of epoch:\n", | |
"0.8923333334922791\n", | |
"Train on 54000 samples, validate on 6000 samples\n", | |
"Epoch 1/2\n", | |
" - 15s - loss: 2.2159 - acc: 0.2832 - val_loss: 0.7372 - val_acc: 0.7705\n", | |
"\n", | |
"Epoch 2/2\n", | |
" - 15s - loss: 1.5323 - acc: 0.4610 - val_loss: 0.5590 - val_acc: 0.8627\n", | |
"\n", | |
"Best validation acc of epoch:\n", | |
"0.8626666666666667\n", | |
"Train on 54000 samples, validate on 6000 samples\n", | |
"Epoch 1/2\n", | |
" - 10s - loss: 2.0434 - acc: 0.2848 - val_loss: 0.7661 - val_acc: 0.8870\n", | |
"\n", | |
"Epoch 2/2\n", | |
" - 9s - loss: 1.3212 - acc: 0.5004 - val_loss: 0.4248 - val_acc: 0.9358\n", | |
"\n", | |
"Best validation acc of epoch:\n", | |
"0.935833333492279\n", | |
"Train on 54000 samples, validate on 6000 samples\n", | |
"Epoch 1/2\n", | |
" 60%|██████ | 3/5 [01:11<00:44, 22.49s/it, best loss: -0.935833333492279]" | |
], | |
"name": "stdout" | |
} | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "jVM817xM-bA1", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"" | |
], | |
"execution_count": 0, | |
"outputs": [] | |
}, | |
{ | |
"cell_type": "code", | |
"metadata": { | |
"id": "S7m2fHyv-etq", | |
"colab_type": "code", | |
"colab": {} | |
}, | |
"source": [ | |
"" | |
], | |
"execution_count": 0, | |
"outputs": [] | |
} | |
] | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment