Skip to content

Instantly share code, notes, and snippets.

@charlesreid1

charlesreid1/example_hyperas.ipynb

Created May 20, 2019 20:53
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save charlesreid1/3ea4561566f6e9277c29ad242cee9811 to your computer and use it in GitHub Desktop.
Save charlesreid1/3ea4561566f6e9277c29ad242cee9811 to your computer and use it in GitHub Desktop.
Download ZIP
example_hyperas.ipynb
Display the source blob
Display the rendered blob
Raw
example_hyperas.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "example_hyperas.ipynb",
"version": "0.3.2",
"provenance": [],
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/charlesreid1/3ea4561566f6e9277c29ad242cee9811/example_hyperas.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"metadata": {
"id": "tOG1pjCA9XKU",
"colab_type": "code",
"colab": {}
},
"source": [
"!pip install --upgrade jupyter-console &> /dev/null"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "lKbEk3su9au8",
"colab_type": "code",
"colab": {}
},
"source": [
"!pip install --upgrade hyperas &> /dev/null"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "tGBuNeOL_Y2F",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 52
},
"outputId": "b43fa103-1d6d-40ac-92c5-019e33d2abd2"
},
"source": [
"from google.colab import drive\n",
"drive.mount('/gdrive')\n",
"%ls /gdrive"
],
"execution_count": 33,
"outputs": [
{
"output_type": "stream",
"text": [
"Drive already mounted at /gdrive; to attempt to forcibly remount, call drive.mount(\"/gdrive\", force_remount=True).\n",
"\u001b[0m\u001b[01;34m'My Drive'\u001b[0m/\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "sP0w6p9y_jwI",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 312
},
"outputId": "07324a32-6212-41ee-c8ec-46dd6ba97eaa"
},
"source": [
"!ls '/gdrive/My Drive/Colab Notebooks'"
],
"execution_count": 34,
"outputs": [
{
"output_type": "stream",
"text": [
"'Copy of hyperparameter_cnn1d_dna_transcription.ipynb'\n",
" data-exploration_dna_transcription.ipynb\n",
" example_hyperas.ipynb\n",
" hyperparameter_cnn1d_dna_transcription.ipynb\n",
" keras_cnn1d_dna_transcription_powerx.ipynb\n",
" keras_cnn1d_dna_transcription-round2.ipynb\n",
" keras-mnist.ipynb\n",
" keras-sklearn_cnn1d_dna_transcription.ipynb\n",
" keras-sklearn_cnn1d_dna_transcription_powerx.ipynb\n",
" keras-sklearn_cnn1d_dna_transcription_quantx.ipynb\n",
" old_deepchem_ch6_improved.ipynb\n",
" old_deepchem-ch6.ipynb\n",
" old_deepchem-chp3.ipynb\n",
" old_keras-ch6-improved.ipynb\n",
" old_keras-ch6.ipynb\n",
" old_Untitled0.ipynb\n",
" Untitled0.ipynb\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "DbaqFwkt9n73",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 3076
},
"outputId": "16eec283-d7ac-4aee-8167-89efb637755a"
},
"source": [
"import numpy as np\n",
"import os\n",
"\n",
"from hyperopt import Trials, STATUS_OK, tpe\n",
"from keras.datasets import mnist\n",
"from keras.layers.core import Dense, Dropout, Activation\n",
"from keras.models import Sequential\n",
"from keras.utils import np_utils\n",
"\n",
"from hyperas import optim\n",
"from hyperas.distributions import choice, uniform\n",
"\n",
"\n",
"def data():\n",
" \"\"\"\n",
" Data providing function:\n",
"\n",
" This function is separated from create_model() so that hyperopt\n",
" won't reload data for each evaluation run.\n",
" \"\"\"\n",
" (x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
" x_train = x_train.reshape(60000, 784)\n",
" x_test = x_test.reshape(10000, 784)\n",
" x_train = x_train.astype('float32')\n",
" x_test = x_test.astype('float32')\n",
" x_train /= 255\n",
" x_test /= 255\n",
" nb_classes = 10\n",
" y_train = np_utils.to_categorical(y_train, nb_classes)\n",
" y_test = np_utils.to_categorical(y_test, nb_classes)\n",
" return x_train, y_train, x_test, y_test\n",
"\n",
"\n",
"def create_model(x_train, y_train, x_test, y_test):\n",
" \"\"\"\n",
" Model providing function:\n",
"\n",
" Create Keras model with double curly brackets dropped-in as needed.\n",
" Return value has to be a valid python dictionary with two customary keys:\n",
" - loss: Specify a numeric evaluation metric to be minimized\n",
" - status: Just use STATUS_OK and see hyperopt documentation if not feasible\n",
" The last one is optional, though recommended, namely:\n",
" - model: specify the model just created so that we can later use it again.\n",
" \"\"\"\n",
" model = Sequential()\n",
" model.add(Dense(512, input_shape=(784,)))\n",
" model.add(Activation('relu'))\n",
" model.add(Dropout({{uniform(0, 1)}}))\n",
" model.add(Dense({{choice([256, 512, 1024])}}))\n",
" model.add(Activation({{choice(['relu', 'sigmoid'])}}))\n",
" model.add(Dropout({{uniform(0, 1)}}))\n",
"\n",
" # If we choose 'four', add an additional fourth layer\n",
" if {{choice(['three', 'four'])}} == 'four':\n",
" model.add(Dense(100))\n",
"\n",
" # We can also choose between complete sets of layers\n",
"\n",
" model.add({{choice([Dropout(0.5), Activation('linear')])}})\n",
" model.add(Activation('relu'))\n",
"\n",
" model.add(Dense(10))\n",
" model.add(Activation('softmax'))\n",
"\n",
" model.compile(loss='categorical_crossentropy', metrics=['accuracy'],\n",
" optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})\n",
"\n",
" result = model.fit(x_train, y_train,\n",
" batch_size={{choice([64, 128])}},\n",
" epochs=2,\n",
" verbose=2,\n",
" validation_split=0.1)\n",
" #get the highest validation accuracy of the training epochs\n",
" validation_acc = np.amax(result.history['val_acc']) \n",
" print('Best validation acc of epoch:', validation_acc)\n",
" return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}\n",
"\n",
"if __name__ == '__main__':\n",
" best_run, best_model = optim.minimize(model=create_model,\n",
" data=data,\n",
" algo=tpe.suggest,\n",
" max_evals=5,\n",
" trials=Trials(),\n",
" notebook_name=os.path.join('..','gdrive','My Drive','Colab Notebooks','example_hyperas'))\n",
" X_train, Y_train, X_test, Y_test = data()\n",
" print(\"Evalutation of best performing model:\")\n",
" print(best_model.evaluate(X_test, Y_test))\n",
" print(\"Best performing model chosen hyper-parameters:\")\n",
" print(best_run)"
],
"execution_count": 0,
"outputs": [
{
"output_type": "stream",
"text": [
">>> Imports:\n",
"#coding=utf-8\n",
"\n",
"try:\n",
" from google.colab import drive\n",
"except:\n",
" pass\n",
"\n",
"try:\n",
" import numpy as np\n",
"except:\n",
" pass\n",
"\n",
"try:\n",
" import os\n",
"except:\n",
" pass\n",
"\n",
"try:\n",
" from hyperopt import Trials, STATUS_OK, tpe\n",
"except:\n",
" pass\n",
"\n",
"try:\n",
" from keras.datasets import mnist\n",
"except:\n",
" pass\n",
"\n",
"try:\n",
" from keras.layers.core import Dense, Dropout, Activation\n",
"except:\n",
" pass\n",
"\n",
"try:\n",
" from keras.models import Sequential\n",
"except:\n",
" pass\n",
"\n",
"try:\n",
" from keras.utils import np_utils\n",
"except:\n",
" pass\n",
"\n",
"try:\n",
" from hyperas import optim\n",
"except:\n",
" pass\n",
"\n",
"try:\n",
" from hyperas.distributions import choice, uniform\n",
"except:\n",
" pass\n",
"\n",
">>> Hyperas search space:\n",
"\n",
"def get_space():\n",
" return {\n",
" 'Dropout': hp.uniform('Dropout', 0, 1),\n",
" 'Dense': hp.choice('Dense', [256, 512, 1024]),\n",
" 'Activation': hp.choice('Activation', ['relu', 'sigmoid']),\n",
" 'Dropout_1': hp.uniform('Dropout_1', 0, 1),\n",
" 'Dropout_2': hp.choice('Dropout_2', ['three', 'four']),\n",
" 'add': hp.choice('add', [Dropout(0.5), Activation('linear')]),\n",
" 'optimizer': hp.choice('optimizer', ['rmsprop', 'adam', 'sgd']),\n",
" 'batch_size': hp.choice('batch_size', [64, 128]),\n",
" }\n",
"\n",
">>> Data\n",
" 1: \n",
" 2: \"\"\"\n",
" 3: Data providing function:\n",
" 4: \n",
" 5: This function is separated from create_model() so that hyperopt\n",
" 6: won't reload data for each evaluation run.\n",
" 7: \"\"\"\n",
" 8: (x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
" 9: x_train = x_train.reshape(60000, 784)\n",
" 10: x_test = x_test.reshape(10000, 784)\n",
" 11: x_train = x_train.astype('float32')\n",
" 12: x_test = x_test.astype('float32')\n",
" 13: x_train /= 255\n",
" 14: x_test /= 255\n",
" 15: nb_classes = 10\n",
" 16: y_train = np_utils.to_categorical(y_train, nb_classes)\n",
" 17: y_test = np_utils.to_categorical(y_test, nb_classes)\n",
" 18: \n",
" 19: \n",
" 20: \n",
">>> Resulting replaced keras model:\n",
"\n",
" 1: def keras_fmin_fnct(space):\n",
" 2: \n",
" 3: \"\"\"\n",
" 4: Model providing function:\n",
" 5: \n",
" 6: Create Keras model with double curly brackets dropped-in as needed.\n",
" 7: Return value has to be a valid python dictionary with two customary keys:\n",
" 8: - loss: Specify a numeric evaluation metric to be minimized\n",
" 9: - status: Just use STATUS_OK and see hyperopt documentation if not feasible\n",
" 10: The last one is optional, though recommended, namely:\n",
" 11: - model: specify the model just created so that we can later use it again.\n",
" 12: \"\"\"\n",
" 13: model = Sequential()\n",
" 14: model.add(Dense(512, input_shape=(784,)))\n",
" 15: model.add(Activation('relu'))\n",
" 16: model.add(Dropout(space['Dropout']))\n",
" 17: model.add(Dense(space['Dense']))\n",
" 18: model.add(Activation(space['Activation']))\n",
" 19: model.add(Dropout(space['Dropout_1']))\n",
" 20: \n",
" 21: # If we choose 'four', add an additional fourth layer\n",
" 22: if space['Dropout_2'] == 'four':\n",
" 23: model.add(Dense(100))\n",
" 24: \n",
" 25: # We can also choose between complete sets of layers\n",
" 26: \n",
" 27: model.add(space['add'])\n",
" 28: model.add(Activation('relu'))\n",
" 29: \n",
" 30: model.add(Dense(10))\n",
" 31: model.add(Activation('softmax'))\n",
" 32: \n",
" 33: model.compile(loss='categorical_crossentropy', metrics=['accuracy'],\n",
" 34: optimizer=space['optimizer'])\n",
" 35: \n",
" 36: result = model.fit(x_train, y_train,\n",
" 37: batch_size=space['batch_size'],\n",
" 38: epochs=2,\n",
" 39: verbose=2,\n",
" 40: validation_split=0.1)\n",
" 41: #get the highest validation accuracy of the training epochs\n",
" 42: validation_acc = np.amax(result.history['val_acc']) \n",
" 43: print('Best validation acc of epoch:', validation_acc)\n",
" 44: return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}\n",
" 45: \n",
"Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n",
"11493376/11490434 [==============================] - 1s 0us/step\n",
" 0%| | 0/5 [00:00<?, ?it/s, best loss: ?]WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Colocations handled automatically by placer.\n",
"WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
"WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Use tf.cast instead.\n",
"Train on 54000 samples, validate on 6000 samples\n",
"Epoch 1/2\n",
" - 10s - loss: 1.7256 - acc: 0.4249 - val_loss: 0.8002 - val_acc: 0.8493\n",
"\n",
"Epoch 2/2\n",
" - 10s - loss: 0.9461 - acc: 0.6979 - val_loss: 0.4520 - val_acc: 0.8923\n",
"\n",
"Best validation acc of epoch:\n",
"0.8923333334922791\n",
"Train on 54000 samples, validate on 6000 samples\n",
"Epoch 1/2\n",
" - 15s - loss: 2.2159 - acc: 0.2832 - val_loss: 0.7372 - val_acc: 0.7705\n",
"\n",
"Epoch 2/2\n",
" - 15s - loss: 1.5323 - acc: 0.4610 - val_loss: 0.5590 - val_acc: 0.8627\n",
"\n",
"Best validation acc of epoch:\n",
"0.8626666666666667\n",
"Train on 54000 samples, validate on 6000 samples\n",
"Epoch 1/2\n",
" - 10s - loss: 2.0434 - acc: 0.2848 - val_loss: 0.7661 - val_acc: 0.8870\n",
"\n",
"Epoch 2/2\n",
" - 9s - loss: 1.3212 - acc: 0.5004 - val_loss: 0.4248 - val_acc: 0.9358\n",
"\n",
"Best validation acc of epoch:\n",
"0.935833333492279\n",
"Train on 54000 samples, validate on 6000 samples\n",
"Epoch 1/2\n",
" 60%|██████ | 3/5 [01:11<00:44, 22.49s/it, best loss: -0.935833333492279]"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "jVM817xM-bA1",
"colab_type": "code",
"colab": {}
},
"source": [
""
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "S7m2fHyv-etq",
"colab_type": "code",
"colab": {}
},
"source": [
""
],
"execution_count": 0,
"outputs": []
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment