hsm207/keras_estimator_feature_column_bug.ipynb

## keras_estimator_feature_column_bug.ipynb
{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "keras_estimator_feature_column_bug.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/hsm207/3ab2a5b31bc3b21f8466d426abaa8b81/keras_estimator_feature_column_bug.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "anBNYmvyK4mv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d53345f2-8f18-4592-e77f-e2728e49aaf8"
      },
      "source": [
        "%tensorflow_version 2.x"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 2.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "plFd4bpO_ehj",
        "colab_type": "code",
        "outputId": "ca750f0f-7946-4d92-f4d1-a9767cb91e61",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!python -c \"import tensorflow as tf; print(tf.version.GIT_VERSION, tf.version.VERSION)\""
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "v2.0.0-rc2-26-g64c3d38 2.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z18OovDp_0Wk",
        "colab_type": "code",
        "outputId": "3168e8c0-753a-4b0f-b997-e248d73fb828",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "!python -c \"import sys; print(sys.version)\""
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "3.6.8 (default, Oct  7 2019, 12:59:55) \n",
            "[GCC 8.3.0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RLy8JGb28_H6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "import pandas as pd\n",
        "\n",
        "\n",
        "def create_model():\n",
        "    store_feature = tf.feature_column.categorical_column_with_vocabulary_list('store',\n",
        "                                                                              vocabulary_list=['a', 'b'])\n",
        "\n",
        "    store_feature = tf.feature_column.embedding_column(store_feature, dimension=64)\n",
        "\n",
        "    loc_feature = tf.feature_column.categorical_column_with_vocabulary_list('loc',\n",
        "                                                                              vocabulary_list=['x', 'y', 'z'])\n",
        "\n",
        "    loc_feature = tf.feature_column.embedding_column(loc_feature, dimension=32)\n",
        "\n",
        "    model = tf.keras.Sequential([\n",
        "        tf.keras.layers.DenseFeatures(feature_columns=[store_feature, loc_feature]),\n",
        "        tf.keras.layers.Dense(32, activation='relu'),\n",
        "        tf.keras.layers.Dense(1, activation='relu')\n",
        "    ])\n",
        "\n",
        "    return model\n",
        "\n",
        "df = pd.DataFrame({'store': ['a', 'b', 'b', 'a'],\n",
        "                   'loc': ['x', 'y', 'z', 'x'],\n",
        "                   'label': [100., 200., 400., 100.]})\n",
        "\n",
        "\n",
        "labels = df.pop('label')\n",
        "\n",
        "train_ds = tf.data.Dataset.from_tensor_slices((dict(df), labels)).batch(2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cw2ULTET9dHk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_1 = create_model()\n",
        "model_1.compile(optimizer='adam', loss='mse', metrics=['mse'])\n",
        "\n",
        "# # call fit to build model because model_1.build() won't work\n",
        "# # also to double check that training works on train_ds\n",
        "# model_1.fit(train_ds)\n",
        "\n",
        "# print('The keras model\"s input names are: {}'.format(model_1.input_names))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U6274TZO9fqg",
        "colab_type": "code",
        "outputId": "bf4b3455-f044-4a37-abc2-c6c3379e33e3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 700
        }
      },
      "source": [
        "est_1 = tf.keras.estimator.model_to_estimator(keras_model=model_1)\n",
        "\n",
        "\n",
        "def input_fn():\n",
        "    ds = tf.data.Dataset.from_tensor_slices((dict(df), labels)).batch(2)\n",
        "    return ds.repeat()\n",
        "\n",
        "train_spec = tf.estimator.TrainSpec(input_fn = input_fn, max_steps=4 * 3)\n",
        "eval_spec = tf.estimator.EvalSpec(input_fn=input_fn,\n",
        "                                  steps=12)\n",
        "\n",
        "tf.estimator.train_and_evaluate(est_1, train_spec, eval_spec)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Using default config.\n",
            "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpbdefa2vg\n",
            "INFO:tensorflow:Using the Keras model provided.\n",
            "INFO:tensorflow:Using config: {'_model_dir': '/tmp/tmpbdefa2vg', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n",
            "graph_options {\n",
            "  rewrite_options {\n",
            "    meta_optimizer_iterations: ONE\n",
            "  }\n",
            "}\n",
            ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7febe034a320>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
            "INFO:tensorflow:Not using Distribute Coordinator.\n",
            "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
            "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
            "WARNING:tensorflow:From /tensorflow-2.0.0/python3.6/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n",
            "WARNING:tensorflow:From /tensorflow-2.0.0/python3.6/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n",
            "INFO:tensorflow:Calling model_fn.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-6-d191f2777eb2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m                                   steps=12)\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_and_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mest_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_spec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meval_spec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/training.py\u001b[0m in \u001b[0;36mtrain_and_evaluate\u001b[0;34m(estimator, train_spec, eval_spec)\u001b[0m\n\u001b[1;32m    471\u001b[0m         '(with task id 0).  Given task id {}'.format(config.task_id))\n\u001b[1;32m    472\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 473\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mexecutor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    474\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    475\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/training.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    611\u001b[0m         config.task_type != run_config_lib.TaskType.EVALUATOR):\n\u001b[1;32m    612\u001b[0m       \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Running training and evaluation locally (non-distributed).'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 613\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_local\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    614\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    615\u001b[0m     \u001b[0;31m# Distributed case.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/training.py\u001b[0m in \u001b[0;36mrun_local\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    712\u001b[0m         \u001b[0mmax_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_spec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    713\u001b[0m         \u001b[0mhooks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain_hooks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 714\u001b[0;31m         saving_listeners=saving_listeners)\n\u001b[0m\u001b[1;32m    715\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    716\u001b[0m     eval_result = listener_for_eval.eval_result or _EvalResult(\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/estimator.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, input_fn, hooks, steps, max_steps, saving_listeners)\u001b[0m\n\u001b[1;32m    368\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    369\u001b[0m       \u001b[0msaving_listeners\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_check_listeners_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msaving_listeners\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 370\u001b[0;31m       \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msaving_listeners\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    371\u001b[0m       \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Loss for final step: %s.'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    372\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/estimator.py\u001b[0m in \u001b[0;36m_train_model\u001b[0;34m(self, input_fn, hooks, saving_listeners)\u001b[0m\n\u001b[1;32m   1158\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_model_distributed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msaving_listeners\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1159\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1160\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_model_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msaving_listeners\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1162\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_train_model_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msaving_listeners\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/estimator.py\u001b[0m in \u001b[0;36m_train_model_default\u001b[0;34m(self, input_fn, hooks, saving_listeners)\u001b[0m\n\u001b[1;32m   1188\u001b[0m       \u001b[0mworker_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_hooks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1189\u001b[0m       estimator_spec = self._call_model_fn(\n\u001b[0;32m-> 1190\u001b[0;31m           features, labels, ModeKeys.TRAIN, self.config)\n\u001b[0m\u001b[1;32m   1191\u001b[0m       \u001b[0mglobal_step_tensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtraining_util\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_global_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1192\u001b[0m       return self._train_with_estimator_spec(estimator_spec, worker_hooks,\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/estimator.py\u001b[0m in \u001b[0;36m_call_model_fn\u001b[0;34m(self, features, labels, mode, config)\u001b[0m\n\u001b[1;32m   1146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1147\u001b[0m     \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Calling model_fn.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1148\u001b[0;31m     \u001b[0mmodel_fn_results\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1149\u001b[0m     \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Done calling model_fn.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1150\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/keras.py\u001b[0m in \u001b[0;36mmodel_fn\u001b[0;34m(features, labels, mode)\u001b[0m\n\u001b[1;32m    286\u001b[0m         \u001b[0mfeatures\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    287\u001b[0m         \u001b[0mlabels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 288\u001b[0;31m         optimizer_config=optimizer_config)\n\u001b[0m\u001b[1;32m    289\u001b[0m     \u001b[0mmodel_output_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    290\u001b[0m     \u001b[0;31m# We need to make sure that the output names of the last layer in the model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/keras.py\u001b[0m in \u001b[0;36m_clone_and_build_model\u001b[0;34m(mode, keras_model, custom_objects, features, labels, optimizer_config)\u001b[0m\n\u001b[1;32m    209\u001b[0m   \u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_learning_phase\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m   input_tensors, target_tensors, sample_weight_tensors = (\n\u001b[0;32m--> 211\u001b[0;31m       _convert_estimator_io_to_keras(keras_model, features, labels))\n\u001b[0m\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m   \u001b[0mcompile_clone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPREDICT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/keras.py\u001b[0m in \u001b[0;36m_convert_estimator_io_to_keras\u001b[0;34m(keras_model, features, labels)\u001b[0m\n\u001b[1;32m    165\u001b[0m     \u001b[0;31m# converting input tensors into sorted list.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    166\u001b[0m     input_tensors = _to_ordered_tensor_list(features, input_names, 'features',\n\u001b[0;32m--> 167\u001b[0;31m                                             'inputs')\n\u001b[0m\u001b[1;32m    168\u001b[0m   target_tensors = _to_ordered_tensor_list(\n\u001b[1;32m    169\u001b[0m       labels, output_names, 'labels', 'outputs')\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/keras.py\u001b[0m in \u001b[0;36m_to_ordered_tensor_list\u001b[0;34m(obj, key_order, obj_name, order_name)\u001b[0m\n\u001b[1;32m    138\u001b[0m                 \u001b[0morder_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder_keys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey_order\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m                 \u001b[0mobj_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj_keys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 140\u001b[0;31m                 different_keys=different_keys))\n\u001b[0m\u001b[1;32m    141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    142\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_convert_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkey_order\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyError\u001b[0m: \"The dictionary passed into features does not have the expected inputs keys defined in the keras model.\\n\\tExpected keys: {'input_1', 'input_2'}\\n\\tfeatures keys: {'store', 'loc'}\\n\\tDifference: {'input_1', 'input_2', 'loc', 'store'}\""
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QYAwDwW3-38j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}
	{
	"nbformat": 4,
	"nbformat_minor": 0,
	"metadata": {
	"colab": {
	"name": "keras_estimator_feature_column_bug.ipynb",
	"provenance": [],
	"include_colab_link": true
	},
	"kernelspec": {
	"name": "python3",
	"display_name": "Python 3"
	},
	"accelerator": "GPU"
	},
	"cells": [
	{
	"cell_type": "markdown",
	"metadata": {
	"id": "view-in-github",
	"colab_type": "text"
	},
	"source": [
	"<a href=\"https://colab.research.google.com/gist/hsm207/3ab2a5b31bc3b21f8466d426abaa8b81/keras_estimator_feature_column_bug.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
	]
	},
	{
	"cell_type": "code",
	"metadata": {
	"id": "anBNYmvyK4mv",
	"colab_type": "code",
	"colab": {
	"base_uri": "https://localhost:8080/",
	"height": 34
	},
	"outputId": "d53345f2-8f18-4592-e77f-e2728e49aaf8"
	},
	"source": [
	"%tensorflow_version 2.x"
	],
	"execution_count": 1,
	"outputs": [
	{
	"output_type": "stream",
	"text": [
	"TensorFlow 2.x selected.\n"
	],
	"name": "stdout"
	}
	]
	},
	{
	"cell_type": "code",
	"metadata": {
	"id": "plFd4bpO_ehj",
	"colab_type": "code",
	"outputId": "ca750f0f-7946-4d92-f4d1-a9767cb91e61",
	"colab": {
	"base_uri": "https://localhost:8080/",
	"height": 34
	}
	},
	"source": [
	"!python -c \"import tensorflow as tf; print(tf.version.GIT_VERSION, tf.version.VERSION)\""
	],
	"execution_count": 2,
	"outputs": [
	{
	"output_type": "stream",
	"text": [
	"v2.0.0-rc2-26-g64c3d38 2.0.0\n"
	],
	"name": "stdout"
	}
	]
	},
	{
	"cell_type": "code",
	"metadata": {
	"id": "z18OovDp_0Wk",
	"colab_type": "code",
	"outputId": "3168e8c0-753a-4b0f-b997-e248d73fb828",
	"colab": {
	"base_uri": "https://localhost:8080/",
	"height": 51
	}
	},
	"source": [
	"!python -c \"import sys; print(sys.version)\""
	],
	"execution_count": 3,
	"outputs": [
	{
	"output_type": "stream",
	"text": [
	"3.6.8 (default, Oct 7 2019, 12:59:55) \n",
	"[GCC 8.3.0]\n"
	],
	"name": "stdout"
	}
	]
	},
	{
	"cell_type": "code",
	"metadata": {
	"id": "RLy8JGb28_H6",
	"colab_type": "code",
	"colab": {}
	},
	"source": [
	"import tensorflow as tf\n",
	"import pandas as pd\n",
	"\n",
	"\n",
	"def create_model():\n",
	" store_feature = tf.feature_column.categorical_column_with_vocabulary_list('store',\n",
	" vocabulary_list=['a', 'b'])\n",
	"\n",
	" store_feature = tf.feature_column.embedding_column(store_feature, dimension=64)\n",
	"\n",
	" loc_feature = tf.feature_column.categorical_column_with_vocabulary_list('loc',\n",
	" vocabulary_list=['x', 'y', 'z'])\n",
	"\n",
	" loc_feature = tf.feature_column.embedding_column(loc_feature, dimension=32)\n",
	"\n",
	" model = tf.keras.Sequential([\n",
	" tf.keras.layers.DenseFeatures(feature_columns=[store_feature, loc_feature]),\n",
	" tf.keras.layers.Dense(32, activation='relu'),\n",
	" tf.keras.layers.Dense(1, activation='relu')\n",
	" ])\n",
	"\n",
	" return model\n",
	"\n",
	"df = pd.DataFrame({'store': ['a', 'b', 'b', 'a'],\n",
	" 'loc': ['x', 'y', 'z', 'x'],\n",
	" 'label': [100., 200., 400., 100.]})\n",
	"\n",
	"\n",
	"labels = df.pop('label')\n",
	"\n",
	"train_ds = tf.data.Dataset.from_tensor_slices((dict(df), labels)).batch(2)"
	],
	"execution_count": 0,
	"outputs": []
	},
	{
	"cell_type": "code",
	"metadata": {
	"id": "Cw2ULTET9dHk",
	"colab_type": "code",
	"colab": {}
	},
	"source": [
	"model_1 = create_model()\n",
	"model_1.compile(optimizer='adam', loss='mse', metrics=['mse'])\n",
	"\n",
	"# # call fit to build model because model_1.build() won't work\n",
	"# # also to double check that training works on train_ds\n",
	"# model_1.fit(train_ds)\n",
	"\n",
	"# print('The keras model\"s input names are: {}'.format(model_1.input_names))"
	],
	"execution_count": 0,
	"outputs": []
	},
	{
	"cell_type": "code",
	"metadata": {
	"id": "U6274TZO9fqg",
	"colab_type": "code",
	"outputId": "bf4b3455-f044-4a37-abc2-c6c3379e33e3",
	"colab": {
	"base_uri": "https://localhost:8080/",
	"height": 700
	}
	},
	"source": [
	"est_1 = tf.keras.estimator.model_to_estimator(keras_model=model_1)\n",
	"\n",
	"\n",
	"def input_fn():\n",
	" ds = tf.data.Dataset.from_tensor_slices((dict(df), labels)).batch(2)\n",
	" return ds.repeat()\n",
	"\n",
	"train_spec = tf.estimator.TrainSpec(input_fn = input_fn, max_steps=4 * 3)\n",
	"eval_spec = tf.estimator.EvalSpec(input_fn=input_fn,\n",
	" steps=12)\n",
	"\n",
	"tf.estimator.train_and_evaluate(est_1, train_spec, eval_spec)"
	],
	"execution_count": 6,
	"outputs": [
	{
	"output_type": "stream",
	"text": [
	"INFO:tensorflow:Using default config.\n",
	"WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpbdefa2vg\n",
	"INFO:tensorflow:Using the Keras model provided.\n",
	"INFO:tensorflow:Using config: {'_model_dir': '/tmp/tmpbdefa2vg', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n",
	"graph_options {\n",
	" rewrite_options {\n",
	" meta_optimizer_iterations: ONE\n",
	" }\n",
	"}\n",
	", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7febe034a320>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
	"INFO:tensorflow:Not using Distribute Coordinator.\n",
	"INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
	"INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
	"WARNING:tensorflow:From /tensorflow-2.0.0/python3.6/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
	"Instructions for updating:\n",
	"If using Keras pass *_constraint arguments to layers.\n",
	"WARNING:tensorflow:From /tensorflow-2.0.0/python3.6/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
	"Instructions for updating:\n",
	"Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n",
	"INFO:tensorflow:Calling model_fn.\n"
	],
	"name": "stdout"
	},
	{
	"output_type": "error",
	"ename": "KeyError",
	"evalue": "ignored",
	"traceback": [
	"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
	"\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
	"\u001b[0;32m<ipython-input-6-d191f2777eb2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 10\u001b[0m steps=12)\n\u001b[1;32m 11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_and_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mest_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_spec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meval_spec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
	"\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/training.py\u001b[0m in \u001b[0;36mtrain_and_evaluate\u001b[0;34m(estimator, train_spec, eval_spec)\u001b[0m\n\u001b[1;32m 471\u001b[0m '(with task id 0). Given task id {}'.format(config.task_id))\n\u001b[1;32m 472\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 473\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mexecutor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 474\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 475\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
	"\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/training.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 611\u001b[0m config.task_type != run_config_lib.TaskType.EVALUATOR):\n\u001b[1;32m 612\u001b[0m \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Running training and evaluation locally (non-distributed).'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 613\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_local\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 614\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 615\u001b[0m \u001b[0;31m# Distributed case.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
	"\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/training.py\u001b[0m in \u001b[0;36mrun_local\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 712\u001b[0m \u001b[0mmax_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_spec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 713\u001b[0m \u001b[0mhooks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain_hooks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 714\u001b[0;31m saving_listeners=saving_listeners)\n\u001b[0m\u001b[1;32m 715\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 716\u001b[0m eval_result = listener_for_eval.eval_result or _EvalResult(\n",
	"\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/estimator.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, input_fn, hooks, steps, max_steps, saving_listeners)\u001b[0m\n\u001b[1;32m 368\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 369\u001b[0m \u001b[0msaving_listeners\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_check_listeners_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msaving_listeners\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 370\u001b[0;31m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msaving_listeners\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 371\u001b[0m \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Loss for final step: %s.'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 372\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
	"\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/estimator.py\u001b[0m in \u001b[0;36m_train_model\u001b[0;34m(self, input_fn, hooks, saving_listeners)\u001b[0m\n\u001b[1;32m 1158\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_model_distributed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msaving_listeners\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1159\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1160\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_model_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msaving_listeners\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1162\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_train_model_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msaving_listeners\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
	"\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/estimator.py\u001b[0m in \u001b[0;36m_train_model_default\u001b[0;34m(self, input_fn, hooks, saving_listeners)\u001b[0m\n\u001b[1;32m 1188\u001b[0m \u001b[0mworker_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_hooks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1189\u001b[0m estimator_spec = self._call_model_fn(\n\u001b[0;32m-> 1190\u001b[0;31m features, labels, ModeKeys.TRAIN, self.config)\n\u001b[0m\u001b[1;32m 1191\u001b[0m \u001b[0mglobal_step_tensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtraining_util\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_global_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1192\u001b[0m return self._train_with_estimator_spec(estimator_spec, worker_hooks,\n",
	"\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/estimator.py\u001b[0m in \u001b[0;36m_call_model_fn\u001b[0;34m(self, features, labels, mode, config)\u001b[0m\n\u001b[1;32m 1146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1147\u001b[0m \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Calling model_fn.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1148\u001b[0;31m \u001b[0mmodel_fn_results\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1149\u001b[0m \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Done calling model_fn.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1150\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
	"\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/keras.py\u001b[0m in \u001b[0;36mmodel_fn\u001b[0;34m(features, labels, mode)\u001b[0m\n\u001b[1;32m 286\u001b[0m \u001b[0mfeatures\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 287\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 288\u001b[0;31m optimizer_config=optimizer_config)\n\u001b[0m\u001b[1;32m 289\u001b[0m \u001b[0mmodel_output_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 290\u001b[0m \u001b[0;31m# We need to make sure that the output names of the last layer in the model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
	"\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/keras.py\u001b[0m in \u001b[0;36m_clone_and_build_model\u001b[0;34m(mode, keras_model, custom_objects, features, labels, optimizer_config)\u001b[0m\n\u001b[1;32m 209\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_learning_phase\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 210\u001b[0m input_tensors, target_tensors, sample_weight_tensors = (\n\u001b[0;32m--> 211\u001b[0;31m _convert_estimator_io_to_keras(keras_model, features, labels))\n\u001b[0m\u001b[1;32m 212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 213\u001b[0m \u001b[0mcompile_clone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPREDICT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
	"\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/keras.py\u001b[0m in \u001b[0;36m_convert_estimator_io_to_keras\u001b[0;34m(keras_model, features, labels)\u001b[0m\n\u001b[1;32m 165\u001b[0m \u001b[0;31m# converting input tensors into sorted list.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 166\u001b[0m input_tensors = _to_ordered_tensor_list(features, input_names, 'features',\n\u001b[0;32m--> 167\u001b[0;31m 'inputs')\n\u001b[0m\u001b[1;32m 168\u001b[0m target_tensors = _to_ordered_tensor_list(\n\u001b[1;32m 169\u001b[0m labels, output_names, 'labels', 'outputs')\n",
	"\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_estimator/python/estimator/keras.py\u001b[0m in \u001b[0;36m_to_ordered_tensor_list\u001b[0;34m(obj, key_order, obj_name, order_name)\u001b[0m\n\u001b[1;32m 138\u001b[0m \u001b[0morder_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder_keys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey_order\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 139\u001b[0m \u001b[0mobj_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj_keys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 140\u001b[0;31m different_keys=different_keys))\n\u001b[0m\u001b[1;32m 141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 142\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_convert_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkey_order\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
	"\u001b[0;31mKeyError\u001b[0m: \"The dictionary passed into features does not have the expected inputs keys defined in the keras model.\\n\\tExpected keys: {'input_1', 'input_2'}\\n\\tfeatures keys: {'store', 'loc'}\\n\\tDifference: {'input_1', 'input_2', 'loc', 'store'}\""
	]
	}
	]
	},
	{
	"cell_type": "code",
	"metadata": {
	"id": "QYAwDwW3-38j",
	"colab_type": "code",
	"colab": {}
	},
	"source": [
	""
	],
	"execution_count": 0,
	"outputs": []
	}
	]
	}