TensorFlow Keras model save bug

keras-save-bug.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.13.1'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyModel(tf.keras.models.Model):\n",
    "    def __init__(self):\n",
    "        super(MyModel, self).__init__()\n",
    "        self.layer = tf.keras.layers.Dense(5)\n",
    "        \n",
    "    def __call__(self, x):\n",
    "        return self.layer(x)\n",
    "    \n",
    "model = MyModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'dense/BiasAdd:0' shape=(3, 5) dtype=float32>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Build the model\n",
    "model(tf.zeros([3, 10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'dense/kernel:0' shape=(10, 5) dtype=float32>,\n",
       " <tf.Variable 'dense/bias:0' shape=(5,) dtype=float32>]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# okay, we have two variables.\n",
    "model.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this is something like a train_op that \"changes\" the content of the variable.\n",
    "assign_op = model.variables[1].assign( tf.ones_like(model.variables[1]) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NOTE: the following line was not executed, but it can run (wait, without any session ?!?!?)\n",
    "# model.save_weights('/tmp/keras.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    }
   ],
   "source": [
    "# Let's create a session\n",
    "sess = tf.train.SingularMonitoredSession()\n",
    "#sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 0.], dtype=float32)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we can see the 'bias' variable is initialized to ZERO\n",
    "sess.run(model.variables[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 1., 1., 1., 1.], dtype=float32)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now let's make the 'bias' variable to all one...\n",
    "sess.run(assign_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 1., 1., 1., 1.], dtype=float32)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sure it is ONE ...\n",
    "sess.run(model.variables[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's save the Keras model parameter. The bias is set to ONE, right ?????\n",
    "#\n",
    "# Since model.save_weights try to create a new op (another bug #26430)\n",
    "# and the graph has been finalized, we will 'unfinalize' the graph with a bit of hack\n",
    "sess.graph._finalized = False\n",
    "model.save_weights('/tmp/keras-one.h5')\n",
    "sess.graph.finalize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NOTE: if we have had the cell [5] executed (e.g. model.save_weights BEFORE creating the session),\n",
    "# we would not need to unfinalize the graph as above because the IsInitializedOp would have been created"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5py\n",
    "#h = h5py.File(\"/tmp/keras.h5\")\n",
    "#h['dense']['dense']['bias:0'].value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 0.], dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's see what is stored in the model file ....\n",
    "\n",
    "h = h5py.File(\"/tmp/keras-one.h5\")\n",
    "h['dense']['dense']['bias:0'].value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## !?!??!?!?!???! it's ZERO!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

tensorflow/tensorflow#26472