Last active
January 3, 2020 23:24
-
-
Save zyenge/2595f3369e7e6128dcc79b1a30c3e3cd to your computer and use it in GitHub Desktop.
testing tf-trt conversion with official sample code
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "- The notebook is run inside offical nvidia container: nvcr.io/nvidia/tensorflow:19.10-py3 \n", | |
| "- python version: 3.6\n", | |
| "- tensorflow version: 1.14.0\n", | |
| "- cuda 10" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 2, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "\u001b[33mWARNING: You are using pip version 19.2.3, however version 19.3.1 is available.\n", | |
| "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n", | |
| "\u001b[33mWARNING: You are using pip version 19.2.3, however version 19.3.1 is available.\n", | |
| "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "!pip install -q git+https://github.com/tensorflow/docs\n", | |
| "!pip install -q seaborn" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 13, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "1.14.0\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "from __future__ import absolute_import, division, print_function, unicode_literals\n", | |
| "import tensorflow as tf\n", | |
| "import tensorflow.keras.backend as K\n", | |
| "import os\n", | |
| "import pathlib\n", | |
| "import matplotlib.pyplot as plt\n", | |
| "import numpy as np\n", | |
| "import pandas as pd\n", | |
| "# import seaborn as sns\n", | |
| "import tensorflow_docs as tfdocs\n", | |
| "import tensorflow_docs.plots\n", | |
| "import tensorflow_docs.modeling\n", | |
| "from tensorflow.python.compiler.tensorrt import trt_convert as trt\n", | |
| "from tensorflow import keras\n", | |
| "from tensorflow.keras import layers\n", | |
| "print(tf.__version__)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "### Show that there are GPUs" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 3, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'),\n", | |
| " PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU'),\n", | |
| " PhysicalDevice(name='/physical_device:GPU:2', device_type='GPU')]" | |
| ] | |
| }, | |
| "execution_count": 3, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "tf.config.experimental.list_physical_devices('GPU')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "### Sample data and model are copied from tensorflow doc: https://www.tensorflow.org/tutorials/keras/regression" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 6, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Downloading data from http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\n", | |
| "32768/30286 [================================] - 0s 2us/step\n" | |
| ] | |
| }, | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<div>\n", | |
| "<style scoped>\n", | |
| " .dataframe tbody tr th:only-of-type {\n", | |
| " vertical-align: middle;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "</style>\n", | |
| "<table border=\"1\" class=\"dataframe\">\n", | |
| " <thead>\n", | |
| " <tr style=\"text-align: right;\">\n", | |
| " <th></th>\n", | |
| " <th>count</th>\n", | |
| " <th>mean</th>\n", | |
| " <th>std</th>\n", | |
| " <th>min</th>\n", | |
| " <th>25%</th>\n", | |
| " <th>50%</th>\n", | |
| " <th>75%</th>\n", | |
| " <th>max</th>\n", | |
| " </tr>\n", | |
| " </thead>\n", | |
| " <tbody>\n", | |
| " <tr>\n", | |
| " <th>Cylinders</th>\n", | |
| " <td>314.0</td>\n", | |
| " <td>5.477707</td>\n", | |
| " <td>1.699788</td>\n", | |
| " <td>3.0</td>\n", | |
| " <td>4.00</td>\n", | |
| " <td>4.0</td>\n", | |
| " <td>8.00</td>\n", | |
| " <td>8.0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>Displacement</th>\n", | |
| " <td>314.0</td>\n", | |
| " <td>195.318471</td>\n", | |
| " <td>104.331589</td>\n", | |
| " <td>68.0</td>\n", | |
| " <td>105.50</td>\n", | |
| " <td>151.0</td>\n", | |
| " <td>265.75</td>\n", | |
| " <td>455.0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>Horsepower</th>\n", | |
| " <td>314.0</td>\n", | |
| " <td>104.869427</td>\n", | |
| " <td>38.096214</td>\n", | |
| " <td>46.0</td>\n", | |
| " <td>76.25</td>\n", | |
| " <td>94.5</td>\n", | |
| " <td>128.00</td>\n", | |
| " <td>225.0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>Weight</th>\n", | |
| " <td>314.0</td>\n", | |
| " <td>2990.251592</td>\n", | |
| " <td>843.898596</td>\n", | |
| " <td>1649.0</td>\n", | |
| " <td>2256.50</td>\n", | |
| " <td>2822.5</td>\n", | |
| " <td>3608.00</td>\n", | |
| " <td>5140.0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>Acceleration</th>\n", | |
| " <td>314.0</td>\n", | |
| " <td>15.559236</td>\n", | |
| " <td>2.789230</td>\n", | |
| " <td>8.0</td>\n", | |
| " <td>13.80</td>\n", | |
| " <td>15.5</td>\n", | |
| " <td>17.20</td>\n", | |
| " <td>24.8</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>Model Year</th>\n", | |
| " <td>314.0</td>\n", | |
| " <td>75.898089</td>\n", | |
| " <td>3.675642</td>\n", | |
| " <td>70.0</td>\n", | |
| " <td>73.00</td>\n", | |
| " <td>76.0</td>\n", | |
| " <td>79.00</td>\n", | |
| " <td>82.0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>Europe</th>\n", | |
| " <td>314.0</td>\n", | |
| " <td>0.178344</td>\n", | |
| " <td>0.383413</td>\n", | |
| " <td>0.0</td>\n", | |
| " <td>0.00</td>\n", | |
| " <td>0.0</td>\n", | |
| " <td>0.00</td>\n", | |
| " <td>1.0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>Japan</th>\n", | |
| " <td>314.0</td>\n", | |
| " <td>0.197452</td>\n", | |
| " <td>0.398712</td>\n", | |
| " <td>0.0</td>\n", | |
| " <td>0.00</td>\n", | |
| " <td>0.0</td>\n", | |
| " <td>0.00</td>\n", | |
| " <td>1.0</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>USA</th>\n", | |
| " <td>314.0</td>\n", | |
| " <td>0.624204</td>\n", | |
| " <td>0.485101</td>\n", | |
| " <td>0.0</td>\n", | |
| " <td>0.00</td>\n", | |
| " <td>1.0</td>\n", | |
| " <td>1.00</td>\n", | |
| " <td>1.0</td>\n", | |
| " </tr>\n", | |
| " </tbody>\n", | |
| "</table>\n", | |
| "</div>" | |
| ], | |
| "text/plain": [ | |
| " count mean std min 25% 50% \\\n", | |
| "Cylinders 314.0 5.477707 1.699788 3.0 4.00 4.0 \n", | |
| "Displacement 314.0 195.318471 104.331589 68.0 105.50 151.0 \n", | |
| "Horsepower 314.0 104.869427 38.096214 46.0 76.25 94.5 \n", | |
| "Weight 314.0 2990.251592 843.898596 1649.0 2256.50 2822.5 \n", | |
| "Acceleration 314.0 15.559236 2.789230 8.0 13.80 15.5 \n", | |
| "Model Year 314.0 75.898089 3.675642 70.0 73.00 76.0 \n", | |
| "Europe 314.0 0.178344 0.383413 0.0 0.00 0.0 \n", | |
| "Japan 314.0 0.197452 0.398712 0.0 0.00 0.0 \n", | |
| "USA 314.0 0.624204 0.485101 0.0 0.00 1.0 \n", | |
| "\n", | |
| " 75% max \n", | |
| "Cylinders 8.00 8.0 \n", | |
| "Displacement 265.75 455.0 \n", | |
| "Horsepower 128.00 225.0 \n", | |
| "Weight 3608.00 5140.0 \n", | |
| "Acceleration 17.20 24.8 \n", | |
| "Model Year 79.00 82.0 \n", | |
| "Europe 0.00 1.0 \n", | |
| "Japan 0.00 1.0 \n", | |
| "USA 1.00 1.0 " | |
| ] | |
| }, | |
| "execution_count": 6, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", | |
| "dataset_path\n", | |
| "column_names = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n", | |
| " 'Acceleration', 'Model Year', 'Origin']\n", | |
| "raw_dataset = pd.read_csv(dataset_path, names=column_names,\n", | |
| " na_values = \"?\", comment='\\t',\n", | |
| " sep=\" \", skipinitialspace=True)\n", | |
| "dataset = raw_dataset.copy()\n", | |
| "dataset = dataset.dropna()\n", | |
| "dataset['Origin'] = dataset['Origin'].map(lambda x: {1: 'USA', 2: 'Europe', 3: 'Japan'}.get(x))\n", | |
| "\n", | |
| "dataset = pd.get_dummies(dataset, prefix='', prefix_sep='')\n", | |
| "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", | |
| "test_dataset = dataset.drop(train_dataset.index)\n", | |
| "train_stats = train_dataset.describe()\n", | |
| "train_stats.pop(\"MPG\")\n", | |
| "train_stats = train_stats.transpose()\n", | |
| "train_stats" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 7, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "train_labels = train_dataset.pop('MPG')\n", | |
| "test_labels = test_dataset.pop('MPG')\n", | |
| "def norm(x):\n", | |
| " return (x - train_stats['mean']) / train_stats['std']\n", | |
| "normed_train_data = norm(train_dataset)\n", | |
| "normed_test_data = norm(test_dataset)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 8, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "def build_model():\n", | |
| " model = keras.Sequential([\n", | |
| " layers.Dense(64, activation='relu', input_shape=[len(train_dataset.keys())]),\n", | |
| " layers.Dense(64, activation='relu'),\n", | |
| " layers.Dense(1)\n", | |
| " ])\n", | |
| "\n", | |
| " optimizer = tf.keras.optimizers.RMSprop(0.001)\n", | |
| "\n", | |
| " model.compile(loss='mse',\n", | |
| " optimizer=optimizer,\n", | |
| " metrics=['mae', 'mse'])\n", | |
| " return model" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 9, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", | |
| "Instructions for updating:\n", | |
| "Call initializer instance with the dtype argument instead of passing it to the constructor\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "model = build_model()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 10, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Model: \"sequential\"\n", | |
| "_________________________________________________________________\n", | |
| "Layer (type) Output Shape Param # \n", | |
| "=================================================================\n", | |
| "dense (Dense) (None, 64) 640 \n", | |
| "_________________________________________________________________\n", | |
| "dense_1 (Dense) (None, 64) 4160 \n", | |
| "_________________________________________________________________\n", | |
| "dense_2 (Dense) (None, 1) 65 \n", | |
| "=================================================================\n", | |
| "Total params: 4,865\n", | |
| "Trainable params: 4,865\n", | |
| "Non-trainable params: 0\n", | |
| "_________________________________________________________________\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "model.summary()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "### show that model can run " | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 14, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "array([[-0.06201069],\n", | |
| " [ 0.30905545],\n", | |
| " [ 0.25227392],\n", | |
| " [ 0.24466842],\n", | |
| " [ 0.21558633],\n", | |
| " [ 0.17524138],\n", | |
| " [ 0.19101134],\n", | |
| " [-0.00892259],\n", | |
| " [ 0.08648324],\n", | |
| " [ 0.40585902]], dtype=float32)" | |
| ] | |
| }, | |
| "execution_count": 14, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "example_batch = normed_train_data[:10]\n", | |
| "example_result = model.predict(example_batch)\n", | |
| "example_result" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "### Export model" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "#### 1-a.try frozen model save" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "frozen_path = \"/root/workspace/tftrt_test/frozen-auto-mpg-test.pb\"\n", | |
| "\n", | |
| "output_graph_def = tf.compat.v1.graph_util.convert_variables_to_constants(\n", | |
| " K.get_session(), K.get_session().graph.as_graph_def(), [\"dense_2/BiasAdd\"])\n", | |
| "\n", | |
| "with tf.gfile.FastGFile(frozen_path, 'wb') as f:\n", | |
| " f.write(output_graph_def.SerializeToString())\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "#### 1-b.try frozen model load\n", | |
| "Sample code from https://docs.nvidia.com/deeplearning/frameworks/tf-trt-user-guide/index.html#using-frozengraph" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 14, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "INFO:tensorflow:Linked TensorRT version: (6, 0, 1)\n", | |
| "INFO:tensorflow:Loaded TensorRT version: (6, 0, 1)\n", | |
| "INFO:tensorflow:Running against TensorRT version 6.0.1\n" | |
| ] | |
| }, | |
| { | |
| "ename": "InvalidArgumentError", | |
| "evalue": "Failed to import metagraph, check error log for more info.", | |
| "output_type": "error", | |
| "traceback": [ | |
| "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", | |
| "\u001b[0;31mInvalidArgumentError\u001b[0m Traceback (most recent call last)", | |
| "\u001b[0;32m<ipython-input-14-c26f6f7bc759>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;31m# nodes_blacklist=['logits', 'classes']) #output nodes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m )\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0mtrt_graph\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconverter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", | |
| "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/compiler/tensorrt/trt_convert.py\u001b[0m in \u001b[0;36mconvert\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 296\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_converted\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 297\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_input_graph_def\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 298\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_convert_graph_def\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 299\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 300\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_convert_saved_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | |
| "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/compiler/tensorrt/trt_convert.py\u001b[0m in \u001b[0;36m_convert_graph_def\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 224\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_add_nodes_blacklist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 225\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 226\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_conversion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 227\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 228\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_collections_to_keep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollection_keys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | |
| "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/compiler/tensorrt/trt_convert.py\u001b[0m in \u001b[0;36m_run_conversion\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 202\u001b[0m \u001b[0mgrappler_session_config\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 203\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_grappler_meta_graph_def\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 204\u001b[0;31m graph_id=b\"tf_graph\")\n\u001b[0m\u001b[1;32m 205\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_converted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 206\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", | |
| "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/grappler/tf_optimizer.py\u001b[0m in \u001b[0;36mOptimizeGraph\u001b[0;34m(config_proto, metagraph, verbose, graph_id, cluster)\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[0mconfig_proto\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSerializeToString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 40\u001b[0m \u001b[0mmetagraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSerializeToString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m verbose, graph_id)\n\u001b[0m\u001b[1;32m 42\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mret_from_swig\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | |
| "\u001b[0;31mInvalidArgumentError\u001b[0m: Failed to import metagraph, check error log for more info." | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "with tf.device('/GPU:2'):\n", | |
| " with tf.gfile.GFile(frozen_path, 'rb') as f:\n", | |
| " frozen_graph = tf.GraphDef()\n", | |
| " frozen_graph.ParseFromString(f.read())\n", | |
| " # Now you can create a TensorRT inference graph from your\n", | |
| " # frozen graph:\n", | |
| " converter = trt.TrtGraphConverter(\n", | |
| " input_graph_def=frozen_graph\n", | |
| "# nodes_blacklist=['logits', 'classes']) #output nodes\n", | |
| " )\n", | |
| " trt_graph = converter.convert()\n", | |
| " " | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "#### 2-a.try save model " | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 20, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "INFO:tensorflow:No assets to save.\n", | |
| "INFO:tensorflow:No assets to write.\n", | |
| "INFO:tensorflow:SavedModel written to: /root/workspace/tftrt_test/saved_model_dir/saved_model.pb\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "save_model_dir = \"/root/workspace/tftrt_test/saved_model_dir\"\n", | |
| "with tf.device('/GPU:2'):\n", | |
| " builder = tf.compat.v1.saved_model.builder.SavedModelBuilder(save_model_dir)\n", | |
| " builder.add_meta_graph_and_variables(sess=K.get_session(),\n", | |
| " tags=[tf.saved_model.tag_constants.SERVING],\n", | |
| " signature_def_map=None)\n", | |
| " builder.save()\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 22, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "saved_model.pb\tvariables\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "!ls /root/workspace/tftrt_test/saved_model_dir" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "#### 2-b try load SavedModel\n", | |
| "sample code from https://docs.nvidia.com/deeplearning/frameworks/tf-trt-user-guide/index.html#using-savedmodel" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 21, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "INFO:tensorflow:Linked TensorRT version: (6, 0, 1)\n", | |
| "INFO:tensorflow:Loaded TensorRT version: (6, 0, 1)\n", | |
| "INFO:tensorflow:Running against TensorRT version 6.0.1\n", | |
| "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compiler/tensorrt/trt_convert.py:245: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.\n", | |
| "Instructions for updating:\n", | |
| "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.\n", | |
| "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n", | |
| "INFO:tensorflow:The specified SavedModel has no variables; no checkpoints were restored.\n", | |
| "INFO:tensorflow:Froze 0 variables.\n", | |
| "INFO:tensorflow:Converted 0 variables to const ops.\n" | |
| ] | |
| }, | |
| { | |
| "ename": "InvalidArgumentError", | |
| "evalue": "Failed to import metagraph, check error log for more info.", | |
| "output_type": "error", | |
| "traceback": [ | |
| "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", | |
| "\u001b[0;31mInvalidArgumentError\u001b[0m Traceback (most recent call last)", | |
| "\u001b[0;32m<ipython-input-21-bde49fe68811>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;31m# maximum_cached_engines=100)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m )\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mconverter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", | |
| "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/compiler/tensorrt/trt_convert.py\u001b[0m in \u001b[0;36mconvert\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 298\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_convert_graph_def\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 299\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 300\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_convert_saved_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 301\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_converted_graph_def\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 302\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", | |
| "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/compiler/tensorrt/trt_convert.py\u001b[0m in \u001b[0;36m_convert_saved_model\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 285\u001b[0m \u001b[0;31m# TODO(laigd): maybe add back AssetFileDef.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 286\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 287\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_conversion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 289\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | |
| "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/compiler/tensorrt/trt_convert.py\u001b[0m in \u001b[0;36m_run_conversion\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 202\u001b[0m \u001b[0mgrappler_session_config\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 203\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_grappler_meta_graph_def\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 204\u001b[0;31m graph_id=b\"tf_graph\")\n\u001b[0m\u001b[1;32m 205\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_converted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 206\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", | |
| "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/grappler/tf_optimizer.py\u001b[0m in \u001b[0;36mOptimizeGraph\u001b[0;34m(config_proto, metagraph, verbose, graph_id, cluster)\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[0mconfig_proto\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSerializeToString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 40\u001b[0m \u001b[0mmetagraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSerializeToString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m verbose, graph_id)\n\u001b[0m\u001b[1;32m 42\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mret_from_swig\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | |
| "\u001b[0;31mInvalidArgumentError\u001b[0m: Failed to import metagraph, check error log for more info." | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "with tf.device(\"/device:GPU:2\"):\n", | |
| " converter = trt.TrtGraphConverter(\n", | |
| " input_saved_model_dir=save_model_dir\n", | |
| " # max_workspace_size_bytes=(11<32),\n", | |
| " # precision_mode=\"FP16\",\n", | |
| " # maximum_cached_engines=100)\n", | |
| " )\n", | |
| " converter.convert()\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "Python 3", | |
| "language": "python", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.6.8" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 4 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment