Skip to content

Instantly share code, notes, and snippets.

@hampen2929
Created November 24, 2018 16:19
Show Gist options
  • Save hampen2929/8f6e877e7aa3e199a0c7520f5371323b to your computer and use it in GitHub Desktop.
Save hampen2929/8f6e877e7aa3e199a0c7520f5371323b to your computer and use it in GitHub Desktop.
semantic segmentaion
Display the source blob
Display the rendered blob
Raw
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# インポート"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"cellView": "code",
"colab": {
"autoexec": {
"startup": false,
"wait_interval": 0
}
},
"colab_type": "code",
"id": "kAbdmRmvq0Je"
},
"outputs": [],
"source": [
"#@title Imports\n",
"\n",
"import os\n",
"from io import BytesIO\n",
"import tarfile\n",
"import tempfile\n",
"from six.moves import urllib\n",
"\n",
"from matplotlib import gridspec\n",
"from matplotlib import pyplot as plt\n",
"import numpy as np\n",
"from PIL import Image\n",
"\n",
"import tensorflow as tf\n",
"\n",
"import cv2"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 関数"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"cellView": "code",
"colab": {
"autoexec": {
"startup": false,
"wait_interval": 0
}
},
"colab_type": "code",
"id": "vN0kU6NJ1Ye5"
},
"outputs": [],
"source": [
"#@title Helper methods\n",
"\n",
"\n",
"class DeepLabModel(object):\n",
" \"\"\"Class to load deeplab model and run inference.\"\"\"\n",
"\n",
" INPUT_TENSOR_NAME = 'ImageTensor:0'\n",
" OUTPUT_TENSOR_NAME = 'SemanticPredictions:0'\n",
" INPUT_SIZE = 513\n",
" FROZEN_GRAPH_NAME = 'frozen_inference_graph'\n",
"\n",
" def __init__(self, tarball_path):\n",
" \"\"\"Creates and loads pretrained deeplab model.\"\"\"\n",
" self.graph = tf.Graph()\n",
"\n",
" graph_def = None\n",
" # Extract frozen graph from tar archive.\n",
" tar_file = tarfile.open(tarball_path)\n",
" for tar_info in tar_file.getmembers():\n",
" if self.FROZEN_GRAPH_NAME in os.path.basename(tar_info.name):\n",
" file_handle = tar_file.extractfile(tar_info)\n",
" graph_def = tf.GraphDef.FromString(file_handle.read())\n",
" break\n",
"\n",
" tar_file.close()\n",
"\n",
" if graph_def is None:\n",
" raise RuntimeError('Cannot find inference graph in tar archive.')\n",
"\n",
" with self.graph.as_default():\n",
" tf.import_graph_def(graph_def, name='')\n",
"\n",
" self.sess = tf.Session(graph=self.graph)\n",
"\n",
" def run(self, image):\n",
" \"\"\"Runs inference on a single image.\n",
"\n",
" Args:\n",
" image: A PIL.Image object, raw input image.\n",
"\n",
" Returns:\n",
" resized_image: RGB image resized from original input image.\n",
" seg_map: Segmentation map of `resized_image`.\n",
" \"\"\"\n",
" width, height = image.size\n",
" resize_ratio = 1.0 * self.INPUT_SIZE / max(width, height)\n",
" target_size = (int(resize_ratio * width), int(resize_ratio * height))\n",
" resized_image = image.convert('RGB').resize(target_size, Image.ANTIALIAS)\n",
" batch_seg_map = self.sess.run(\n",
" self.OUTPUT_TENSOR_NAME,\n",
" feed_dict={self.INPUT_TENSOR_NAME: [np.asarray(resized_image)]})\n",
" seg_map = batch_seg_map[0]\n",
" return resized_image, seg_map\n",
"\n",
"\n",
"def create_pascal_label_colormap():\n",
" \"\"\"Creates a label colormap used in PASCAL VOC segmentation benchmark.\n",
"\n",
" Returns:\n",
" A Colormap for visualizing segmentation results.\n",
" \"\"\"\n",
" colormap = np.zeros((256, 3), dtype=int)\n",
" ind = np.arange(256, dtype=int)\n",
"\n",
" for shift in reversed(range(8)):\n",
" for channel in range(3):\n",
" colormap[:, channel] |= ((ind >> channel) & 1) << shift\n",
" ind >>= 3\n",
"\n",
" return colormap\n",
"\n",
"\n",
"def label_to_color_image(label):\n",
" \"\"\"Adds color defined by the dataset colormap to the label.\n",
"\n",
" Args:\n",
" label: A 2D array with integer type, storing the segmentation label.\n",
"\n",
" Returns:\n",
" result: A 2D array with floating type. The element of the array\n",
" is the color indexed by the corresponding element in the input label\n",
" to the PASCAL color map.\n",
"\n",
" Raises:\n",
" ValueError: If label is not of rank 2 or its value is larger than color\n",
" map maximum entry.\n",
" \"\"\"\n",
" if label.ndim != 2:\n",
" raise ValueError('Expect 2-D input label')\n",
"\n",
" colormap = create_pascal_label_colormap()\n",
"\n",
" if np.max(label) >= len(colormap):\n",
" raise ValueError('label value too large.')\n",
"\n",
" return colormap[label]\n",
"\n",
"\n",
"def vis_segmentation(image, seg_map):\n",
" \"\"\"Visualizes input image, segmentation map and overlay view.\"\"\"\n",
" plt.figure(figsize=(15, 5))\n",
" grid_spec = gridspec.GridSpec(1, 4, width_ratios=[6, 6, 6, 1])\n",
"\n",
" plt.subplot(grid_spec[0])\n",
" plt.imshow(image)\n",
" plt.axis('off')\n",
" plt.title('input image')\n",
"\n",
" plt.subplot(grid_spec[1])\n",
" seg_image = label_to_color_image(seg_map).astype(np.uint8)\n",
" plt.imshow(seg_image)\n",
" plt.axis('off')\n",
" plt.title('segmentation map')\n",
"\n",
" plt.subplot(grid_spec[2])\n",
" plt.imshow(image)\n",
" plt.imshow(seg_image, alpha=0.7)\n",
" plt.axis('off')\n",
" plt.title('segmentation overlay')\n",
"\n",
" unique_labels = np.unique(seg_map)\n",
" ax = plt.subplot(grid_spec[3])\n",
" plt.imshow(\n",
" FULL_COLOR_MAP[unique_labels].astype(np.uint8), interpolation='nearest')\n",
" ax.yaxis.tick_right()\n",
" plt.yticks(range(len(unique_labels)), LABEL_NAMES[unique_labels])\n",
" plt.xticks([], [])\n",
" ax.tick_params(width=0.0)\n",
" plt.grid('off')\n",
" plt.show()\n",
"\n",
"\n",
"LABEL_NAMES = np.asarray([\n",
" 'background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus',\n",
" 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike',\n",
" 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tv'\n",
"])\n",
"\n",
"FULL_LABEL_MAP = np.arange(len(LABEL_NAMES)).reshape(len(LABEL_NAMES), 1)\n",
"FULL_COLOR_MAP = label_to_color_image(FULL_LABEL_MAP)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"colab": {
"autoexec": {
"startup": false,
"wait_interval": 0
}
},
"colab_type": "code",
"id": "c4oXKmnjw6i_"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"downloading model, this might take a while...\n",
"download completed! loading DeepLab model...\n",
"model loaded successfully!\n"
]
}
],
"source": [
"#@title Select and download models {display-mode: \"form\"}\n",
"\n",
"MODEL_NAME = 'mobilenetv2_coco_voctrainaug' # @param ['mobilenetv2_coco_voctrainaug', 'mobilenetv2_coco_voctrainval', 'xception_coco_voctrainaug', 'xception_coco_voctrainval']\n",
"\n",
"_DOWNLOAD_URL_PREFIX = 'http://download.tensorflow.org/models/'\n",
"_MODEL_URLS = {\n",
" 'mobilenetv2_coco_voctrainaug':\n",
" 'deeplabv3_mnv2_pascal_train_aug_2018_01_29.tar.gz',\n",
" 'mobilenetv2_coco_voctrainval':\n",
" 'deeplabv3_mnv2_pascal_trainval_2018_01_29.tar.gz',\n",
" 'xception_coco_voctrainaug':\n",
" 'deeplabv3_pascal_train_aug_2018_01_04.tar.gz',\n",
" 'xception_coco_voctrainval':\n",
" 'deeplabv3_pascal_trainval_2018_01_04.tar.gz',\n",
"}\n",
"_TARBALL_NAME = 'deeplab_model.tar.gz'\n",
"\n",
"model_dir = tempfile.mkdtemp()\n",
"tf.gfile.MakeDirs(model_dir)\n",
"\n",
"download_path = os.path.join(model_dir, _TARBALL_NAME)\n",
"print('downloading model, this might take a while...')\n",
"urllib.request.urlretrieve(_DOWNLOAD_URL_PREFIX + _MODEL_URLS[MODEL_NAME],\n",
" download_path)\n",
"print('download completed! loading DeepLab model...')\n",
"\n",
"MODEL = DeepLabModel(download_path)\n",
"print('model loaded successfully!')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 画像生成"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"path = './sample_image/'\n",
"file_list = os.listdir(path)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"for file in file_list:\n",
" \n",
" original_im = cv2.imread(path + file)\n",
" pilImg = Image.fromarray(np.uint8(original_im))\n",
" resized_im, seg_map = MODEL.run(pilImg)\n",
" seg_image = label_to_color_image(seg_map).astype(np.uint8)\n",
" \n",
" plt.imshow(resized_im)\n",
" plt.imshow(seg_image, alpha=0.7)\n",
" plt.tick_params(labelbottom='off', bottom='off')\n",
" plt.tick_params(labelleft='off', left='off')\n",
" plt.savefig('image_overlay/img_ol_' + file, bbox_inches='tight',pad_inches=-0.1)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 動画生成"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"movie_name = 'tennis'\n",
"movie = movie_name+'.mp4'"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"files_name = os.listdir('image_overlay')\n",
"files_name = files_name[1:len(files_name)]\n",
"files_name = sorted(files_name)\n",
"\n",
"fps = 30\n",
"height = 180\n",
"width = 327\n",
"\n",
"fourcc = cv2.VideoWriter_fourcc('m','p','4','v')\n",
"\n",
"video = cv2.VideoWriter(movie, fourcc, fps, (int(width), int(height)))"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"files_dir = 'image_overlay'\n",
"\n",
"files_name = os.listdir(files_dir)\n",
"files_name = sorted(files_name)\n",
"\n",
"for i in range(1, len(files_name)-1):\n",
" img = cv2.imread(files_dir +'/'+files_name[i])\n",
" img = cv2.resize(img, (int(width), int(height)))\n",
" video.write(img)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"cellView": "code",
"colab": {
"autoexec": {
"startup": false,
"wait_interval": 0
}
},
"colab_type": "code",
"id": "kAbdmRmvq0Je"
},
"outputs": [],
"source": [
"#@title Imports\n",
"\n",
"import os\n",
"from io import BytesIO\n",
"import tarfile\n",
"import tempfile\n",
"from six.moves import urllib\n",
"\n",
"from matplotlib import gridspec\n",
"from matplotlib import pyplot as plt\n",
"import numpy as np\n",
"from PIL import Image\n",
"\n",
"import tensorflow as tf\n",
"\n",
"import cv2\n",
"\n",
"# 関数\n",
"\n",
"#@title Helper methods\n",
"\n",
"\n",
"class DeepLabModel(object):\n",
" \"\"\"Class to load deeplab model and run inference.\"\"\"\n",
"\n",
" INPUT_TENSOR_NAME = 'ImageTensor:0'\n",
" OUTPUT_TENSOR_NAME = 'SemanticPredictions:0'\n",
" INPUT_SIZE = 513\n",
" FROZEN_GRAPH_NAME = 'frozen_inference_graph'\n",
"\n",
" def __init__(self, tarball_path):\n",
" \"\"\"Creates and loads pretrained deeplab model.\"\"\"\n",
" self.graph = tf.Graph()\n",
"\n",
" graph_def = None\n",
" # Extract frozen graph from tar archive.\n",
" tar_file = tarfile.open(tarball_path)\n",
" for tar_info in tar_file.getmembers():\n",
" if self.FROZEN_GRAPH_NAME in os.path.basename(tar_info.name):\n",
" file_handle = tar_file.extractfile(tar_info)\n",
" graph_def = tf.GraphDef.FromString(file_handle.read())\n",
" break\n",
"\n",
" tar_file.close()\n",
"\n",
" if graph_def is None:\n",
" raise RuntimeError('Cannot find inference graph in tar archive.')\n",
"\n",
" with self.graph.as_default():\n",
" tf.import_graph_def(graph_def, name='')\n",
"\n",
" self.sess = tf.Session(graph=self.graph)\n",
"\n",
" def run(self, image):\n",
" \"\"\"Runs inference on a single image.\n",
"\n",
" Args:\n",
" image: A PIL.Image object, raw input image.\n",
"\n",
" Returns:\n",
" resized_image: RGB image resized from original input image.\n",
" seg_map: Segmentation map of `resized_image`.\n",
" \"\"\"\n",
" width, height = image.size\n",
" resize_ratio = 1.0 * self.INPUT_SIZE / max(width, height)\n",
" target_size = (int(resize_ratio * width), int(resize_ratio * height))\n",
" resized_image = image.convert('RGB').resize(target_size, Image.ANTIALIAS)\n",
" batch_seg_map = self.sess.run(\n",
" self.OUTPUT_TENSOR_NAME,\n",
" feed_dict={self.INPUT_TENSOR_NAME: [np.asarray(resized_image)]})\n",
" seg_map = batch_seg_map[0]\n",
" return resized_image, seg_map\n",
"\n",
"\n",
"def create_pascal_label_colormap():\n",
" \"\"\"Creates a label colormap used in PASCAL VOC segmentation benchmark.\n",
"\n",
" Returns:\n",
" A Colormap for visualizing segmentation results.\n",
" \"\"\"\n",
" colormap = np.zeros((256, 3), dtype=int)\n",
" ind = np.arange(256, dtype=int)\n",
"\n",
" for shift in reversed(range(8)):\n",
" for channel in range(3):\n",
" colormap[:, channel] |= ((ind >> channel) & 1) << shift\n",
" ind >>= 3\n",
"\n",
" return colormap\n",
"\n",
"\n",
"def label_to_color_image(label):\n",
" \"\"\"Adds color defined by the dataset colormap to the label.\n",
"\n",
" Args:\n",
" label: A 2D array with integer type, storing the segmentation label.\n",
"\n",
" Returns:\n",
" result: A 2D array with floating type. The element of the array\n",
" is the color indexed by the corresponding element in the input label\n",
" to the PASCAL color map.\n",
"\n",
" Raises:\n",
" ValueError: If label is not of rank 2 or its value is larger than color\n",
" map maximum entry.\n",
" \"\"\"\n",
" if label.ndim != 2:\n",
" raise ValueError('Expect 2-D input label')\n",
"\n",
" colormap = create_pascal_label_colormap()\n",
"\n",
" if np.max(label) >= len(colormap):\n",
" raise ValueError('label value too large.')\n",
"\n",
" return colormap[label]\n",
"\n",
"\n",
"def vis_segmentation(image, seg_map):\n",
" \"\"\"Visualizes input image, segmentation map and overlay view.\"\"\"\n",
" plt.figure(figsize=(15, 5))\n",
" grid_spec = gridspec.GridSpec(1, 4, width_ratios=[6, 6, 6, 1])\n",
"\n",
" plt.subplot(grid_spec[0])\n",
" plt.imshow(image)\n",
" plt.axis('off')\n",
" plt.title('input image')\n",
"\n",
" plt.subplot(grid_spec[1])\n",
" seg_image = label_to_color_image(seg_map).astype(np.uint8)\n",
" plt.imshow(seg_image)\n",
" plt.axis('off')\n",
" plt.title('segmentation map')\n",
"\n",
" plt.subplot(grid_spec[2])\n",
" plt.imshow(image)\n",
" plt.imshow(seg_image, alpha=0.7)\n",
" plt.axis('off')\n",
" plt.title('segmentation overlay')\n",
"\n",
" unique_labels = np.unique(seg_map)\n",
" ax = plt.subplot(grid_spec[3])\n",
" plt.imshow(\n",
" FULL_COLOR_MAP[unique_labels].astype(np.uint8), interpolation='nearest')\n",
" ax.yaxis.tick_right()\n",
" plt.yticks(range(len(unique_labels)), LABEL_NAMES[unique_labels])\n",
" plt.xticks([], [])\n",
" ax.tick_params(width=0.0)\n",
" plt.grid('off')\n",
" plt.show()\n",
"\n",
"\n",
"LABEL_NAMES = np.asarray([\n",
" 'background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus',\n",
" 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike',\n",
" 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tv'\n",
"])\n",
"\n",
"FULL_LABEL_MAP = np.arange(len(LABEL_NAMES)).reshape(len(LABEL_NAMES), 1)\n",
"FULL_COLOR_MAP = label_to_color_image(FULL_LABEL_MAP)\n",
"\n",
"#@title Select and download models {display-mode: \"form\"}\n",
"\n",
"MODEL_NAME = 'mobilenetv2_coco_voctrainaug' # @param ['mobilenetv2_coco_voctrainaug', 'mobilenetv2_coco_voctrainval', 'xception_coco_voctrainaug', 'xception_coco_voctrainval']\n",
"\n",
"_DOWNLOAD_URL_PREFIX = 'http://download.tensorflow.org/models/'\n",
"_MODEL_URLS = {\n",
" 'mobilenetv2_coco_voctrainaug':\n",
" 'deeplabv3_mnv2_pascal_train_aug_2018_01_29.tar.gz',\n",
" 'mobilenetv2_coco_voctrainval':\n",
" 'deeplabv3_mnv2_pascal_trainval_2018_01_29.tar.gz',\n",
" 'xception_coco_voctrainaug':\n",
" 'deeplabv3_pascal_train_aug_2018_01_04.tar.gz',\n",
" 'xception_coco_voctrainval':\n",
" 'deeplabv3_pascal_trainval_2018_01_04.tar.gz',\n",
"}\n",
"_TARBALL_NAME = 'deeplab_model.tar.gz'\n",
"\n",
"model_dir = tempfile.mkdtemp()\n",
"tf.gfile.MakeDirs(model_dir)\n",
"\n",
"download_path = os.path.join(model_dir, _TARBALL_NAME)\n",
"print('downloading model, this might take a while...')\n",
"urllib.request.urlretrieve(_DOWNLOAD_URL_PREFIX + _MODEL_URLS[MODEL_NAME],\n",
" download_path)\n",
"print('download completed! loading DeepLab model...')\n",
"\n",
"MODEL = DeepLabModel(download_path)\n",
"print('model loaded successfully!')\n",
"\n",
"# 画像生成\n",
"\n",
"path = './sample_image/'\n",
"file_list = os.listdir(path)\n",
"\n",
"for file in file_list:\n",
" \n",
" original_im = cv2.imread(path + file)\n",
" pilImg = Image.fromarray(np.uint8(original_im))\n",
" resized_im, seg_map = MODEL.run(pilImg)\n",
" seg_image = label_to_color_image(seg_map).astype(np.uint8)\n",
" \n",
" plt.imshow(resized_im)\n",
" plt.imshow(seg_image, alpha=0.7)\n",
" plt.tick_params(labelbottom='off', bottom='off')\n",
" plt.tick_params(labelleft='off', left='off')\n",
" plt.savefig('image_overlay/img_ol_' + file, bbox_inches='tight',pad_inches=-0.1)\n",
"\n",
"# 動画生成\n",
"\n",
"movie_name = 'tennis'\n",
"movie = movie_name+'.mp4'\n",
"\n",
"files_name = os.listdir('image_overlay')\n",
"files_name = files_name[1:len(files_name)]\n",
"files_name = sorted(files_name)\n",
"\n",
"fps = 30\n",
"height = 180\n",
"width = 327\n",
"\n",
"fourcc = cv2.VideoWriter_fourcc('m','p','4','v')\n",
"\n",
"video = cv2.VideoWriter(movie, fourcc, fps, (int(width), int(height)))\n",
"\n",
"files_dir = 'image_overlay'\n",
"\n",
"files_name = os.listdir(files_dir)\n",
"files_name = sorted(files_name)\n",
"\n",
"for i in range(1, len(files_name)-1):\n",
" img = cv2.imread(files_dir +'/'+files_name[i])\n",
" img = cv2.resize(img, (int(width), int(height)))\n",
" video.write(img)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"colab": {
"collapsed_sections": [],
"default_view": {},
"name": "DeepLab Demo.ipynb",
"provenance": [],
"version": "0.3.2",
"views": {}
},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.6"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": true
}
},
"nbformat": 4,
"nbformat_minor": 1
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment