Skip to content

Instantly share code, notes, and snippets.

@shang-vikas
Created June 23, 2018 19:21
Show Gist options
  • Star 3 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save shang-vikas/509cc87d4b37694308f717b1cbb3585a to your computer and use it in GitHub Desktop.
Save shang-vikas/509cc87d4b37694308f717b1cbb3585a to your computer and use it in GitHub Desktop.
Display the source blob
Display the rendered blob
Raw
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using TensorFlow backend.\n"
]
}
],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import tensorflow as tf\n",
"import os,sys\n",
"\n",
"from keras.datasets import mnist"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"(x_train,y_train),(x_test,y_test) = mnist.load_data()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"x_train = x_train.astype(np.int64)\n",
"x_test = x_test.astype(np.int64)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"train_path = os.path.abspath('./mnist_train.tfrecords')\n",
"test_path = os.path.abspath('./mnist_test.tfrecords')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`Numpy int format ---> Numpy string format ---> TF Example ---> Serialized Example ---> TFRecords Format`"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"def _int64_feature(value):\n",
" return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n",
"\n",
"def _bytes_feature(value):\n",
" return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"def convert_to_records(x,y,path):\n",
" print('writing to {}'.format(path))\n",
" with tf.python_io.TFRecordWriter(path) as writer:\n",
" for i in range(x.shape[0]):\n",
" example = tf.train.Example(features = tf.train.Features(\n",
" feature = \n",
" {\n",
" 'image_raw':_bytes_feature(x[i].tostring()),\n",
" 'label':_int64_feature(int(y[i]))\n",
" }\n",
" )\n",
" )\n",
" writer.write(example.SerializeToString())\n",
" if i%5000==0:\n",
" print('writing {}th image'.format(i))\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"writing to /home/lilhope/Downloads/fastai/mnist_train.tfrecords\n",
"writing 0 iteration\n",
"writing 5000 iteration\n",
"writing 10000 iteration\n",
"writing 15000 iteration\n",
"writing 20000 iteration\n",
"writing 25000 iteration\n",
"writing 30000 iteration\n",
"writing 35000 iteration\n",
"writing 40000 iteration\n",
"writing 45000 iteration\n",
"writing 50000 iteration\n",
"writing 55000 iteration\n"
]
},
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"convert_to_records(x_train,y_train,train_path)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"writing to /home/lilhope/Downloads/fastai/mnist_test.tfrecords\n",
"writing 0 iteration\n",
"writing 5000 iteration\n"
]
},
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"convert_to_records(x_test,y_test,test_path)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment