Skip to content

Instantly share code, notes, and snippets.

@aman-tiwari
Created November 1, 2016 20:45
Show Gist options
  • Save aman-tiwari/2d889c542042b8a37b209d0c998d6de5 to your computer and use it in GitHub Desktop.
Save aman-tiwari/2d889c542042b8a37b209d0c998d6de5 to your computer and use it in GitHub Desktop.
read remote image files in tensorflow // url file reader
import tensorflow as tf
from skimage import io
import time
N_THREADS = 64
BATCH_SIZE = 32
N_IMGS = 1000
random_img = 'url to load'
# Urls should be a list of urls.
# You could make this function stateless by passing in a placeholder that is the
# index of the url you want to load rather than auto incrementing it.
# io.imread doesn't engage the GIL so multiple requests can be made in parallel.
def url_example_producer(urls):
def load_nth_url(n):
return io.imread(urls[n])
url_iter = tf.Variable(0, name='url_iter').count_up_to(len(urls))
return tf.py_func(load_nth_url, [url_iter], stateful=True, Tout=[tf.uint8])
one_example = url_example_producer([random_img] * N_IMGS)
queue = tf.RandomShuffleQueue(capacity=500, min_after_dequeue=100, dtypes=tf.uint8, shapes=(300, 300, 3))
enq_op = queue.enqueue(one_example)
inputs = queue.dequeue_many(BATCH_SIZE)
qr = tf.train.QueueRunner(queue, [enq_op] * N_THREADS)
with tf.Session() as sess:
tf.initialize_all_variables().run()
coord = tf.train.Coordinator()
enqueue_threads = qr.create_threads(sess, coord=coord, start=True)
t = time.time()
try:
while not coord.should_stop():
# Load images
images = sess.run(inputs)
print len(images), images[0].shape
except tf.errors.OutOfRangeError:
print('Done loading -- limit reached', 'Took:', time.time() - t)
finally:
# When done, ask the threads to stop.
coord.request_stop()
# Wait for threads to finish.
coord.join(enqueue_threads)
@ContentAndMaterialPortrait

hi , i want to read url image directly ,because we will use this in tensor serving,but ,the serving does not support tf.py_func

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment