Skip to content

Instantly share code, notes, and snippets.

@skaae
Created August 29, 2019 14:01
Show Gist options
  • Save skaae/e7b5fb0ada0a54bd006feb69836dca19 to your computer and use it in GitHub Desktop.
Save skaae/e7b5fb0ada0a54bd006feb69836dca19 to your computer and use it in GitHub Desktop.
tf.data
# Skriv records fil med et sample og forsoeg at loade det igen med tf.data
import numpy as np
import tensorflow as tf
## WRITE SINGLE EXAMPLE
label = 1
FILEPATH = "TEST_RECORDS"
def _int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
feature = {'label': _int64_feature(int(label))}
example = tf.train.Example(features=tf.train.Features(feature=feature))
example_serialized = example.SerializeToString()
example_proto = tf.train.Example.FromString(example_serialized)
assert example_proto.features.feature["label"].int64_list.value[0] == 1
with tf.python_io.TFRecordWriter(FILEPATH) as writer:
writer.write(example_serialized)
# PARSE SINGLE EXAMPLE
def _parse_function(proto):
feature = {'label': tf.FixedLenFeature([], tf.int64)}
parsed_features = tf.parse_single_example(proto, feature)
return parsed_features['label']
dataset = tf.data.TFRecordDataset(FILEPATH)
dataset_parsed = dataset.map(_parse_function)
with tf.Session() as sess:
for d in dataset_parsed:
print(d)
print(sess.run(d))
break
Tensor("IteratorGetNext_3:0", shape=(), dtype=int64)
---------------------------------------------------------------------------
NotFoundError Traceback (most recent call last)
~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1355 try:
-> 1356 return fn(*args)
1357 except errors.OpError as e:
~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
1340 return self._call_tf_sessionrun(
-> 1341 options, feed_dict, fetch_list, target_list, run_metadata)
1342
~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
1428 self._session, options, feed_dict, fetch_list, target_list,
-> 1429 run_metadata)
1430
NotFoundError: Resource AnonymousIterator/AnonymousIterator3/N10tensorflow4data16IteratorResourceE does not exist.
[[{{node IteratorGetNext_3}}]]
During handling of the above exception, another exception occurred:
NotFoundError Traceback (most recent call last)
<ipython-input-5-cd2d71b54fea> in <module>
36 for d in dataset_parsed:
37 print(d)
---> 38 print(sess.run(d))
39 break
~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
948 try:
949 result = self._run(None, fetches, feed_dict, options_ptr,
--> 950 run_metadata_ptr)
951 if run_metadata:
952 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
1171 if final_fetches or final_targets or (handle and feed_dict_tensor):
1172 results = self._do_run(handle, final_targets, final_fetches,
-> 1173 feed_dict_tensor, options, run_metadata)
1174 else:
1175 results = []
~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1348 if handle is None:
1349 return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1350 run_metadata)
1351 else:
1352 return self._do_call(_prun_fn, handle, feeds, fetches)
~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1368 pass
1369 message = error_interpolation.interpolate(message, self._graph)
-> 1370 raise type(e)(node_def, op, message)
1371
1372 def _extend_graph(self):
NotFoundError: Resource AnonymousIterator/AnonymousIterator3/N10tensorflow4data16IteratorResourceE does not exist.
[[node IteratorGetNext_3 (defined at <ipython-input-5-cd2d71b54fea>:36) ]]
Original stack trace for 'IteratorGetNext_3':
File "/home/skaae/miniconda/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/home/skaae/miniconda/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "/home/skaae/miniconda/lib/python3.7/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel/kernelapp.py", line 563, in start
self.io_loop.start()
File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/platform/asyncio.py", line 148, in start
self.asyncio_loop.run_forever()
File "/home/skaae/miniconda/lib/python3.7/asyncio/base_events.py", line 539, in run_forever
self._run_once()
File "/home/skaae/miniconda/lib/python3.7/asyncio/base_events.py", line 1775, in _run_once
handle._run()
File "/home/skaae/miniconda/lib/python3.7/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/ioloop.py", line 690, in <lambda>
lambda f: self._run_callback(functools.partial(callback, future))
File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/ioloop.py", line 743, in _run_callback
ret = callback()
File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/gen.py", line 787, in inner
self.run()
File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/gen.py", line 748, in run
yielded = self.gen.send(value)
File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel/kernelbase.py", line 365, in process_one
yield gen.maybe_future(dispatch(*args))
File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel/kernelbase.py", line 272, in dispatch_shell
yield gen.maybe_future(handler(stream, idents, msg))
File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel/kernelbase.py", line 542, in execute_request
user_expressions, allow_stdin,
File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel/ipkernel.py", line 294, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel/zmqshell.py", line 536, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/home/skaae/miniconda/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 2855, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/home/skaae/miniconda/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 2881, in _run_cell
return runner(coro)
File "/home/skaae/miniconda/lib/python3.7/site-packages/IPython/core/async_helpers.py", line 68, in _pseudo_sync_runner
coro.send(None)
File "/home/skaae/miniconda/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 3058, in run_cell_async
interactivity=interactivity, compiler=compiler, result=result)
File "/home/skaae/miniconda/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 3249, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "/home/skaae/miniconda/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 3326, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-5-cd2d71b54fea>", line 36, in <module>
for d in dataset_parsed:
File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/data/ops/iterator_ops.py", line 586, in __next__
return self.next()
File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/data/ops/iterator_ops.py", line 623, in next
return self._next_internal()
File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/data/ops/iterator_ops.py", line 596, in _next_internal
output_shapes=self._flat_output_shapes)
File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/ops/gen_dataset_ops.py", line 1947, in iterator_get_next
output_shapes=output_shapes, name=name)
File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py", line 788, in _apply_op_helper
op_def=op_def)
File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/util/deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 3616, in create_op
op_def=op_def)
File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 2005, in __init__
self._traceback = tf_stack.extract_stack()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment