tf.data

data.py

# Skriv records fil med et sample og forsoeg at loade det igen med tf.data


import numpy as np
import tensorflow as tf

## WRITE SINGLE EXAMPLE
label = 1
FILEPATH = "TEST_RECORDS"

def _int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

feature = {'label':  _int64_feature(int(label))}


example = tf.train.Example(features=tf.train.Features(feature=feature))
example_serialized = example.SerializeToString()

example_proto = tf.train.Example.FromString(example_serialized)
assert example_proto.features.feature["label"].int64_list.value[0] == 1

with tf.python_io.TFRecordWriter(FILEPATH) as writer:
    writer.write(example_serialized)

    
# PARSE SINGLE EXAMPLE
def _parse_function(proto):
    feature = {'label': tf.FixedLenFeature([], tf.int64)}
    parsed_features = tf.parse_single_example(proto, feature)
    return parsed_features['label']


dataset = tf.data.TFRecordDataset(FILEPATH)
dataset_parsed = dataset.map(_parse_function)

with tf.Session() as sess:    
    for d in dataset_parsed:
        print(d)
        print(sess.run(d))
        break

gistfile1.txt

Tensor("IteratorGetNext_3:0", shape=(), dtype=int64)

---------------------------------------------------------------------------
NotFoundError                             Traceback (most recent call last)
~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
   1355     try:
-> 1356       return fn(*args)
   1357     except errors.OpError as e:

~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
   1340       return self._call_tf_sessionrun(
-> 1341           options, feed_dict, fetch_list, target_list, run_metadata)
   1342 

~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
   1428         self._session, options, feed_dict, fetch_list, target_list,
-> 1429         run_metadata)
   1430 

NotFoundError: Resource AnonymousIterator/AnonymousIterator3/N10tensorflow4data16IteratorResourceE does not exist.
	 [[{{node IteratorGetNext_3}}]]

During handling of the above exception, another exception occurred:

NotFoundError                             Traceback (most recent call last)
<ipython-input-5-cd2d71b54fea> in <module>
     36     for d in dataset_parsed:
     37         print(d)
---> 38         print(sess.run(d))
     39         break

~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
    948     try:
    949       result = self._run(None, fetches, feed_dict, options_ptr,
--> 950                          run_metadata_ptr)
    951       if run_metadata:
    952         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
   1171     if final_fetches or final_targets or (handle and feed_dict_tensor):
   1172       results = self._do_run(handle, final_targets, final_fetches,
-> 1173                              feed_dict_tensor, options, run_metadata)
   1174     else:
   1175       results = []

~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
   1348     if handle is None:
   1349       return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1350                            run_metadata)
   1351     else:
   1352       return self._do_call(_prun_fn, handle, feeds, fetches)

~/miniconda/lib/python3.7/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
   1368           pass
   1369       message = error_interpolation.interpolate(message, self._graph)
-> 1370       raise type(e)(node_def, op, message)
   1371 
   1372   def _extend_graph(self):

NotFoundError: Resource AnonymousIterator/AnonymousIterator3/N10tensorflow4data16IteratorResourceE does not exist.
	 [[node IteratorGetNext_3 (defined at <ipython-input-5-cd2d71b54fea>:36) ]]

Original stack trace for 'IteratorGetNext_3':
  File "/home/skaae/miniconda/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/home/skaae/miniconda/lib/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel_launcher.py", line 16, in <module>
    app.launch_new_instance()
  File "/home/skaae/miniconda/lib/python3.7/site-packages/traitlets/config/application.py", line 658, in launch_instance
    app.start()
  File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel/kernelapp.py", line 563, in start
    self.io_loop.start()
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/platform/asyncio.py", line 148, in start
    self.asyncio_loop.run_forever()
  File "/home/skaae/miniconda/lib/python3.7/asyncio/base_events.py", line 539, in run_forever
    self._run_once()
  File "/home/skaae/miniconda/lib/python3.7/asyncio/base_events.py", line 1775, in _run_once
    handle._run()
  File "/home/skaae/miniconda/lib/python3.7/asyncio/events.py", line 88, in _run
    self._context.run(self._callback, *self._args)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/ioloop.py", line 690, in <lambda>
    lambda f: self._run_callback(functools.partial(callback, future))
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/ioloop.py", line 743, in _run_callback
    ret = callback()
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/gen.py", line 787, in inner
    self.run()
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/gen.py", line 748, in run
    yielded = self.gen.send(value)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel/kernelbase.py", line 365, in process_one
    yield gen.maybe_future(dispatch(*args))
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/gen.py", line 209, in wrapper
    yielded = next(result)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel/kernelbase.py", line 272, in dispatch_shell
    yield gen.maybe_future(handler(stream, idents, msg))
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/gen.py", line 209, in wrapper
    yielded = next(result)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel/kernelbase.py", line 542, in execute_request
    user_expressions, allow_stdin,
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tornado/gen.py", line 209, in wrapper
    yielded = next(result)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel/ipkernel.py", line 294, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/ipykernel/zmqshell.py", line 536, in run_cell
    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 2855, in run_cell
    raw_cell, store_history, silent, shell_futures)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 2881, in _run_cell
    return runner(coro)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/IPython/core/async_helpers.py", line 68, in _pseudo_sync_runner
    coro.send(None)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 3058, in run_cell_async
    interactivity=interactivity, compiler=compiler, result=result)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 3249, in run_ast_nodes
    if (await self.run_code(code, result,  async_=asy)):
  File "/home/skaae/miniconda/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 3326, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-5-cd2d71b54fea>", line 36, in <module>
    for d in dataset_parsed:
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/data/ops/iterator_ops.py", line 586, in __next__
    return self.next()
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/data/ops/iterator_ops.py", line 623, in next
    return self._next_internal()
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/data/ops/iterator_ops.py", line 596, in _next_internal
    output_shapes=self._flat_output_shapes)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/ops/gen_dataset_ops.py", line 1947, in iterator_get_next
    output_shapes=output_shapes, name=name)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py", line 788, in _apply_op_helper
    op_def=op_def)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/util/deprecation.py", line 507, in new_func
    return func(*args, **kwargs)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 3616, in create_op
    op_def=op_def)
  File "/home/skaae/miniconda/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 2005, in __init__
    self._traceback = tf_stack.extract_stack()