Created
August 22, 2018 21:50
-
-
Save zmjjmz/3f621aaafc5683238ade6224e5dedcb6 to your computer and use it in GitHub Desktop.
Keras unconsumed output failure reproduction script.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import numpy | |
| import tensorflow | |
| print("Tensorflow version: {0}".format(tensorflow.VERSION)) | |
| DATA_SIZE = 1024 | |
| BATCH_SIZE = 32 | |
| N_EPOCHS = 1 | |
| EMBED_DIM = 100 | |
| DATA_DICT = {'input': numpy.random.rand( | |
| DATA_SIZE, EMBED_DIM).astype('float32')} | |
| LABELS = tensorflow.keras.utils.to_categorical( | |
| numpy.random.randint(2, size=(DATA_SIZE,)), num_classes=2) | |
| def get_input_fn(incl_reporting=False): | |
| y_dict = {'classes':LABELS} | |
| if incl_reporting: | |
| y_dict['reporting'] = LABELS | |
| return tensorflow.estimator.inputs.numpy_input_fn( | |
| x=DATA_DICT, y=y_dict, | |
| shuffle=True, | |
| batch_size=BATCH_SIZE, | |
| num_epochs=N_EPOCHS, | |
| ) | |
| def create_keras_model(): | |
| inp = tensorflow.keras.layers.Input(shape=(EMBED_DIM,), name='input') | |
| class_layer = tensorflow.keras.layers.Dense( | |
| 2, input_shape=(EMBED_DIM,), activation='softmax', name='classes')(inp) | |
| reporting_layer = tensorflow.keras.layers.ELU(alpha=0.5, name='reporting')(inp) | |
| model = tensorflow.keras.models.Model( | |
| inputs=[inp], outputs=[class_layer, reporting_layer]) | |
| model.compile(optimizer='sgd', | |
| loss={'classes': 'categorical_crossentropy'}, | |
| metrics={'classes': 'accuracy'}) | |
| return model | |
| if __name__ == "__main__": | |
| keras_model = create_keras_model() | |
| print("Attempting to train with keras Model fit method") | |
| keras_model.fit(x=DATA_DICT, y={'classes':LABELS}, | |
| batch_size=BATCH_SIZE, epochs=N_EPOCHS) | |
| print("Converting keras model to Estimator") | |
| train_input_fn = get_input_fn(incl_reporting=True) | |
| eval_input_fn = get_input_fn(incl_reporting=True) | |
| evalspec = tensorflow.estimator.EvalSpec(eval_input_fn, steps=None, | |
| start_delay_secs=0, throttle_secs=1) | |
| steps = ((DATA_SIZE // BATCH_SIZE) + 1) * N_EPOCHS | |
| trainspec = tensorflow.estimator.TrainSpec(train_input_fn, max_steps=steps) | |
| estimator = tensorflow.keras.estimator.model_to_estimator(keras_model) | |
| print("Attempting to train with Estimator train_and_evaluate") | |
| tensorflow.estimator.train_and_evaluate( | |
| estimator=estimator, train_spec=trainspec, eval_spec=evalspec) |
What happens if this is run without providing the fake data to train_input_fn (i.e. setting incl_reporting to False on line 53):
(dataplayground2) zach@wa1okdba002:~/.fuckups$ python keras_multiout_repro.py
/home/u1/zach/proj/dataplayground2/local/lib/python2.7/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.
floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
from ._conv import register_converters as _register_converters
Tensorflow version: 1.9.0
WARNING:tensorflow:Output "reporting" missing from loss dictionary. We assume this was done on purpose, and we will not be expecting any data to be passed to "reporting" dur
ing training.
Attempting to train with keras Model fit method
Epoch 1/1
2018-08-22 17:51:39.399511: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
1024/1024 [==============================] - 0s 105us/step - loss: 0.7447 - classes_loss: 0.7447 - classes_acc: 0.4795
Converting keras model to Estimator
WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpVTv5EZ
WARNING:tensorflow:Output "reporting" missing from loss dictionary. We assume this was done on purpose, and we will not be expecting any data to be passed to "reporting" dur
ing training.
Attempting to train with Estimator train_and_evaluate
Traceback (most recent call last):
File "keras_multiout_repro.py", line 65, in <module>
estimator=estimator, train_spec=trainspec, eval_spec=evalspec)
File "/home/u1/zach/proj/dataplayground2/local/lib/python2.7/site-packages/tensorflow/python/estimator/training.py", line 447, in train_and_evaluate
return executor.run()
File "/home/u1/zach/proj/dataplayground2/local/lib/python2.7/site-packages/tensorflow/python/estimator/training.py", line 531, in run
return self.run_local()
File "/home/u1/zach/proj/dataplayground2/local/lib/python2.7/site-packages/tensorflow/python/estimator/training.py", line 669, in run_local
hooks=train_hooks)
File "/home/u1/zach/proj/dataplayground2/local/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 366, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/home/u1/zach/proj/dataplayground2/local/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 1119, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "/home/u1/zach/proj/dataplayground2/local/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 1132, in _train_model_default
features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
File "/home/u1/zach/proj/dataplayground2/local/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 1107, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "/home/u1/zach/proj/dataplayground2/local/lib/python2.7/site-packages/tensorflow/python/estimator/keras.py", line 344, in model_fn
labels)
File "/home/u1/zach/proj/dataplayground2/local/lib/python2.7/site-packages/tensorflow/python/estimator/keras.py", line 289, in _clone_and_build_model
is_input=False)
File "/home/u1/zach/proj/dataplayground2/local/lib/python2.7/site-packages/tensorflow/python/estimator/keras.py", line 127, in _create_ordered_io
for io_name in keras_io_names]
KeyError: 'reporting'
Note that it failed in _train_model specifically.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
What happens if this is run as is: