Created
March 13, 2020 21:49
-
-
Save zmjjmz/ade6ecb5eaf011788b1da62db4413cad to your computer and use it in GitHub Desktop.
TF save issue
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import tensorflow as tf | |
| from tensorflow import feature_column as fc | |
| import pandas as pd | |
| import numpy as np | |
| FC_DISPATCH = { | |
| 'numeric': lambda key, training_vals, **kwargs: fc.numeric_column(key, **kwargs), | |
| 'categorical': lambda key, training_vals, **kwargs: fc.indicator_column( | |
| fc.categorical_column_with_vocabulary_list( | |
| key, list(training_vals.unique()), **kwargs)) | |
| } | |
| class FTSModelBase(tf.keras.Model): | |
| def __init__(self, features_column_map, training_df, label_name='conversion_prob', **super_kwargs): | |
| # assemble the feature columns | |
| # create feature cols | |
| self.feature_cols = [ | |
| FC_DISPATCH[coldict['coltype']]( | |
| colname, training_df[colname], **coldict['col_kwargs']) | |
| for colname, coldict in features_column_map.items() | |
| ] | |
| self.inputs = { | |
| colname: tf.keras.layers.Input(name=colname, shape=(), dtype=training_df[colname].dtype) | |
| for colname in features_column_map.keys() | |
| } | |
| self.label_name = label_name | |
| outputs = self.get_outputs() | |
| super(FTSModelBase, self).__init__( | |
| inputs=self.inputs, outputs=outputs, **super_kwargs) | |
| def features(self): | |
| return tf.keras.layers.DenseFeatures(self.feature_cols, name='features')(self.inputs) | |
| def final_representation(self): | |
| return self.features() | |
| def classification_layer(self, n_classes=2): | |
| final_rep_layer = self.final_representation() | |
| return tf.keras.layers.Dense(n_classes, activation=tf.nn.softmax, | |
| name=self.label_name)(final_rep_layer) | |
| def get_outputs(self): | |
| # get output | |
| return self.classification_layer() | |
| def convert_dfvecs_forkeras(df, input_vector_columns, renames=None): | |
| renames = renames or {} | |
| return {renames.get(k, k): np.stack(v) for k, v in df[input_vector_columns].to_dict(orient='list').items()} | |
| def dfcol_tocategorical(df, col, rename=None, dtype='float32'): | |
| if rename is not None: | |
| label_name = rename | |
| else: | |
| label_name = col | |
| return { | |
| label_name: tf.keras.utils.to_categorical( | |
| df[col].values, | |
| num_classes=df[col].nunique(), | |
| dtype=dtype) | |
| } | |
| if __name__ == "__main__": | |
| test_df = pd.DataFrame.from_dict({ | |
| 'a': [1, 2, 3, 4], | |
| 'b': ['a', 'b', 'b', 'c'], | |
| 'conversion_prob': [0, 1, 0, 1] | |
| }) | |
| features = ['a', 'b'] | |
| label_col = 'conversion_prob' | |
| feature_spec = { | |
| 'a': { | |
| 'coltype': 'numeric', | |
| 'col_kwargs': {}, | |
| }, | |
| 'b': { | |
| 'coltype': 'categorical', | |
| 'col_kwargs': {}, | |
| } | |
| } | |
| model = FTSModelBase(feature_spec, test_df) | |
| model.compile(loss='binary_crossentropy') | |
| model.fit( | |
| convert_dfvecs_forkeras(test_df, features), | |
| dfcol_tocategorical(test_df, label_col), | |
| ) | |
| print(model.predict(convert_dfvecs_forkeras(test_df, features))) | |
| model.save('test_model.saved_model', save_format='tf') | |
| # this fails | |
| model.save('test_model_2.saved_model', save_format='tf') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2020-03-13 17:46:40.199609: W tensorflow/stream_executor/platform/default/dso_loader.cc:55] Could not load dynamic library 'libnvinfer.so.6'; dlerror: libnvinfer.so.6: cannot open shared object file: No such file or directory | |
| 2020-03-13 17:46:40.199722: W tensorflow/stream_executor/platform/default/dso_loader.cc:55] Could not load dynamic library 'libnvinfer_plugin.so.6'; dlerror: libnvinfer_plugin.so.6: cannot open shared object file: No such file or directory | |
| 2020-03-13 17:46:40.199735: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:30] Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. | |
| 2020-03-13 17:46:42.360437: W tensorflow/stream_executor/platform/default/dso_loader.cc:55] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory | |
| 2020-03-13 17:46:42.360463: E tensorflow/stream_executor/cuda/cuda_driver.cc:351] failed call to cuInit: UNKNOWN ERROR (303) | |
| WARNING: Logging before flag parsing goes to stderr. | |
| W0313 17:46:42.379219 140347342612224 deprecation.py:323] From /home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/feature_column/feature_column_v2.py:4267: IndicatorColumn._variable_shape (from tensorflow.python.feature_column.feature_column_v2) is deprecated and will be removed in a future version. | |
| Instructions for updating: | |
| The old _FeatureColumn APIs are being deprecated. Please use the new FeatureColumn APIs instead. | |
| W0313 17:46:42.379442 140347342612224 deprecation.py:323] From /home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/feature_column/feature_column_v2.py:4322: VocabularyListCategoricalColumn._num_buckets (from tensorflow.python.feature_column.feature_column_v2) is deprecated and will be removed in a future version. | |
| Instructions for updating: | |
| The old _FeatureColumn APIs are being deprecated. Please use the new FeatureColumn APIs instead. | |
| Train on 4 samples | |
| 4/4 [==============================] - 0s 121ms/sample - loss: 0.6714 | |
| 2020-03-13 17:46:43.428767: W tensorflow/python/util/util.cc:319] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them. | |
| W0313 17:46:43.660733 140347342612224 deprecation.py:506] From /home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1786: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version. | |
| Instructions for updating: | |
| If using Keras pass *_constraint arguments to layers. | |
| [[0.56088257 0.4391174 ] | |
| [0.3391014 0.66089857] | |
| [0.27229086 0.7277092 ] | |
| [0.3142721 0.6857279 ]] | |
| Traceback (most recent call last): | |
| File "tf_models.py", line 101, in <module> | |
| model.save('test_model_2.saved_model', save_format='tf') | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/keras/engine/network.py", line 1008, in save | |
| signatures, options) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/keras/saving/save.py", line 115, in save_model | |
| signatures, options) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/keras/saving/saved_model/save.py", line 78, in save | |
| save_lib.save(model, filepath, signatures, options) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/saved_model/save.py", line 899, in save | |
| _ = _SaveableView(checkpoint_graph_view) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/saved_model/save.py", line 187, in __init__ | |
| function._list_all_concrete_functions_for_serialization()) # pylint: disable=protected-access | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/eager/def_function.py", line 799, in _list_all_concrete_functions_for_serialization | |
| self.get_concrete_function() | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/eager/def_function.py", line 909, in get_concrete_function | |
| self._initialize(args, kwargs, add_initializers_to=initializers) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/eager/def_function.py", line 497, in _initialize | |
| *args, **kwds)) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/eager/function.py", line 2389, in _get_concrete_function_internal_garbage_collected | |
| graph_function, _, _ = self._maybe_define_function(args, kwargs) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/eager/function.py", line 2703, in _maybe_define_function | |
| graph_function = self._create_graph_function(args, kwargs) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/eager/function.py", line 2593, in _create_graph_function | |
| capture_by_value=self._capture_by_value), | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/framework/func_graph.py", line 978, in func_graph_from_py_func | |
| func_outputs = python_func(*func_args, **func_kwargs) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/eager/def_function.py", line 439, in wrapped_fn | |
| return weak_wrapped_fn().__wrapped__(*args, **kwds) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/training/tracking/tracking.py", line 244, in _initializer | |
| self._initialize() | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/ops/lookup_ops.py", line 181, in _initialize | |
| return self._initializer.initialize(self) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/ops/lookup_ops.py", line 464, in initialize | |
| self._keys, self._values) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/ops/gen_lookup_ops.py", line 702, in lookup_table_import_v2 | |
| values=values, name=name) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/framework/op_def_library.py", line 742, in _apply_op_helper | |
| attrs=attr_protos, op_def=op_def) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/framework/func_graph.py", line 591, in _create_op_internal | |
| inp = self.capture(inp) | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/framework/func_graph.py", line 630, in capture | |
| if tensor.graph is not self: | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/framework/ops.py", line 416, in graph | |
| return self._op.graph | |
| File "/home/u1/zach/proj/dataplayground3/lib/python3.5/site-packages/tensorflow_core/python/framework/ops.py", line 2225, in graph | |
| return self._graph | |
| AttributeError: 'Operation' object has no attribute '_graph' |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment