Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Facial Emotion Recognition - WSL
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Load FER data"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['ytest', 'imgsize', 'yval', 'ytrain', 'xtest', 'xtrain', 'xval', 'categories']\n",
"(25120, 2304) (25120, 7)\n"
]
}
],
"source": [
"fer = np.load(\"data/face_emotion.npz\")\n",
"print fer.files\n",
"print fer[\"xtrain\"].shape, fer[\"ytrain\"].shape"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(5382, 2304) (5382, 7)\n"
]
}
],
"source": [
"print fer[\"xtest\"].shape, fer[\"ytest\"].shape"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"xtrain = fer[\"xtrain\"]\n",
"ytrain = fer[\"ytrain\"]\n",
"xtest = fer[\"xtest\"]\n",
"ytest = fer[\"ytest\"]"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"train_mean = np.mean(xtrain, axis=0)\n",
"train_std = np.std(xtrain, axis=0)\n",
" \n",
"def whiten(x, train_mean=train_mean, train_std=train_std):\n",
" return (x - train_mean) / train_std"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Load pre-trained CNN"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Create the network\n",
"* from `.meta` file"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"INFO:tensorflow:Restoring parameters from fe_challenge/fe_cnn-NAG-whiten-200\n"
]
}
],
"source": [
"saver = tf.train.import_meta_graph(\"fe_challenge/fe_cnn-NAG-whiten-200.meta\")\n",
"\n",
"sess = tf.Session()\n",
"saver.restore(sess, \"fe_challenge/fe_cnn-NAG-whiten-200\")\n",
"graph = tf.get_default_graph()"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[u'save/RestoreV2/shape_and_slices',\n",
" u'save/RestoreV2/tensor_names',\n",
" u'save/SaveV2/shape_and_slices',\n",
" u'save/SaveV2/tensor_names',\n",
" u'save/Const',\n",
" u'save/RestoreV2',\n",
" u'Const',\n",
" u'ArgMax_1/dimension',\n",
" u'ArgMax/dimension',\n",
" u'Adam/epsilon',\n",
" u'Adam/beta2',\n",
" u'Adam/beta1',\n",
" u'LOGIT/biases/Adam_1',\n",
" u'save/Assign_58',\n",
" u'LOGIT/biases/Adam_1/read',\n",
" u'LOGIT/biases/Adam_1/Initializer/zeros',\n",
" u'LOGIT/biases/Adam_1/Assign',\n",
" u'LOGIT/biases/Adam',\n",
" u'save/Assign_57',\n",
" u'LOGIT/biases/Adam/read',\n",
" u'LOGIT/biases/Adam/Initializer/zeros',\n",
" u'LOGIT/biases/Adam/Assign',\n",
" u'LOGIT/weights/Adam_1',\n",
" u'save/Assign_61',\n",
" u'LOGIT/weights/Adam_1/read',\n",
" u'LOGIT/weights/Adam_1/Initializer/zeros/Const',\n",
" u'LOGIT/weights/Adam_1/Initializer/zeros/shape_as_tensor',\n",
" u'LOGIT/weights/Adam_1/Initializer/zeros',\n",
" u'LOGIT/weights/Adam_1/Assign',\n",
" u'LOGIT/weights/Adam',\n",
" u'save/Assign_60',\n",
" u'LOGIT/weights/Adam/read',\n",
" u'LOGIT/weights/Adam/Initializer/zeros/Const',\n",
" u'LOGIT/weights/Adam/Initializer/zeros/shape_as_tensor',\n",
" u'LOGIT/weights/Adam/Initializer/zeros',\n",
" u'LOGIT/weights/Adam/Assign',\n",
" u'DENSE2/BatchNorm/beta/Adam_1',\n",
" u'save/Assign_50',\n",
" u'DENSE2/BatchNorm/beta/Adam_1/read',\n",
" u'DENSE2/BatchNorm/beta/Adam_1/Initializer/zeros/Const',\n",
" u'DENSE2/BatchNorm/beta/Adam_1/Initializer/zeros/shape_as_tensor',\n",
" u'DENSE2/BatchNorm/beta/Adam_1/Initializer/zeros',\n",
" u'DENSE2/BatchNorm/beta/Adam_1/Assign',\n",
" u'DENSE2/BatchNorm/beta/Adam',\n",
" u'save/Assign_49',\n",
" u'DENSE2/BatchNorm/beta/Adam/read',\n",
" u'DENSE2/BatchNorm/beta/Adam/Initializer/zeros/Const',\n",
" u'DENSE2/BatchNorm/beta/Adam/Initializer/zeros/shape_as_tensor',\n",
" u'DENSE2/BatchNorm/beta/Adam/Initializer/zeros',\n",
" u'DENSE2/BatchNorm/beta/Adam/Assign',\n",
" u'DENSE2/weights/Adam_1',\n",
" u'save/Assign_55',\n",
" u'DENSE2/weights/Adam_1/read',\n",
" u'DENSE2/weights/Adam_1/Initializer/zeros/Const',\n",
" u'DENSE2/weights/Adam_1/Initializer/zeros/shape_as_tensor',\n",
" u'DENSE2/weights/Adam_1/Initializer/zeros',\n",
" u'DENSE2/weights/Adam_1/Assign',\n",
" u'DENSE2/weights/Adam',\n",
" u'save/Assign_54',\n",
" u'DENSE2/weights/Adam/read',\n",
" u'DENSE2/weights/Adam/Initializer/zeros/Const',\n",
" u'DENSE2/weights/Adam/Initializer/zeros/shape_as_tensor',\n",
" u'DENSE2/weights/Adam/Initializer/zeros',\n",
" u'DENSE2/weights/Adam/Assign',\n",
" u'DENSE1/BatchNorm/beta/Adam_1',\n",
" u'save/Assign_42',\n",
" u'DENSE1/BatchNorm/beta/Adam_1/read',\n",
" u'DENSE1/BatchNorm/beta/Adam_1/Initializer/zeros/Const',\n",
" u'DENSE1/BatchNorm/beta/Adam_1/Initializer/zeros/shape_as_tensor',\n",
" u'DENSE1/BatchNorm/beta/Adam_1/Initializer/zeros',\n",
" u'DENSE1/BatchNorm/beta/Adam_1/Assign',\n",
" u'DENSE1/BatchNorm/beta/Adam',\n",
" u'save/Assign_41',\n",
" u'DENSE1/BatchNorm/beta/Adam/read',\n",
" u'DENSE1/BatchNorm/beta/Adam/Initializer/zeros/Const',\n",
" u'DENSE1/BatchNorm/beta/Adam/Initializer/zeros/shape_as_tensor',\n",
" u'DENSE1/BatchNorm/beta/Adam/Initializer/zeros',\n",
" u'DENSE1/BatchNorm/beta/Adam/Assign',\n",
" u'DENSE1/weights/Adam_1',\n",
" u'save/Assign_47',\n",
" u'DENSE1/weights/Adam_1/read',\n",
" u'DENSE1/weights/Adam_1/Initializer/zeros/Const',\n",
" u'DENSE1/weights/Adam_1/Initializer/zeros/shape_as_tensor',\n",
" u'DENSE1/weights/Adam_1/Initializer/zeros',\n",
" u'DENSE1/weights/Adam_1/Assign',\n",
" u'DENSE1/weights/Adam',\n",
" u'save/Assign_46',\n",
" u'DENSE1/weights/Adam/read',\n",
" u'DENSE1/weights/Adam/Initializer/zeros/Const',\n",
" u'DENSE1/weights/Adam/Initializer/zeros/shape_as_tensor',\n",
" u'DENSE1/weights/Adam/Initializer/zeros',\n",
" u'DENSE1/weights/Adam/Assign',\n",
" u'CONV6/BatchNorm/beta/Adam_1',\n",
" u'save/Assign_34',\n",
" u'CONV6/BatchNorm/beta/Adam_1/read',\n",
" u'CONV6/BatchNorm/beta/Adam_1/Initializer/zeros',\n",
" u'CONV6/BatchNorm/beta/Adam_1/Assign',\n",
" u'CONV6/BatchNorm/beta/Adam',\n",
" u'save/Assign_33',\n",
" u'CONV6/BatchNorm/beta/Adam/read',\n",
" u'CONV6/BatchNorm/beta/Adam/Initializer/zeros',\n",
" u'CONV6/BatchNorm/beta/Adam/Assign',\n",
" u'CONV6/weights/Adam_1',\n",
" u'save/Assign_39',\n",
" u'CONV6/weights/Adam_1/read',\n",
" u'CONV6/weights/Adam_1/Initializer/zeros/Const',\n",
" u'CONV6/weights/Adam_1/Initializer/zeros/shape_as_tensor',\n",
" u'CONV6/weights/Adam_1/Initializer/zeros',\n",
" u'CONV6/weights/Adam_1/Assign',\n",
" u'CONV6/weights/Adam',\n",
" u'save/Assign_38',\n",
" u'CONV6/weights/Adam/read',\n",
" u'CONV6/weights/Adam/Initializer/zeros/Const',\n",
" u'CONV6/weights/Adam/Initializer/zeros/shape_as_tensor',\n",
" u'CONV6/weights/Adam/Initializer/zeros',\n",
" u'CONV6/weights/Adam/Assign',\n",
" u'CONV5/BatchNorm/beta/Adam_1',\n",
" u'save/Assign_26',\n",
" u'CONV5/BatchNorm/beta/Adam_1/read',\n",
" u'CONV5/BatchNorm/beta/Adam_1/Initializer/zeros',\n",
" u'CONV5/BatchNorm/beta/Adam_1/Assign',\n",
" u'CONV5/BatchNorm/beta/Adam',\n",
" u'save/Assign_25',\n",
" u'CONV5/BatchNorm/beta/Adam/read',\n",
" u'CONV5/BatchNorm/beta/Adam/Initializer/zeros',\n",
" u'CONV5/BatchNorm/beta/Adam/Assign',\n",
" u'CONV5/weights/Adam_1',\n",
" u'save/Assign_31',\n",
" u'CONV5/weights/Adam_1/read',\n",
" u'CONV5/weights/Adam_1/Initializer/zeros/Const',\n",
" u'CONV5/weights/Adam_1/Initializer/zeros/shape_as_tensor',\n",
" u'CONV5/weights/Adam_1/Initializer/zeros',\n",
" u'CONV5/weights/Adam_1/Assign',\n",
" u'CONV5/weights/Adam',\n",
" u'save/Assign_30',\n",
" u'CONV5/weights/Adam/read',\n",
" u'CONV5/weights/Adam/Initializer/zeros/Const',\n",
" u'CONV5/weights/Adam/Initializer/zeros/shape_as_tensor',\n",
" u'CONV5/weights/Adam/Initializer/zeros',\n",
" u'CONV5/weights/Adam/Assign',\n",
" u'CONV4/BatchNorm/beta/Adam_1',\n",
" u'save/Assign_18',\n",
" u'CONV4/BatchNorm/beta/Adam_1/read',\n",
" u'CONV4/BatchNorm/beta/Adam_1/Initializer/zeros',\n",
" u'CONV4/BatchNorm/beta/Adam_1/Assign',\n",
" u'CONV4/BatchNorm/beta/Adam',\n",
" u'save/Assign_17',\n",
" u'CONV4/BatchNorm/beta/Adam/read',\n",
" u'CONV4/BatchNorm/beta/Adam/Initializer/zeros',\n",
" u'CONV4/BatchNorm/beta/Adam/Assign',\n",
" u'CONV4/weights/Adam_1',\n",
" u'save/Assign_23',\n",
" u'CONV4/weights/Adam_1/read',\n",
" u'CONV4/weights/Adam_1/Initializer/zeros/Const',\n",
" u'CONV4/weights/Adam_1/Initializer/zeros/shape_as_tensor',\n",
" u'CONV4/weights/Adam_1/Initializer/zeros',\n",
" u'CONV4/weights/Adam_1/Assign',\n",
" u'CONV4/weights/Adam',\n",
" u'save/Assign_22',\n",
" u'CONV4/weights/Adam/read',\n",
" u'CONV4/weights/Adam/Initializer/zeros/Const',\n",
" u'CONV4/weights/Adam/Initializer/zeros/shape_as_tensor',\n",
" u'CONV4/weights/Adam/Initializer/zeros',\n",
" u'CONV4/weights/Adam/Assign',\n",
" u'CONV3/BatchNorm/beta/Adam_1',\n",
" u'save/Assign_10',\n",
" u'CONV3/BatchNorm/beta/Adam_1/read',\n",
" u'CONV3/BatchNorm/beta/Adam_1/Initializer/zeros',\n",
" u'CONV3/BatchNorm/beta/Adam_1/Assign',\n",
" u'CONV3/BatchNorm/beta/Adam',\n",
" u'save/Assign_9',\n",
" u'CONV3/BatchNorm/beta/Adam/read',\n",
" u'CONV3/BatchNorm/beta/Adam/Initializer/zeros',\n",
" u'CONV3/BatchNorm/beta/Adam/Assign',\n",
" u'CONV3/weights/Adam_1',\n",
" u'save/Assign_15',\n",
" u'CONV3/weights/Adam_1/read',\n",
" u'CONV3/weights/Adam_1/Initializer/zeros/Const',\n",
" u'CONV3/weights/Adam_1/Initializer/zeros/shape_as_tensor',\n",
" u'CONV3/weights/Adam_1/Initializer/zeros',\n",
" u'CONV3/weights/Adam_1/Assign',\n",
" u'CONV3/weights/Adam',\n",
" u'save/Assign_14',\n",
" u'CONV3/weights/Adam/read',\n",
" u'CONV3/weights/Adam/Initializer/zeros/Const',\n",
" u'CONV3/weights/Adam/Initializer/zeros/shape_as_tensor',\n",
" u'CONV3/weights/Adam/Initializer/zeros',\n",
" u'CONV3/weights/Adam/Assign',\n",
" u'beta2_power',\n",
" u'save/Assign_64',\n",
" u'beta2_power/read',\n",
" u'beta2_power/initial_value',\n",
" u'beta2_power/Assign',\n",
" u'beta1_power',\n",
" u'save/Assign_63',\n",
" u'beta1_power/read',\n",
" u'beta1_power/initial_value',\n",
" u'beta1_power/Assign',\n",
" u'gradients/CONV3/Conv2D_grad/Const',\n",
" u'gradients/CONV3/lrelu/mul_1_grad/Shape',\n",
" u'gradients/CONV3/lrelu/mul_grad/Shape',\n",
" u'gradients/CONV4/Conv2D_grad/Const',\n",
" u'gradients/CONV4/lrelu/mul_1_grad/Shape',\n",
" u'gradients/CONV4/lrelu/mul_grad/Shape',\n",
" u'gradients/CONV5/Conv2D_grad/Const',\n",
" u'gradients/CONV5/lrelu/mul_1_grad/Shape',\n",
" u'gradients/CONV5/lrelu/mul_grad/Shape',\n",
" u'gradients/CONV6/Conv2D_grad/Const',\n",
" u'gradients/CONV6/lrelu/mul_1_grad/Shape',\n",
" u'gradients/CONV6/lrelu/mul_grad/Shape',\n",
" u'gradients/DENSE1/lrelu/mul_1_grad/Shape',\n",
" u'gradients/DENSE1/lrelu/mul_grad/Shape',\n",
" u'gradients/DROP1/cond/dropout/div_grad/Shape_1',\n",
" u'gradients/DENSE2/lrelu/mul_1_grad/Shape',\n",
" u'gradients/DENSE2/lrelu/mul_grad/Shape',\n",
" u'gradients/DROP2/cond/dropout/div_grad/Shape_1',\n",
" u'gradients/LOGIT/Softmax_grad/Reshape/shape',\n",
" u'gradients/LOGIT/Softmax_grad/Sum/reduction_indices',\n",
" u'gradients/softmax_cross_entropy_loss/xentropy_grad/ExpandDims_1/dim',\n",
" u'gradients/softmax_cross_entropy_loss/xentropy_grad/ExpandDims/dim',\n",
" u'gradients/softmax_cross_entropy_loss/num_present/broadcast_weights/ones_like_grad/Const',\n",
" u'gradients/softmax_cross_entropy_loss/num_present_grad/Reshape/shape',\n",
" u'gradients/softmax_cross_entropy_loss/Sum_grad/Reshape/shape',\n",
" u'gradients/softmax_cross_entropy_loss/Select_grad/zeros_like',\n",
" u'gradients/softmax_cross_entropy_loss/Sum_1_grad/Const',\n",
" u'gradients/softmax_cross_entropy_loss/Sum_1_grad/Reshape/shape',\n",
" u'gradients/softmax_cross_entropy_loss/div_grad/Shape_1',\n",
" u'gradients/softmax_cross_entropy_loss/div_grad/Shape',\n",
" u'gradients/softmax_cross_entropy_loss/div_grad/BroadcastGradientArgs',\n",
" u'gradients/softmax_cross_entropy_loss/value_grad/zeros_like',\n",
" u'gradients/grad_ys_0',\n",
" u'gradients/Shape',\n",
" u'gradients/Fill',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/Const_5',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/Const_4',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/Const_3',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/Const_2',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/Const_1',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/Const',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_scalar/x',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/values/rank',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/weights/rank',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_scalar',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/pred_id',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/is_same_rank/Switch_1',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/is_same_rank/Switch',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/is_same_rank',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/pred_id',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/Switch_1',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/Switch',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/switch_f',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/switch_t',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/x',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims_1/dim',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/concat/axis',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ones_like/Const',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ones_like/Shape',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ones_like',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims/dim',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/Switch_1',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/Switch',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/switch_f',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/switch_t',\n",
" u'softmax_cross_entropy_loss/xentropy/Slice_2/begin',\n",
" u'softmax_cross_entropy_loss/xentropy/Sub_2/y',\n",
" u'softmax_cross_entropy_loss/xentropy/concat_1/axis',\n",
" u'softmax_cross_entropy_loss/xentropy/concat_1/values_0',\n",
" u'softmax_cross_entropy_loss/xentropy/Slice_1/size',\n",
" u'softmax_cross_entropy_loss/xentropy/Sub_1/y',\n",
" u'softmax_cross_entropy_loss/xentropy/Rank_2',\n",
" u'softmax_cross_entropy_loss/xentropy/Sub_1',\n",
" u'softmax_cross_entropy_loss/xentropy/Slice_1/begin',\n",
" u'softmax_cross_entropy_loss/xentropy/concat/axis',\n",
" u'softmax_cross_entropy_loss/xentropy/concat/values_0',\n",
" u'softmax_cross_entropy_loss/xentropy/Slice/size',\n",
" u'softmax_cross_entropy_loss/xentropy/Sub/y',\n",
" u'softmax_cross_entropy_loss/xentropy/Rank_1',\n",
" u'softmax_cross_entropy_loss/xentropy/Sub',\n",
" u'softmax_cross_entropy_loss/xentropy/Slice/begin',\n",
" u'softmax_cross_entropy_loss/xentropy/Rank',\n",
" u'softmax_cross_entropy_loss/xentropy/Sub_2',\n",
" u'softmax_cross_entropy_loss/xentropy/Slice_2/size',\n",
" u'Placeholder_3',\n",
" u'gradients/softmax_cross_entropy_loss/Mul_grad/Shape_1',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/weights/shape',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims_1/Switch',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims_1/Switch_1',\n",
" u'softmax_cross_entropy_loss/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims_1',\n",
" u'InverseTimeDecay/Const',\n",
" u'InverseTimeDecay/Cast_3',\n",
" u'InverseTimeDecay/Cast_2/x',\n",
" u'InverseTimeDecay/Cast_1/x',\n",
" u'InverseTimeDecay/learning_rate',\n",
" u'Variable',\n",
" u'save/Assign_62',\n",
" u'Variable/read',\n",
" u'InverseTimeDecay/Cast',\n",
" u'InverseTimeDecay/truediv',\n",
" u'InverseTimeDecay/Floor',\n",
" u'InverseTimeDecay/Mul',\n",
" u'InverseTimeDecay/Add',\n",
" u'InverseTimeDecay',\n",
" u'Variable/initial_value',\n",
" u'Variable/Assign',\n",
" u'LOGIT/biases',\n",
" u'save/Assign_56',\n",
" u'LOGIT/biases/read',\n",
" u'LOGIT/biases/Initializer/zeros',\n",
" u'LOGIT/biases/Assign',\n",
" u'LOGIT/weights',\n",
" u'save/Assign_59',\n",
" u'LOGIT/weights/read',\n",
" u'LOGIT/weights/Initializer/truncated_normal/stddev',\n",
" u'LOGIT/weights/Initializer/truncated_normal/mean',\n",
" u'LOGIT/weights/Initializer/truncated_normal/shape',\n",
" u'LOGIT/weights/Initializer/truncated_normal/TruncatedNormal',\n",
" u'LOGIT/weights/Initializer/truncated_normal/mul',\n",
" u'LOGIT/weights/Initializer/truncated_normal',\n",
" u'LOGIT/weights/Assign',\n",
" u'DENSE2/lrelu/mul_1/x',\n",
" u'DENSE2/lrelu/mul/x',\n",
" u'DENSE2/BatchNorm/moving_variance',\n",
" u'save/Assign_52',\n",
" u'DENSE2/BatchNorm/moving_variance/read',\n",
" u'DENSE2/BatchNorm/moving_variance/Initializer/ones/Const',\n",
" u'DENSE2/BatchNorm/moving_variance/Initializer/ones/shape_as_tensor',\n",
" u'DENSE2/BatchNorm/moving_variance/Initializer/ones',\n",
" u'DENSE2/BatchNorm/moving_variance/Assign',\n",
" u'DENSE2/BatchNorm/moving_mean',\n",
" u'save/Assign_51',\n",
" u'DENSE2/BatchNorm/moving_mean/read',\n",
" u'DENSE2/BatchNorm/moving_mean/Initializer/zeros/Const',\n",
" u'DENSE2/BatchNorm/moving_mean/Initializer/zeros/shape_as_tensor',\n",
" u'DENSE2/BatchNorm/moving_mean/Initializer/zeros',\n",
" u'DENSE2/BatchNorm/moving_mean/Assign',\n",
" u'DENSE2/BatchNorm/Const',\n",
" u'DENSE2/BatchNorm/beta',\n",
" u'save/Assign_48',\n",
" u'DENSE2/BatchNorm/beta/read',\n",
" u'DENSE2/BatchNorm/beta/Initializer/zeros/Const',\n",
" u'DENSE2/BatchNorm/beta/Initializer/zeros/shape_as_tensor',\n",
" u'DENSE2/BatchNorm/beta/Initializer/zeros',\n",
" u'DENSE2/BatchNorm/beta/Assign',\n",
" u'DENSE2/BatchNorm/Reshape/shape',\n",
" u'DENSE2/weights',\n",
" u'save/Assign_53',\n",
" u'DENSE2/weights/read',\n",
" u'DENSE2/weights/Initializer/truncated_normal/stddev',\n",
" u'DENSE2/weights/Initializer/truncated_normal/mean',\n",
" u'DENSE2/weights/Initializer/truncated_normal/shape',\n",
" u'DENSE2/weights/Initializer/truncated_normal/TruncatedNormal',\n",
" u'DENSE2/weights/Initializer/truncated_normal/mul',\n",
" u'DENSE2/weights/Initializer/truncated_normal',\n",
" u'DENSE2/weights/Assign',\n",
" u'DENSE1/lrelu/mul_1/x',\n",
" u'DENSE1/lrelu/mul/x',\n",
" u'DENSE1/BatchNorm/moving_variance',\n",
" u'save/Assign_44',\n",
" u'DENSE1/BatchNorm/moving_variance/read',\n",
" u'DENSE1/BatchNorm/moving_variance/Initializer/ones/Const',\n",
" u'DENSE1/BatchNorm/moving_variance/Initializer/ones/shape_as_tensor',\n",
" u'DENSE1/BatchNorm/moving_variance/Initializer/ones',\n",
" u'DENSE1/BatchNorm/moving_variance/Assign',\n",
" u'DENSE1/BatchNorm/moving_mean',\n",
" u'save/Assign_43',\n",
" u'DENSE1/BatchNorm/moving_mean/read',\n",
" u'DENSE1/BatchNorm/moving_mean/Initializer/zeros/Const',\n",
" u'DENSE1/BatchNorm/moving_mean/Initializer/zeros/shape_as_tensor',\n",
" u'DENSE1/BatchNorm/moving_mean/Initializer/zeros',\n",
" u'DENSE1/BatchNorm/moving_mean/Assign',\n",
" u'DENSE1/BatchNorm/Const',\n",
" u'DENSE1/BatchNorm/beta',\n",
" u'save/Assign_40',\n",
" u'DENSE1/BatchNorm/beta/read',\n",
" u'DENSE1/BatchNorm/beta/Initializer/zeros/Const',\n",
" u'DENSE1/BatchNorm/beta/Initializer/zeros/shape_as_tensor',\n",
" u'DENSE1/BatchNorm/beta/Initializer/zeros',\n",
" u'DENSE1/BatchNorm/beta/Assign',\n",
" u'DENSE1/BatchNorm/Reshape/shape',\n",
" u'DENSE1/weights',\n",
" u'save/Assign_45',\n",
" u'DENSE1/weights/read',\n",
" u'DENSE1/weights/Initializer/truncated_normal/stddev',\n",
" u'DENSE1/weights/Initializer/truncated_normal/mean',\n",
" u'DENSE1/weights/Initializer/truncated_normal/shape',\n",
" u'DENSE1/weights/Initializer/truncated_normal/TruncatedNormal',\n",
" u'DENSE1/weights/Initializer/truncated_normal/mul',\n",
" u'DENSE1/weights/Initializer/truncated_normal',\n",
" u'DENSE1/weights/Assign',\n",
" u'Flatten/flatten/Reshape/shape/1',\n",
" u'Flatten/flatten/strided_slice/stack_2',\n",
" u'Flatten/flatten/strided_slice/stack_1',\n",
" u'Flatten/flatten/strided_slice/stack',\n",
" u'CONV6/lrelu/mul_1/x',\n",
" u'CONV6/lrelu/mul/x',\n",
" u'CONV6/BatchNorm/moving_variance',\n",
" u'save/Assign_36',\n",
" u'CONV6/BatchNorm/moving_variance/read',\n",
" u'CONV6/BatchNorm/moving_variance/Initializer/ones',\n",
" u'CONV6/BatchNorm/moving_variance/Assign',\n",
" u'CONV6/BatchNorm/moving_mean',\n",
" u'save/Assign_35',\n",
" u'CONV6/BatchNorm/moving_mean/read',\n",
" u'CONV6/BatchNorm/moving_mean/Initializer/zeros',\n",
" u'CONV6/BatchNorm/moving_mean/Assign',\n",
" u'CONV6/BatchNorm/Const',\n",
" u'CONV6/BatchNorm/beta',\n",
" u'save/Assign_32',\n",
" u'CONV6/BatchNorm/beta/read',\n",
" u'CONV6/BatchNorm/beta/Initializer/zeros',\n",
" u'CONV6/BatchNorm/beta/Assign',\n",
" u'CONV6/dilation_rate',\n",
" u'CONV6/weights',\n",
" u'save/Assign_37',\n",
" u'CONV6/weights/read',\n",
" u'CONV6/weights/Initializer/truncated_normal/stddev',\n",
" u'CONV6/weights/Initializer/truncated_normal/mean',\n",
" u'CONV6/weights/Initializer/truncated_normal/shape',\n",
" u'CONV6/weights/Initializer/truncated_normal/TruncatedNormal',\n",
" u'CONV6/weights/Initializer/truncated_normal/mul',\n",
" u'CONV6/weights/Initializer/truncated_normal',\n",
" u'CONV6/weights/Assign',\n",
" u'CONV5/lrelu/mul_1/x',\n",
" u'CONV5/lrelu/mul/x',\n",
" u'CONV5/BatchNorm/moving_variance',\n",
" u'save/Assign_28',\n",
" u'CONV5/BatchNorm/moving_variance/read',\n",
" u'CONV5/BatchNorm/moving_variance/Initializer/ones',\n",
" u'CONV5/BatchNorm/moving_variance/Assign',\n",
" u'CONV5/BatchNorm/moving_mean',\n",
" u'save/Assign_27',\n",
" u'CONV5/BatchNorm/moving_mean/read',\n",
" u'CONV5/BatchNorm/moving_mean/Initializer/zeros',\n",
" u'CONV5/BatchNorm/moving_mean/Assign',\n",
" u'CONV5/BatchNorm/Const',\n",
" u'CONV5/BatchNorm/beta',\n",
" u'save/Assign_24',\n",
" u'CONV5/BatchNorm/beta/read',\n",
" u'CONV5/BatchNorm/beta/Initializer/zeros',\n",
" u'CONV5/BatchNorm/beta/Assign',\n",
" u'CONV5/dilation_rate',\n",
" u'CONV5/weights',\n",
" u'save/Assign_29',\n",
" u'CONV5/weights/read',\n",
" u'CONV5/weights/Initializer/truncated_normal/stddev',\n",
" u'CONV5/weights/Initializer/truncated_normal/mean',\n",
" u'CONV5/weights/Initializer/truncated_normal/shape',\n",
" u'CONV5/weights/Initializer/truncated_normal/TruncatedNormal',\n",
" u'CONV5/weights/Initializer/truncated_normal/mul',\n",
" u'CONV5/weights/Initializer/truncated_normal',\n",
" u'CONV5/weights/Assign',\n",
" u'CONV4/lrelu/mul_1/x',\n",
" u'CONV4/lrelu/mul/x',\n",
" u'CONV4/BatchNorm/moving_variance',\n",
" u'save/Assign_20',\n",
" u'CONV4/BatchNorm/moving_variance/read',\n",
" u'CONV4/BatchNorm/moving_variance/Initializer/ones',\n",
" u'CONV4/BatchNorm/moving_variance/Assign',\n",
" u'CONV4/BatchNorm/moving_mean',\n",
" u'save/Assign_19',\n",
" u'CONV4/BatchNorm/moving_mean/read',\n",
" u'CONV4/BatchNorm/moving_mean/Initializer/zeros',\n",
" u'CONV4/BatchNorm/moving_mean/Assign',\n",
" u'CONV4/BatchNorm/Const',\n",
" u'CONV4/BatchNorm/beta',\n",
" u'save/Assign_16',\n",
" u'CONV4/BatchNorm/beta/read',\n",
" u'CONV4/BatchNorm/beta/Initializer/zeros',\n",
" u'CONV4/BatchNorm/beta/Assign',\n",
" u'CONV4/dilation_rate',\n",
" u'CONV4/weights',\n",
" u'save/Assign_21',\n",
" u'CONV4/weights/read',\n",
" u'CONV4/weights/Initializer/truncated_normal/stddev',\n",
" u'CONV4/weights/Initializer/truncated_normal/mean',\n",
" u'CONV4/weights/Initializer/truncated_normal/shape',\n",
" u'CONV4/weights/Initializer/truncated_normal/TruncatedNormal',\n",
" u'CONV4/weights/Initializer/truncated_normal/mul',\n",
" u'CONV4/weights/Initializer/truncated_normal',\n",
" u'CONV4/weights/Assign',\n",
" u'CONV3/lrelu/mul_1/x',\n",
" u'CONV3/lrelu/mul/x',\n",
" u'CONV3/BatchNorm/moving_variance',\n",
" u'save/Assign_12',\n",
" u'CONV3/BatchNorm/moving_variance/read',\n",
" u'CONV3/BatchNorm/moving_variance/Initializer/ones',\n",
" u'CONV3/BatchNorm/moving_variance/Assign',\n",
" u'CONV3/BatchNorm/moving_mean',\n",
" u'save/Assign_11',\n",
" u'CONV3/BatchNorm/moving_mean/read',\n",
" u'CONV3/BatchNorm/moving_mean/Initializer/zeros',\n",
" u'CONV3/BatchNorm/moving_mean/Assign',\n",
" u'CONV3/BatchNorm/Const',\n",
" u'CONV3/BatchNorm/beta',\n",
" u'save/Assign_8',\n",
" u'CONV3/BatchNorm/beta/read',\n",
" u'CONV3/BatchNorm/beta/Initializer/zeros',\n",
" u'CONV3/BatchNorm/beta/Assign',\n",
" u'CONV3/dilation_rate',\n",
" u'CONV3/weights',\n",
" u'save/Assign_13',\n",
" u'CONV3/weights/read',\n",
" u'CONV3/weights/Initializer/truncated_normal/stddev',\n",
" u'CONV3/weights/Initializer/truncated_normal/mean',\n",
" u'CONV3/weights/Initializer/truncated_normal/shape',\n",
" u'CONV3/weights/Initializer/truncated_normal/TruncatedNormal',\n",
" u'CONV3/weights/Initializer/truncated_normal/mul',\n",
" u'CONV3/weights/Initializer/truncated_normal',\n",
" u'CONV3/weights/Assign',\n",
" u'CONV2/lrelu/mul_1/x',\n",
" u'CONV2/lrelu/mul/x',\n",
" u'CONV2/BatchNorm/moving_variance',\n",
" u'save/Assign_6',\n",
" u'CONV2/BatchNorm/moving_variance/read',\n",
" u'CONV2/BatchNorm/moving_variance/Initializer/ones',\n",
" u'CONV2/BatchNorm/moving_variance/Assign',\n",
" u'CONV2/BatchNorm/moving_mean',\n",
" u'save/Assign_5',\n",
" u'CONV2/BatchNorm/moving_mean/read',\n",
" u'CONV2/BatchNorm/moving_mean/Initializer/zeros',\n",
" u'CONV2/BatchNorm/moving_mean/Assign',\n",
" u'CONV2/BatchNorm/Const',\n",
" u'CONV2/BatchNorm/beta',\n",
" u'save/Assign_4',\n",
" u'CONV2/BatchNorm/beta/read',\n",
" u'CONV2/BatchNorm/beta/Initializer/zeros',\n",
" u'CONV2/BatchNorm/beta/Assign',\n",
" u'CONV2/dilation_rate',\n",
" u'CONV2/weights',\n",
" u'save/Assign_7',\n",
" u'CONV2/weights/read',\n",
" u'CONV2/weights/Initializer/truncated_normal/stddev',\n",
" u'CONV2/weights/Initializer/truncated_normal/mean',\n",
" u'CONV2/weights/Initializer/truncated_normal/shape',\n",
" u'CONV2/weights/Initializer/truncated_normal/TruncatedNormal',\n",
" u'CONV2/weights/Initializer/truncated_normal/mul',\n",
" u'CONV2/weights/Initializer/truncated_normal',\n",
" u'CONV2/weights/Assign',\n",
" u'CONV1/lrelu/mul_1/x',\n",
" u'CONV1/lrelu/mul/x',\n",
" u'CONV1/BatchNorm/moving_variance',\n",
" u'save/Assign_2',\n",
" u'CONV1/BatchNorm/moving_variance/read',\n",
" u'CONV1/BatchNorm/moving_variance/Initializer/ones',\n",
" u'CONV1/BatchNorm/moving_variance/Assign',\n",
" u'CONV1/BatchNorm/moving_mean',\n",
" u'save/Assign_1',\n",
" u'CONV1/BatchNorm/moving_mean/read',\n",
" u'CONV1/BatchNorm/moving_mean/Initializer/zeros',\n",
" u'CONV1/BatchNorm/moving_mean/Assign',\n",
" u'CONV1/BatchNorm/Const',\n",
" u'CONV1/BatchNorm/beta',\n",
" u'save/Assign',\n",
" u'CONV1/BatchNorm/beta/read',\n",
" u'CONV1/BatchNorm/beta/Initializer/zeros',\n",
" u'CONV1/BatchNorm/beta/Assign',\n",
" u'CONV1/dilation_rate',\n",
" u'CONV1/weights',\n",
" u'save/Assign_3',\n",
" u'save/restore_all',\n",
" u'save/SaveV2',\n",
" u'save/control_dependency',\n",
" u'CONV1/weights/read',\n",
" u'CONV1/weights/Initializer/truncated_normal/stddev',\n",
" u'CONV1/weights/Initializer/truncated_normal/mean',\n",
" u'CONV1/weights/Initializer/truncated_normal/shape',\n",
" u'CONV1/weights/Initializer/truncated_normal/TruncatedNormal',\n",
" u'CONV1/weights/Initializer/truncated_normal/mul',\n",
" u'CONV1/weights/Initializer/truncated_normal',\n",
" u'CONV1/weights/Assign',\n",
" u'init',\n",
" u'Reshape/shape',\n",
" u'Placeholder_2',\n",
" u'DROP2/cond/pred_id',\n",
" u'DROP2/cond/Switch',\n",
" u'DROP2/cond/switch_f',\n",
" u'DROP2/cond/switch_t',\n",
" u'DROP2/cond/dropout/random_uniform/max',\n",
" u'DROP2/cond/dropout/random_uniform/min',\n",
" u'DROP2/cond/dropout/random_uniform/sub',\n",
" u'DROP2/cond/dropout/keep_prob',\n",
" u'DENSE2/BatchNorm/cond_1/pred_id',\n",
" u'DENSE2/BatchNorm/cond_1/AssignMovingAvg_1/Switch',\n",
" u'DENSE2/BatchNorm/cond_1/AssignMovingAvg_1/sub/Switch',\n",
" u'DENSE2/BatchNorm/cond_1/AssignMovingAvg/Switch',\n",
" u'DENSE2/BatchNorm/cond_1/AssignMovingAvg/sub/Switch',\n",
" u'DENSE2/BatchNorm/cond_1/Switch',\n",
" u'DENSE2/BatchNorm/cond_1/switch_f',\n",
" u'DENSE2/BatchNorm/cond_1/switch_t',\n",
" u'DENSE2/BatchNorm/cond_1/AssignMovingAvg_1/decay',\n",
" u'DENSE2/BatchNorm/cond_1/AssignMovingAvg/decay',\n",
" u'DENSE2/BatchNorm/cond/pred_id',\n",
" u'gradients/Switch_7',\n",
" u'gradients/zeros_7/Const',\n",
" u'gradients/Shape_8',\n",
" u'gradients/zeros_7',\n",
" u'gradients/Switch_5',\n",
" u'gradients/zeros_5/Const',\n",
" u'gradients/Shape_6',\n",
" u'gradients/zeros_5',\n",
" u'DENSE2/BatchNorm/cond/FusedBatchNorm_1/Switch_4',\n",
" u'DENSE2/BatchNorm/cond/FusedBatchNorm_1/Switch_3',\n",
" u'DENSE2/BatchNorm/cond/FusedBatchNorm_1/Switch_2',\n",
" u'DENSE2/BatchNorm/cond/FusedBatchNorm_1/Switch_1',\n",
" u'DENSE2/BatchNorm/cond/FusedBatchNorm/Switch_2',\n",
" u'DENSE2/BatchNorm/cond/FusedBatchNorm/Switch_1',\n",
" u'DENSE2/BatchNorm/cond/Switch',\n",
" u'DENSE2/BatchNorm/cond/switch_f',\n",
" u'DENSE2/BatchNorm/cond/switch_t',\n",
" u'DENSE2/BatchNorm/cond/Const_1',\n",
" u'DENSE2/BatchNorm/cond/Const',\n",
" u'DROP1/cond/pred_id',\n",
" u'DROP1/cond/Switch',\n",
" u'DROP1/cond/switch_f',\n",
" u'DROP1/cond/switch_t',\n",
" u'DROP1/cond/dropout/random_uniform/max',\n",
" u'DROP1/cond/dropout/random_uniform/min',\n",
" u'DROP1/cond/dropout/random_uniform/sub',\n",
" u'DROP1/cond/dropout/keep_prob',\n",
" u'DENSE1/BatchNorm/cond_1/pred_id',\n",
" u'DENSE1/BatchNorm/cond_1/AssignMovingAvg_1/Switch',\n",
" u'DENSE1/BatchNorm/cond_1/AssignMovingAvg_1/sub/Switch',\n",
" u'DENSE1/BatchNorm/cond_1/AssignMovingAvg/Switch',\n",
" u'DENSE1/BatchNorm/cond_1/AssignMovingAvg/sub/Switch',\n",
" u'DENSE1/BatchNorm/cond_1/Switch',\n",
" u'DENSE1/BatchNorm/cond_1/switch_f',\n",
" u'DENSE1/BatchNorm/cond_1/switch_t',\n",
" u'DENSE1/BatchNorm/cond_1/AssignMovingAvg_1/decay',\n",
" u'DENSE1/BatchNorm/cond_1/AssignMovingAvg/decay',\n",
" u'DENSE1/BatchNorm/cond/pred_id',\n",
" u'gradients/Switch_15',\n",
" u'gradients/zeros_15/Const',\n",
" u'gradients/Shape_16',\n",
" u'gradients/zeros_15',\n",
" u'gradients/Switch_13',\n",
" u'gradients/zeros_13/Const',\n",
" u'gradients/Shape_14',\n",
" u'gradients/zeros_13',\n",
" u'DENSE1/BatchNorm/cond/FusedBatchNorm_1/Switch_4',\n",
" u'DENSE1/BatchNorm/cond/FusedBatchNorm_1/Switch_3',\n",
" u'DENSE1/BatchNorm/cond/FusedBatchNorm_1/Switch_2',\n",
" u'DENSE1/BatchNorm/cond/FusedBatchNorm_1/Switch_1',\n",
" u'DENSE1/BatchNorm/cond/FusedBatchNorm/Switch_2',\n",
" u'DENSE1/BatchNorm/cond/FusedBatchNorm/Switch_1',\n",
" u'DENSE1/BatchNorm/cond/Switch',\n",
" u'DENSE1/BatchNorm/cond/switch_f',\n",
" u'DENSE1/BatchNorm/cond/switch_t',\n",
" u'DENSE1/BatchNorm/cond/Const_1',\n",
" u'DENSE1/BatchNorm/cond/Const',\n",
" u'CONV6/BatchNorm/cond_1/pred_id',\n",
" u'CONV6/BatchNorm/cond_1/AssignMovingAvg_1/Switch',\n",
" u'CONV6/BatchNorm/cond_1/AssignMovingAvg_1/sub/Switch',\n",
" u'CONV6/BatchNorm/cond_1/AssignMovingAvg/Switch',\n",
" u'CONV6/BatchNorm/cond_1/AssignMovingAvg/sub/Switch',\n",
" u'CONV6/BatchNorm/cond_1/Switch',\n",
" u'CONV6/BatchNorm/cond_1/switch_f',\n",
" u'CONV6/BatchNorm/cond_1/switch_t',\n",
" u'CONV6/BatchNorm/cond_1/AssignMovingAvg_1/decay',\n",
" u'CONV6/BatchNorm/cond_1/AssignMovingAvg/decay',\n",
" u'CONV6/BatchNorm/cond/pred_id',\n",
" u'gradients/Switch_21',\n",
" u'gradients/zeros_21/Const',\n",
" u'gradients/Shape_22',\n",
" u'gradients/zeros_21',\n",
" u'gradients/Switch_19',\n",
" u'gradients/zeros_19/Const',\n",
" u'gradients/Shape_20',\n",
" u'gradients/zeros_19',\n",
" u'CONV6/BatchNorm/cond/FusedBatchNorm_1/Switch_4',\n",
" u'CONV6/BatchNorm/cond/FusedBatchNorm_1/Switch_3',\n",
" u'CONV6/BatchNorm/cond/FusedBatchNorm_1/Switch_2',\n",
" u'CONV6/BatchNorm/cond/FusedBatchNorm_1/Switch_1',\n",
" u'CONV6/BatchNorm/cond/FusedBatchNorm/Switch_2',\n",
" u'CONV6/BatchNorm/cond/FusedBatchNorm/Switch_1',\n",
" u'CONV6/BatchNorm/cond/Switch',\n",
" u'CONV6/BatchNorm/cond/switch_f',\n",
" u'CONV6/BatchNorm/cond/switch_t',\n",
" u'CONV6/BatchNorm/cond/Const_1',\n",
" u'CONV6/BatchNorm/cond/Const',\n",
" u'CONV5/BatchNorm/cond_1/pred_id',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg_1/Switch',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg_1/sub/Switch',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg/Switch',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg/sub/Switch',\n",
" u'CONV5/BatchNorm/cond_1/Switch',\n",
" u'CONV5/BatchNorm/cond_1/switch_f',\n",
" u'CONV5/BatchNorm/cond_1/switch_t',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg_1/decay',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg/decay',\n",
" u'CONV5/BatchNorm/cond/pred_id',\n",
" u'gradients/Switch_27',\n",
" u'gradients/zeros_27/Const',\n",
" u'gradients/Shape_28',\n",
" u'gradients/zeros_27',\n",
" u'gradients/Switch_25',\n",
" u'gradients/zeros_25/Const',\n",
" u'gradients/Shape_26',\n",
" u'gradients/zeros_25',\n",
" u'CONV5/BatchNorm/cond/FusedBatchNorm_1/Switch_4',\n",
" u'CONV5/BatchNorm/cond/FusedBatchNorm_1/Switch_3',\n",
" u'CONV5/BatchNorm/cond/FusedBatchNorm_1/Switch_2',\n",
" u'CONV5/BatchNorm/cond/FusedBatchNorm_1/Switch_1',\n",
" u'CONV5/BatchNorm/cond/FusedBatchNorm/Switch_2',\n",
" u'CONV5/BatchNorm/cond/FusedBatchNorm/Switch_1',\n",
" u'CONV5/BatchNorm/cond/Switch',\n",
" u'CONV5/BatchNorm/cond/switch_f',\n",
" u'CONV5/BatchNorm/cond/switch_t',\n",
" u'CONV5/BatchNorm/cond/Const_1',\n",
" u'CONV5/BatchNorm/cond/Const',\n",
" u'CONV4/BatchNorm/cond_1/pred_id',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg_1/Switch',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg_1/sub/Switch',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg/Switch',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg/sub/Switch',\n",
" u'CONV4/BatchNorm/cond_1/Switch',\n",
" u'CONV4/BatchNorm/cond_1/switch_f',\n",
" u'CONV4/BatchNorm/cond_1/switch_t',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg_1/decay',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg/decay',\n",
" u'CONV4/BatchNorm/cond/pred_id',\n",
" u'gradients/Switch_33',\n",
" u'gradients/zeros_33/Const',\n",
" u'gradients/Shape_34',\n",
" u'gradients/zeros_33',\n",
" u'gradients/Switch_31',\n",
" u'gradients/zeros_31/Const',\n",
" u'gradients/Shape_32',\n",
" u'gradients/zeros_31',\n",
" u'CONV4/BatchNorm/cond/FusedBatchNorm_1/Switch_4',\n",
" u'CONV4/BatchNorm/cond/FusedBatchNorm_1/Switch_3',\n",
" u'CONV4/BatchNorm/cond/FusedBatchNorm_1/Switch_2',\n",
" u'CONV4/BatchNorm/cond/FusedBatchNorm_1/Switch_1',\n",
" u'CONV4/BatchNorm/cond/FusedBatchNorm/Switch_2',\n",
" u'CONV4/BatchNorm/cond/FusedBatchNorm/Switch_1',\n",
" u'CONV4/BatchNorm/cond/Switch',\n",
" u'CONV4/BatchNorm/cond/switch_f',\n",
" u'CONV4/BatchNorm/cond/switch_t',\n",
" u'CONV4/BatchNorm/cond/Const_1',\n",
" u'CONV4/BatchNorm/cond/Const',\n",
" u'CONV3/BatchNorm/cond_1/pred_id',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg_1/Switch',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg_1/sub/Switch',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg/Switch',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg/sub/Switch',\n",
" u'CONV3/BatchNorm/cond_1/Switch',\n",
" u'CONV3/BatchNorm/cond_1/switch_f',\n",
" u'CONV3/BatchNorm/cond_1/switch_t',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg_1/decay',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg/decay',\n",
" u'CONV3/BatchNorm/cond/pred_id',\n",
" u'gradients/Switch_39',\n",
" u'gradients/zeros_39/Const',\n",
" u'gradients/Shape_40',\n",
" u'gradients/zeros_39',\n",
" u'gradients/Switch_37',\n",
" u'gradients/zeros_37/Const',\n",
" u'gradients/Shape_38',\n",
" u'gradients/zeros_37',\n",
" u'CONV3/BatchNorm/cond/FusedBatchNorm_1/Switch_4',\n",
" u'CONV3/BatchNorm/cond/FusedBatchNorm_1/Switch_3',\n",
" u'CONV3/BatchNorm/cond/FusedBatchNorm_1/Switch_2',\n",
" u'CONV3/BatchNorm/cond/FusedBatchNorm_1/Switch_1',\n",
" u'CONV3/BatchNorm/cond/FusedBatchNorm/Switch_2',\n",
" u'CONV3/BatchNorm/cond/FusedBatchNorm/Switch_1',\n",
" u'CONV3/BatchNorm/cond/Switch',\n",
" u'CONV3/BatchNorm/cond/switch_f',\n",
" u'CONV3/BatchNorm/cond/switch_t',\n",
" u'CONV3/BatchNorm/cond/Const_1',\n",
" u'CONV3/BatchNorm/cond/Const',\n",
" u'CONV2/BatchNorm/cond_1/pred_id',\n",
" u'CONV2/BatchNorm/cond_1/AssignMovingAvg_1/Switch',\n",
" u'CONV2/BatchNorm/cond_1/AssignMovingAvg_1/sub/Switch',\n",
" u'CONV2/BatchNorm/cond_1/AssignMovingAvg/Switch',\n",
" u'CONV2/BatchNorm/cond_1/AssignMovingAvg/sub/Switch',\n",
" u'CONV2/BatchNorm/cond_1/Switch',\n",
" u'CONV2/BatchNorm/cond_1/switch_f',\n",
" u'CONV2/BatchNorm/cond_1/switch_t',\n",
" u'CONV2/BatchNorm/cond_1/AssignMovingAvg_1/decay',\n",
" u'CONV2/BatchNorm/cond_1/AssignMovingAvg/decay',\n",
" u'CONV2/BatchNorm/cond/pred_id',\n",
" u'CONV2/BatchNorm/cond/FusedBatchNorm_1/Switch_4',\n",
" u'CONV2/BatchNorm/cond/FusedBatchNorm_1/Switch_3',\n",
" u'CONV2/BatchNorm/cond/FusedBatchNorm_1/Switch_2',\n",
" u'CONV2/BatchNorm/cond/FusedBatchNorm_1/Switch_1',\n",
" u'CONV2/BatchNorm/cond/FusedBatchNorm/Switch_2',\n",
" u'CONV2/BatchNorm/cond/FusedBatchNorm/Switch_1',\n",
" u'CONV2/BatchNorm/cond/Switch',\n",
" u'CONV2/BatchNorm/cond/switch_f',\n",
" u'CONV2/BatchNorm/cond/switch_t',\n",
" u'CONV2/BatchNorm/cond/Const_1',\n",
" u'CONV2/BatchNorm/cond/Const',\n",
" u'CONV1/BatchNorm/cond_1/pred_id',\n",
" u'CONV1/BatchNorm/cond_1/AssignMovingAvg_1/Switch',\n",
" u'CONV1/BatchNorm/cond_1/AssignMovingAvg_1/sub/Switch',\n",
" u'CONV1/BatchNorm/cond_1/AssignMovingAvg/Switch',\n",
" u'CONV1/BatchNorm/cond_1/AssignMovingAvg/sub/Switch',\n",
" u'CONV1/BatchNorm/cond_1/Switch',\n",
" u'CONV1/BatchNorm/cond_1/switch_f',\n",
" u'CONV1/BatchNorm/cond_1/switch_t',\n",
" u'CONV1/BatchNorm/cond_1/AssignMovingAvg_1/decay',\n",
" u'CONV1/BatchNorm/cond_1/AssignMovingAvg/decay',\n",
" u'CONV1/BatchNorm/cond/pred_id',\n",
" u'CONV1/BatchNorm/cond/FusedBatchNorm_1/Switch_4',\n",
" u'CONV1/BatchNorm/cond/FusedBatchNorm_1/Switch_3',\n",
" u'CONV1/BatchNorm/cond/FusedBatchNorm_1/Switch_2',\n",
" u'CONV1/BatchNorm/cond/FusedBatchNorm_1/Switch_1',\n",
" u'CONV1/BatchNorm/cond/FusedBatchNorm/Switch_2',\n",
" u'CONV1/BatchNorm/cond/FusedBatchNorm/Switch_1',\n",
" u'CONV1/BatchNorm/cond/Switch',\n",
" u'CONV1/BatchNorm/cond/switch_f',\n",
" u'CONV1/BatchNorm/cond/switch_t',\n",
" u'CONV1/BatchNorm/cond/Const_1',\n",
" u'CONV1/BatchNorm/cond/Const',\n",
" u'Placeholder_1',\n",
" u'ArgMax_1',\n",
" u'softmax_cross_entropy_loss/labels_stop_gradient',\n",
" u'softmax_cross_entropy_loss/xentropy/Shape_2',\n",
" u'softmax_cross_entropy_loss/xentropy/Slice_1',\n",
" u'softmax_cross_entropy_loss/xentropy/concat_1',\n",
" u'softmax_cross_entropy_loss/xentropy/Reshape_1',\n",
" u'Placeholder',\n",
" u'Reshape',\n",
" u'gradients/CONV3/Conv2D_grad/ShapeN',\n",
" u'CONV3/Conv2D',\n",
" u'gradients/Switch_38',\n",
" u'gradients/zeros_38/Const',\n",
" u'gradients/Shape_39',\n",
" u'gradients/zeros_38',\n",
" u'gradients/Switch_36',\n",
" u'gradients/zeros_36/Const',\n",
" u'gradients/Shape_37',\n",
" u'gradients/zeros_36',\n",
" u'CONV3/BatchNorm/cond/FusedBatchNorm_1/Switch',\n",
" u'CONV3/BatchNorm/cond/FusedBatchNorm_1',\n",
" u'gradients/zeros_like_44',\n",
" u'gradients/zeros_like_43',\n",
" u'gradients/zeros_like_42',\n",
" u'gradients/zeros_like_41',\n",
" u'CONV3/BatchNorm/cond/FusedBatchNorm/Switch',\n",
" u'CONV3/BatchNorm/cond/FusedBatchNorm',\n",
" u'gradients/zeros_like_48',\n",
" u'gradients/zeros_like_47',\n",
" u'gradients/zeros_like_46',\n",
" u'gradients/zeros_like_45',\n",
" u'CONV3/BatchNorm/cond/Merge_2',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg_1/sub/Switch_1',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg_1/sub',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg_1/mul',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg_1',\n",
" u'CONV3/BatchNorm/cond/Merge_1',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg/sub/Switch_1',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg/sub',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg/mul',\n",
" u'CONV3/BatchNorm/cond_1/AssignMovingAvg',\n",
" u'CONV3/BatchNorm/cond/Merge',\n",
" u'gradients/Switch_35',\n",
" u'gradients/zeros_35/Const',\n",
" u'gradients/Shape_36',\n",
" u'gradients/zeros_35',\n",
" u'gradients/Switch_34',\n",
" u'gradients/zeros_34/Const',\n",
" u'gradients/Shape_35',\n",
" u'gradients/zeros_34',\n",
" u'CONV3/BatchNorm/cond_1/Switch_1',\n",
" u'CONV3/BatchNorm/cond_1/Identity/Switch',\n",
" u'CONV3/BatchNorm/cond_1/Identity',\n",
" u'CONV3/BatchNorm/cond_1/Merge',\n",
" u'gradients/CONV3/lrelu/Abs_grad/Sign',\n",
" u'gradients/CONV3/lrelu/mul_grad/Shape_1',\n",
" u'gradients/CONV3/lrelu/mul_grad/BroadcastGradientArgs',\n",
" u'CONV3/lrelu/Abs',\n",
" u'gradients/CONV3/lrelu/mul_1_grad/Shape_1',\n",
" u'gradients/CONV3/lrelu/mul_1_grad/BroadcastGradientArgs',\n",
" u'CONV3/lrelu/mul_1',\n",
" u'gradients/CONV3/lrelu/add_grad/Shape_1',\n",
" u'CONV3/lrelu/mul',\n",
" u'gradients/CONV3/lrelu/add_grad/Shape',\n",
" u'gradients/CONV3/lrelu/add_grad/BroadcastGradientArgs',\n",
" u'CONV3/lrelu/add',\n",
" u'gradients/CONV4/Conv2D_grad/ShapeN',\n",
" u'CONV4/Conv2D',\n",
" u'gradients/Switch_32',\n",
" u'gradients/zeros_32/Const',\n",
" u'gradients/Shape_33',\n",
" u'gradients/zeros_32',\n",
" u'gradients/Switch_30',\n",
" u'gradients/zeros_30/Const',\n",
" u'gradients/Shape_31',\n",
" u'gradients/zeros_30',\n",
" u'CONV4/BatchNorm/cond/FusedBatchNorm_1/Switch',\n",
" u'CONV4/BatchNorm/cond/FusedBatchNorm_1',\n",
" u'gradients/zeros_like_36',\n",
" u'gradients/zeros_like_35',\n",
" u'gradients/zeros_like_34',\n",
" u'gradients/zeros_like_33',\n",
" u'CONV4/BatchNorm/cond/FusedBatchNorm/Switch',\n",
" u'CONV4/BatchNorm/cond/FusedBatchNorm',\n",
" u'gradients/zeros_like_40',\n",
" u'gradients/zeros_like_39',\n",
" u'gradients/zeros_like_38',\n",
" u'gradients/zeros_like_37',\n",
" u'CONV4/BatchNorm/cond/Merge_2',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg_1/sub/Switch_1',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg_1/sub',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg_1/mul',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg_1',\n",
" u'CONV4/BatchNorm/cond/Merge_1',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg/sub/Switch_1',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg/sub',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg/mul',\n",
" u'CONV4/BatchNorm/cond_1/AssignMovingAvg',\n",
" u'CONV4/BatchNorm/cond/Merge',\n",
" u'gradients/Switch_29',\n",
" u'gradients/zeros_29/Const',\n",
" u'gradients/Shape_30',\n",
" u'gradients/zeros_29',\n",
" u'gradients/Switch_28',\n",
" u'gradients/zeros_28/Const',\n",
" u'gradients/Shape_29',\n",
" u'gradients/zeros_28',\n",
" u'CONV4/BatchNorm/cond_1/Switch_1',\n",
" u'CONV4/BatchNorm/cond_1/Identity/Switch',\n",
" u'CONV4/BatchNorm/cond_1/Identity',\n",
" u'CONV4/BatchNorm/cond_1/Merge',\n",
" u'gradients/CONV4/lrelu/Abs_grad/Sign',\n",
" u'gradients/CONV4/lrelu/mul_grad/Shape_1',\n",
" u'gradients/CONV4/lrelu/mul_grad/BroadcastGradientArgs',\n",
" u'CONV4/lrelu/Abs',\n",
" u'gradients/CONV4/lrelu/mul_1_grad/Shape_1',\n",
" u'gradients/CONV4/lrelu/mul_1_grad/BroadcastGradientArgs',\n",
" u'CONV4/lrelu/mul_1',\n",
" u'gradients/CONV4/lrelu/add_grad/Shape_1',\n",
" u'CONV4/lrelu/mul',\n",
" u'gradients/CONV4/lrelu/add_grad/Shape',\n",
" u'gradients/CONV4/lrelu/add_grad/BroadcastGradientArgs',\n",
" u'CONV4/lrelu/add',\n",
" u'POOL2/MaxPool',\n",
" u'gradients/CONV5/Conv2D_grad/ShapeN',\n",
" u'CONV5/Conv2D',\n",
" u'gradients/Switch_26',\n",
" u'gradients/zeros_26/Const',\n",
" u'gradients/Shape_27',\n",
" u'gradients/zeros_26',\n",
" u'gradients/Switch_24',\n",
" u'gradients/zeros_24/Const',\n",
" u'gradients/Shape_25',\n",
" u'gradients/zeros_24',\n",
" u'CONV5/BatchNorm/cond/FusedBatchNorm_1/Switch',\n",
" u'CONV5/BatchNorm/cond/FusedBatchNorm_1',\n",
" u'gradients/zeros_like_28',\n",
" u'gradients/zeros_like_27',\n",
" u'gradients/zeros_like_26',\n",
" u'gradients/zeros_like_25',\n",
" u'CONV5/BatchNorm/cond/FusedBatchNorm/Switch',\n",
" u'CONV5/BatchNorm/cond/FusedBatchNorm',\n",
" u'gradients/zeros_like_32',\n",
" u'gradients/zeros_like_31',\n",
" u'gradients/zeros_like_30',\n",
" u'gradients/zeros_like_29',\n",
" u'CONV5/BatchNorm/cond/Merge_2',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg_1/sub/Switch_1',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg_1/sub',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg_1/mul',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg_1',\n",
" u'CONV5/BatchNorm/cond/Merge_1',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg/sub/Switch_1',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg/sub',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg/mul',\n",
" u'CONV5/BatchNorm/cond_1/AssignMovingAvg',\n",
" u'CONV5/BatchNorm/cond/Merge',\n",
" u'gradients/Switch_23',\n",
" u'gradients/zeros_23/Const',\n",
" u'gradients/Shape_24',\n",
" u'gradients/zeros_23',\n",
" u'gradients/Switch_22',\n",
" u'gradients/zeros_22/Const',\n",
" u'gradients/Shape_23',\n",
" u'gradients/zeros_22',\n",
" u'CONV5/BatchNorm/cond_1/Switch_1',\n",
" u'CONV5/BatchNorm/cond_1/Identity/Switch',\n",
" u'CONV5/BatchNorm/cond_1/Identity',\n",
" u'CONV5/BatchNorm/cond_1/Merge',\n",
" u'gradients/CONV5/lrelu/Abs_grad/Sign',\n",
" u'gradients/CONV5/lrelu/mul_grad/Shape_1',\n",
" u'gradients/CONV5/lrelu/mul_grad/BroadcastGradientArgs',\n",
" u'CONV5/lrelu/Abs',\n",
" u'gradients/CONV5/lrelu/mul_1_grad/Shape_1',\n",
" u'gradients/CONV5/lrelu/mul_1_grad/BroadcastGradientArgs',\n",
" u'CONV5/lrelu/mul_1',\n",
" u'gradients/CONV5/lrelu/add_grad/Shape_1',\n",
" u'CONV5/lrelu/mul',\n",
" u'gradients/CONV5/lrelu/add_grad/Shape',\n",
" u'gradients/CONV5/lrelu/add_grad/BroadcastGradientArgs',\n",
" u'CONV5/lrelu/add',\n",
" u'gradients/CONV6/Conv2D_grad/ShapeN',\n",
" u'CONV6/Conv2D',\n",
" u'gradients/Switch_20',\n",
" u'gradients/zeros_20/Const',\n",
" u'gradients/Shape_21',\n",
" u'gradients/zeros_20',\n",
" ...]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"[n.name for n in tf.get_default_graph().as_graph_def().node]"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"X = graph.get_tensor_by_name(\"Placeholder:0\")\n",
"y = graph.get_tensor_by_name(\"Placeholder_1:0\")\n",
"is_training = graph.get_tensor_by_name(\"Placeholder_2:0\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"PERFORMANCE OF PRE-TRAINED NET IS: 0.623046875\n"
]
}
],
"source": [
"ss = graph.get_tensor_by_name(\"LOGIT/Softmax:0\")\n",
"oo = sess.run(ss, feed_dict={X: whiten(xtest[:512]), y: ytest[:512], is_training: False})\n",
"performance = np.equal(np.argmax(oo, 1), np.argmax(ytest[:512], 1)).sum() / 512.\n",
"\n",
"print \"PERFORMANCE OF PRE-TRAINED NET IS:\", performance"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Get the last CONV layer"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"conv6 = graph.get_tensor_by_name(\"CONV6/lrelu/add:0\")\n",
"conv6 = tf.stop_gradient(conv6)\n",
"conv6_shape = conv6.get_shape().as_list()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[None, 24, 24, 128]"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"conv6_shape"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Add Global Average Pooling + FC layer\n",
"* [Learning Deep Features for Discriminative Localization](https://arxiv.org/abs/1512.04150)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"def lrelu(x, leak=.2, scope=\"lrelu\"):\n",
" with tf.variable_scope(scope):\n",
" f1 = (1-leak) / 2.\n",
" f2 = (1+leak) / 2.\n",
" return f1*x + f2*tf.abs(x)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"def add_gap(is_training):\n",
" batchnorm_params = {\"is_training\": is_training, \"decay\": 0.9, \"updates_collections\": None}\n",
"\n",
" conv_last = slim.conv2d(conv6, 1024, [3,3], padding=\"SAME\",\n",
" normalizer_fn=slim.batch_norm,\n",
" normalizer_params=batchnorm_params,\n",
" activation_fn=lrelu,\n",
" weights_initializer=tf.truncated_normal_initializer(stddev=.05),\n",
" scope=\"CONV_LAST\")\n",
" gap = slim.avg_pool2d(conv_last, conv6_shape[1:3], scope=\"GAP\")\n",
" gap_flat = slim.flatten(gap, scope=\"GAP_FLAT\")\n",
" logit = slim.fully_connected(gap_flat, 7, \n",
" activation_fn=tf.nn.softmax,\n",
" scope=\"WSL_LOGIT\")\n",
" return conv_last, gap, logit"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"conv_last, gap, logit = add_gap(is_training)\n",
"pred = tf.argmax(logit, 1)\n",
"truth = tf.argmax(y, 1)\n",
"acc = tf.reduce_mean(tf.cast(tf.equal(truth, pred), tf.float32))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"## Data preprocessing & augmentation"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"from scipy import ndimage"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"def augment_data(images):\n",
" images_r = images.reshape(-1, 48, 48)\n",
" images_aug = []\n",
" for img in images_r:\n",
" # FLIP\n",
" if np.random.randint(0, 2) == 1:\n",
" img_flipped = np.fliplr(img)\n",
" else:\n",
" img_flipped = img\n",
" # ROTATE\n",
" angle = np.random.randint(-10, 10)\n",
" img_rotated = ndimage.rotate(img_flipped, angle, reshape=False)\n",
" # scaled\n",
" scale = np.random.uniform(1., 1.1)\n",
" img_scaled = ndimage.zoom(img_rotated, scale)\n",
" center = img_scaled.shape[0] // 2\n",
" st = (center-24); en = (center+24)\n",
" img_scaled = img_scaled[st:en, st:en]\n",
" # SHIFT\n",
" shift = np.random.randint(-2, 2)\n",
" img_shifted = ndimage.shift(img_scaled, shift)\n",
" # add to the list\n",
" img_flattend = img_shifted.flatten()\n",
" images_aug.append(img_flattend)\n",
" images_aug = np.array(images_aug)\n",
" return images_aug"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"## Train CNN-GAP"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([ 5.74696866, 7.0324748 , 3.99173685, 5.90225564, 8.93950178,\n",
" 7.31082654, 65.7591623 ])"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"class_counts = np.unique(np.argmax(fer[\"ytrain\"], axis=1), return_counts=True)[1]\n",
"class_weights = (class_counts.sum() / class_counts.astype(float))\n",
"class_weights"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"weights = tf.placeholder(tf.float32, [None])\n",
"loss = tf.losses.softmax_cross_entropy(y, logit, weights=weights, scope=\"LOSS_WSL\")\n",
"#graph.get_tensor_by_name(\"softmax_cross_entropy_loss/value:0\")"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"learning_rate = .005\n",
"global_step = tf.Variable(0, trainable=False)\n",
"decay_steps = 50.\n",
"decay_rate = .93\n",
"learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, \n",
" decay_steps, decay_rate, staircase=True)\n",
"\n",
"#optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate, momentum=.9, use_nesterov=True).minimize(loss)\n",
"optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, epsilon=.0001, name=\"Adam_wsl\").minimize(loss, global_step=global_step)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"INITIALIZED\n"
]
}
],
"source": [
"uninitialized = [v for v in tf.global_variables() if v.name.split(':')[0] in sess.run(tf.report_uninitialized_variables())]\n",
"sess.run(tf.variables_initializer(uninitialized))\n",
"print \"VARIABLES INITIALIZED\""
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFEdJREFUeJzt3Xu0HeVdh/HnQLiVFpJwMGISDUJWWVQWt0NIelu0sRCwbbIsUBBLgtGsanpTujS1VRBoF1ixLasWTRskIJVGWkxEJE25WKvcwi3hUpqUEpMsIBxyaQEpBl7/eH/HM9mcTebc9j6H9/msNWu/8+6Z2e/sPWe+M+/M3qcjpYQkqTx7tLsBkqT2MAAkqVAGgCQVygCQpEIZAJJUKANAkgplAEhSoQwASSqUASBJhRrT7ga8nmeffTZt2LCh3c2QpFGlq6urGzh4d9ON6ADYsGEDJ5xwQrubIUmjSkqp1pGzXUCSVCgDQJIKZQBIUqEMAEkqlAEgSYUyACSpUAaAJBXKAJCkQhkAklSoEf1NYI18l6+9s91N2MX5R81odxOkUcMzAEkqlAEgSYUyACSpUAaAJBXKAJCkQtUNgLHADcAPgceAGcB4YBWwLh7HxbQdwBXAemANcFxlOXNj+nVRliS1Sd0A+ApwC3AEcDQ5BBYBtwJT43FRTHtq1E0FFgBXRv144ALgRGBalHtCQ5LUYnUC4EDg3cCSGH8Z2A7MBpZG3VJgTpRnA9cACbiLfPZwCHAK+UxhK7AtyrMGvQaSpAGpEwCHAs8Cfw88AHwD2B+YADwV0zwd4wATgY2V+TdFXbP6RguA1cDqzs7OWishSeq/OgEwhtyPfyVwLPACvd09PVIMQ2Ex0AV0dXd3D9EiJUmN6gTAphjujvEbyIHwDLlrh3jcEuXNwOTK/JOirlm9JKkN6gTA0+Sum7fG+EzgUWAFvXfyzAWWR3kFcC75bqDpwA5yV9FK4GTyhd9xUV456DWQJA1I3R+D+zhwHbA38ARwHjk8lgHzgQ3AmTHtzcBp5NtAX4xpIV/8vRi4N8YvijpJUhvUDYAHyf3yjWb2UZeAhU2Wc1UMkqQ285vAklQoA0CSCmUASFKhDABJKpQBIEmFMgAkqVAGgCQVygCQpEIZAJJUKANAkgplAEhSoQwASSqUASBJhTIAJKlQBoAkFcoAkKRCGQCSVCgDQJIKZQBIUqEMAEkqlAEgSYUyACSpUAaAJBXKAJCkQtUNgCeBtcCDwOqoGw+sAtbF47io7wCuANYDa4DjKsuZG9Ovi7IkqU36cwbwHuAYoCvGFwG3AlPjcVHUnxp1U4EFwJVRPx64ADgRmBblntCQJLXYYLqAZgNLo7wUmFOpvwZIwF3AWOAQ4BTymcJWYFuUZw3i9SVJg1A3ABLwXeA+8lE9wATgqSg/HeMAE4GNlXk3RV2z+kYLyN1Mqzs7O2s2T5LUX2NqTvdOYDPwC+Qj9x82PJ9iGAqLY6C7u3uolilJalD3DGBzPG4BbiT34T9D7tohHrdUpp1cmXdS1DWrlyS1QZ0A2B94S6V8MvAwsILeO3nmAsujvAI4l3w30HRgB7mraGXMOy6Gk6NOktQGdbqAJpCP+num/yZwC3AvsAyYD2wAzoxpbgZOI98G+iJwXtRvBS6O+QAuijpJUhvUCYAngKP7qH8OmNlHfQIWNlnWVTFIktrMbwJLUqEMAEkqlAEgSYUyACSpUAaAJBXKAJCkQhkAklQoA0CSCmUASFKhDABJKpQBIEmFMgAkqVAGgCQVygCQpEIZAJJUqLr/E3hUunztne1uwv87/6gZ7W6CJO3CMwBJKpQBIEmFMgAkqVBv6GsAUl9G0rUh8PqQ2sczAEkqlAEgSYUyACSpUAaAJBWqPwGwJ/AAcFOMHwrcDawHvgXsHfX7xPj6eH5KZRmfifrHgVMG2mhJ0uD1JwA+CTxWGb8M+BJwOLANmB/182P88Hj+sqg/EjgLeBswC/gaOVQkSW1QNwAmAb8BfCPGO4D3AjfE+FJgTpRnxzjx/MyYfjZwPfBz4CfkM4Fpg2i7JGkQ6gbAl4E/Bl6N8YOA7cDOGN8ETIzyRGBjlHcCO2L6an3jPFULgNXA6s7OzprNkyT1V50AeD+wBbhvmNvSYzHQBXR1d3e36CUlqTx1vgn8DuCDwGnAvsABwFeAsTH/TnIX0eaYfjMwmXyEPwY4EHiuUt+jOo8kqcXqnAF8hryznkK+iHsbcA5wO3B6TDMXWB7lFTFOPH8bkKL+LPJdQocCU4F7BrsCkqSBGcxvAf0J+aLuJeTbQ5dE/RLgWvJF3q3knT7AI8Ay4FHyWcNC4JVBvL4kaRD6GwB3xADwBH3fxfMScEaT+T8fgySpzfwmsCQVygCQpEIZAJJUKANAkgplAEhSoQwASSqUASBJhTIAJKlQg/kmsIbY5WvvbHcTdnH+UTPa3QRJw8gzAEkqlAEgSYUyACSpUAaAJBXKi8CShpw3NIwOngFIUqEMAEkqlAEgSYUyACSpUAaAJBXKAJCkQhkAklQoA0CSCmUASFKh6gTAvsA9wEPAI8BfRP2hwN3AeuBbwN5Rv0+Mr4/np1SW9Zmofxw4ZXBNlyQNRp0A+DnwXuBo4BhgFjAduAz4EnA4sA2YH9PPj/HD4/nLov5I4CzgbbGMrwF7DsVKSJL6r04AJOD5KO8VQyKHwg1RvxSYE+XZMU48PxPoiPrryYHyE/KZwLTBNV+SNFB1rwHsCTwIbAFWAT8GtgM74/lNwMQoTwQ2RnknsAM4qKG+cR5JUovVDYBXyN0/k8hH7UcMW4tgAbAaWN3Z2TmMLyNJZevvXUDbgduBGcBYen9OehKwOcqbgclRHgMcCDzXUN84T9VioAvo6u7u7mfzJEl11QmAg8k7e4D9gPcBj5GD4PSonwssj/KKGCeev418zWAF+SLwPuQ7iKaS7y6SJLVBnX8Icwj5ou6e5MBYBtwEPEq+qHsJ8ACwJKZfAlxLvsi7lbzTh3wL6bKYbyewkNy1JElqgzoBsAY4to/6J+j7Lp6XgDOaLOvzMUiS2sxvAktSofyfwNIoMJL+x67/X/eNwzMASSqUASBJhTIAJKlQBoAkFcoAkKRCGQCSVCgDQJIKZQBIUqEMAEkqlAEgSYUyACSpUAaAJBXKAJCkQhkAklQoA0CSCmUASFKhDABJKpQBIEmFMgAkqVAGgCQVygCQpEIZAJJUqDoBMBm4HXgUeAT4ZNSPB1YB6+JxXNR3AFcA64E1wHGVZc2N6ddFWZLUJnUCYCdwPnAkMB1YGOVFwK3A1HhcFNOfGnVTgQXAlVE/HrgAOBGYFuWe0JAktVidAHgKuD/KPwMeAyYCs4GlUb8UmBPl2cA1QALuAsYChwCnkM8UtgLbojxr0GsgSRqQ/l4DmAIcC9wNTCCHA8DTMQ45HDZW5tkUdc3qJUltMKYf074Z+DbwKeCnDc+lGIbCghjo7OwcokVKkhrVPQPYi7zzvw74TtQ9Q+7aIR63RHkz+cJxj0lR16y+0WKgC+jq7u6u2TxJUn/VCYAOYAm57/+vK/Ur6L2TZy6wvFJ/bsw3HdhB7ipaCZxMvvA7LsorB9d8SdJA1ekCegfwEWAt8GDU/SlwKbAMmA9sAM6M524GTiPfBvoicF7UbwUuBu6N8YuiTpLUBnUC4Afko/m+zOyjLpFvFe3LVTFIktrMbwJLUqEMAEkqlAEgSYUyACSpUAaAJBXKAJCkQhkAklQoA0CSCmUASFKhDABJKpQBIEmFMgAkqVAGgCQVygCQpEIZAJJUKANAkgplAEhSoQwASSqUASBJhTIAJKlQBoAkFcoAkKRCGQCSVCgDQJIKVScArgK2AA9X6sYDq4B18Tgu6juAK4D1wBrguMo8c2P6dVGWJLVRnQC4GpjVULcIuBWYGo+Lov7UqJsKLACujPrxwAXAicC0KI9DktQ2dQLg+8DWhrrZwNIoLwXmVOqvARJwFzAWOAQ4hXymsBXYFuXGUJEktdBArwFMAJ6K8tMxDjAR2FiZblPUNauXJLXJmCFYRophqCyIgc7OziFcrCSpaqBnAM+Qu3aIxy1R3gxMrkw3Keqa1fdlMdAFdHV3dw+weZKk3RloAKyg906eucDySv255LuBpgM7yF1FK4GTyRd+x0V55QBfW5I0BOp0Af0jcBLQSe67vwC4FFgGzAc2AGfGtDcDp5FvA30ROC/qtwIXA/fG+EW89sKyJKmF6gTA2U3qZ/ZRl4CFTaa/KgZJ0gjgN4ElqVAGgCQVygCQpEIZAJJUKANAkgplAEhSoQwASSqUASBJhTIAJKlQBoAkFcoAkKRCGQCSVCgDQJIKZQBIUqEMAEkq1FD8T2BJGvUuX3tnu5uwi/OPmjHsr+EZgCQVygCQpEIZAJJUKANAkgplAEhSoQwASSqUASBJhTIAJKlQ7QiAWcDjwHpgURteX5JE6wNgT+BvgFOBI4Gz41GS1GKtDoBp5CP/J4CXgeuB2S1ugySJ1gfARGBjZXxT1EmSWqwjpdTK1zudfA3gd2P8I8CJwMcq0yyIAeCt5OsF7dQJdLe5Df1lm1tjtLV5tLUXbPNA/Qpw8O4mavWvgW4GJlfGJ0Vd1eIYRorVQFe7G9FPtrk1RlubR1t7wTYPq1Z3Ad0LTAUOBfYGzgJWtLgNkiRafwawk9zds5J8R9BVwCMtboMkifb8Q5ibYxgtRlJ3VF22uTVGW5tHW3vBNg+rVl8EliSNEP4UhCQVygCoZwrwWwOc9/khbAfAJ4DHgOuGeLlDYQrwcLsb0WZTGJ3vwc3A2HY3YoS5EPg0cBHw6y14vTm0+JcRDIB6ptA8AFp9HeUPgPcB5wxiGe249qPWqvsZd5D3A6cB24evOcOiVdvxnwPfa8HrGABDbAr5aPnr5LuNvgvsBxwG3ALcB/wHcERMfzX5y2o9eo7eLwXeBTwI/CEwj3z76m3ArcCb4/F+YC3D9/MWfwv8KvBvwGfJd1HdAzxQec0p5HW6P4a3R/1JUb8CeHSY2gf57q7G9/v3yLcAPwR8G3hTTHs1eZ1WAz8C3h/184DlwB3AOuCCqL8I+FTltT4PfHI4VgLYH/jXaPPDwIfJO4J7Y3wxeecJcHxM9xCwsAXteJL8ZSPI95vfEeULgWuB/4zHefT9Pk4hf8Hymljm5Moy+3o9yOv47+S/mZXAIQNYl3+O+R+h98uez5M/x4eAu4AJUX9YjK8FLqH3b/Ekdt2Oh3qb+Cx5W/wB+YuosOt+4dJ43TXAX9Vo602VZX+V/Jn0tZy3Ax8Evkjezxw2iHWoL6X0Rh6mpJR2ppSOifFlKaXfTindmlKaGnUnppRui/LVKaXTK/M/H48npZRuqtTPSyltSimNj/ExKaUDotyZUlqfUupoWMZQDU/Ga3wh1oWU0tiU0o9SSvunlN6UUto36qemlFZX1uGFlNKhbXi/D6pMc0lK6eOV9/uWlNIe0dZN0fZ5KaWnYr79UkoPp5S6Yvn3x7x7pJR+3LDsoRw+lFL6emX8wMrnTUrp2pTSB6K8JqX07ih/Mdo7nO3o2QaI9+WOKF+YUrov3rOe7bTZ+/hqSml6Zbk9y+zr9fZKKf1XSungqPtwSumqAaxLz/vX05aDUtbzPv5lSulzUb4ppXR2lD+adv1brG7HQ7lNHJ9SWpvy39ABKf8dfzr17hcOSik9nnr/tsfWaGt1v/HV+EyaLafndYZje+5zeKOfAQD8hJyokI8+ppDT9p+i/u8Y2NHMKmBrlDuAL5DT/Hvk3zea0GS+oXIy+ee0HyQf4e0L/DKwF/kIfC15HaunlPeQ34/h1Nf7/Wvko7a15K6rt1WmXwa8Sj5CfYLes7FVwHPA/wDfAd5JPkp9DjiWvP4PxPhwWEvuaruMfPa3A3gPcHc8995Yj7ExfD/mu7YF7Xg9K8jvWY++3keADeSj1jqv91byZ7iK/Nl+jvwt/v76BL1H+pPJXwp9md6j5J7tBWAGefsF+GbDcqrb8ZMM3TbxLuBG4EXgp7z2S6o7gJeAJcBvxnS7a2tfmi2n5UroC/55pfwKece8HTimj2l30tsttgf528rNvFApn0P+3Y3jgf8lb5T7Dqy5tXUAH+K1v5V0IfAMcDR5HV6qPPcCw6/x/d6PfAo9h/zHP498atyj8T7ktJv6b8QyfpHcBTZcfgQcR+4bv4TcxbeQ3OWykfw+D/dn3Kwd1e20sQ2Nn3Gz97HZttDX691I7raZ0c+2V51EvpA6g7zDu4Pc9v+ttOkV6u2TGtveqm1iJ/kXjWeSu4Q+Rj4QeL3pqwfZ+1bq+7OcYVPCGUCjn5KPHs6I8Q7yzhLyjvv4KH+QfDQN8DPgLa+zzAOBLeSN+T3kH2IabiuBj9PbD31spS1PkY+qP0Luk2+3t5DbtBevvXh9Bnk7PIx8faMn0N4HjCcHyBxyvzbkndEs4ATyezBcfom8o/oHcr/scVHfTb7m09MnvD2GniPrwVycr9uOJ+ndTj+0m/mbvY/9eb3HyQc4PQGwF7uexdVxILAtln0EMH03099F77qdtZtph2qb+D75PdqPvM1+oOH5N5PX42bytcCe/Uaztm4gn4HvQz5LnLmb5exuPzPkSjgD6Ms5wJXkU9m9yP+X4CFy18nyKN9C75HGGvLRyUPko9ltDcu7DvgX8unzauCHw9r67GLgy9G2Pcih9n7ga+QLreey6zq005+Ru06ejcfqRv7f5FP6A4CP0nvGcg95PSaRd0aro/5l4HbyTveVYWzzUeQd4KvkYP998s7hYeBp8sXgHueRjzwT+cL3cLdjP3L3wcX0XgBupq/3cUo/X+9lcuBdQd5xjSFve/35GZdbyJ/vY+RA6av7qepT0d7Pxryv1/U1VNvE/cC3yH/nW9j1M4a83S4nH8l3AH+0m7ZuJHdxPkz++3xgN8u5nrwP+gT5/f7xINalFr8JrHa6mtz/e0ND/TxyV8vHeK09yH+oZ5CvG6i5eTR/H0e6N5GvWyTyUfXZNL+7rt3bRH/aOqKUegag0elIcmDciDv/N7rjybdNdpCP7H+nyXQjYZuo29YRxzMASSpUiReBJUkYAJJULANAkgplAEhSoQwASSqUASBJhfo/ExN45o84UVEAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"ylabels = np.argmax(ytrain, 1)\n",
"plt.bar(range(7), np.unique(ylabels, return_counts=True)[1])\n",
"plt.xticks(range(7), [i.split(\"_\")[-1] for i in fer[\"categories\"]]);"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"saver = tf.train.Saver(max_to_keep=10)"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"205th: loss 12.05923, train_acc 0.530, test_acc 0.518\n",
"210th: loss 12.05393, train_acc 0.534, test_acc 0.521\n",
"215th: loss 12.05033, train_acc 0.534, test_acc 0.521\n",
"220th: loss 12.04805, train_acc 0.536, test_acc 0.523\n",
"225th: loss 12.04639, train_acc 0.533, test_acc 0.521\n",
"230th: loss 12.04842, train_acc 0.536, test_acc 0.520\n",
"235th: loss 12.04325, train_acc 0.537, test_acc 0.516\n",
"240th: loss 12.04510, train_acc 0.535, test_acc 0.518\n",
"245th: loss 12.04531, train_acc 0.535, test_acc 0.520\n",
"250th: loss 12.05065, train_acc 0.535, test_acc 0.523\n",
"255th: loss 12.04703, train_acc 0.536, test_acc 0.521\n",
"260th: loss 12.04881, train_acc 0.532, test_acc 0.520\n",
"265th: loss 12.04207, train_acc 0.536, test_acc 0.516\n",
"270th: loss 12.03846, train_acc 0.536, test_acc 0.516\n",
"275th: loss 12.02645, train_acc 0.539, test_acc 0.525\n",
"280th: loss 12.02972, train_acc 0.540, test_acc 0.523\n",
"285th: loss 12.03574, train_acc 0.537, test_acc 0.523\n",
"290th: loss 12.03511, train_acc 0.539, test_acc 0.523\n",
"295th: loss 12.03618, train_acc 0.535, test_acc 0.523\n",
"300th: loss 12.03244, train_acc 0.537, test_acc 0.521\n",
"305th: loss 12.03350, train_acc 0.537, test_acc 0.523\n",
"310th: loss 12.02948, train_acc 0.539, test_acc 0.520\n",
"315th: loss 12.03728, train_acc 0.536, test_acc 0.521\n",
"320th: loss 12.02750, train_acc 0.539, test_acc 0.521\n",
"325th: loss 12.03099, train_acc 0.537, test_acc 0.529\n",
"330th: loss 12.03196, train_acc 0.538, test_acc 0.523\n",
"335th: loss 12.02832, train_acc 0.537, test_acc 0.527\n",
"340th: loss 12.03112, train_acc 0.539, test_acc 0.529\n",
"345th: loss 12.02682, train_acc 0.538, test_acc 0.525\n",
"350th: loss 12.02889, train_acc 0.539, test_acc 0.527\n",
"355th: loss 12.03508, train_acc 0.537, test_acc 0.525\n",
"360th: loss 12.02948, train_acc 0.540, test_acc 0.525\n",
"365th: loss 12.02996, train_acc 0.538, test_acc 0.523\n",
"370th: loss 12.01945, train_acc 0.540, test_acc 0.523\n",
"375th: loss 12.02630, train_acc 0.537, test_acc 0.523\n",
"380th: loss 12.01695, train_acc 0.541, test_acc 0.520\n",
"385th: loss 12.02300, train_acc 0.539, test_acc 0.521\n",
"390th: loss 12.02415, train_acc 0.538, test_acc 0.523\n",
"395th: loss 12.01636, train_acc 0.540, test_acc 0.523\n",
"400th: loss 12.01147, train_acc 0.540, test_acc 0.527\n",
"DONE\n"
]
}
],
"source": [
"st = 200\n",
"n_epochs = 200\n",
"batch_size = 256\n",
"display_step = 5\n",
"save_step = 5\n",
"\n",
"n_batches = int(xtrain.shape[0] / batch_size)\n",
"if st == 0:\n",
" losses, train_accs, test_accs = [], [], []\n",
"\n",
"ylabels = np.argmax(ytrain, 1)\n",
"yweights = class_weights[ylabels]\n",
"ytestweights = class_weights[np.argmax(ytest[:512], 1)]\n",
"\n",
"for epoch in range(st, st+n_epochs):\n",
" avg_loss, avg_train_acc = 0., 0.\n",
" for i in range(n_batches):\n",
" batch_xs = xtrain[(i * batch_size):((i+1) * batch_size)]\n",
" batch_ys = ytrain[(i * batch_size):((i+1) * batch_size)]\n",
" batch_weights = yweights[(i * batch_size):((i+1) * batch_size)]\n",
" batch_xs = whiten(augment_data(batch_xs))\n",
"\n",
" if (epoch+1) % display_step != 0:\n",
" sess.run(optimizer, feed_dict={X: batch_xs, y: batch_ys, \n",
" is_training: True,\n",
" weights: batch_weights})\n",
" else:\n",
" _, curr_loss, curr_train_acc = sess.run([optimizer, loss, acc], feed_dict={X: batch_xs, y: batch_ys, \n",
" is_training: True,\n",
" weights: batch_weights})\n",
" avg_loss += curr_loss / n_batches\n",
" avg_train_acc += curr_train_acc / n_batches\n",
" \n",
" if (epoch+1) % display_step == 0:\n",
" test_acc = sess.run(acc, feed_dict={X: whiten(xtest[:512]), y: ytest[:512], \n",
" is_training: False,\n",
" weights: ytestweights})\n",
" losses.append(avg_loss)\n",
" train_accs.append(avg_train_acc)\n",
" test_accs.append(test_acc)\n",
"\n",
" if (epoch+1) % display_step == 0:\n",
" print \"{:03d}th: loss {:0.5f}, train_acc {:0.3f}, test_acc {:0.3f}\".format(epoch+1, avg_loss, avg_train_acc, test_acc)\n",
"\n",
" if (epoch+1) % save_step == 0:\n",
" saver.save(sess, \"fe_challenge/fe_cnn-localizer\", global_step=epoch+1)\n",
"\n",
"print \"DONE\""
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.legend.Legend at 0x7fac145b0690>"
]
},
"execution_count": 40,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA3QAAAFpCAYAAADUR2rXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XdUVccWwOEfXYqoIAqKWLCLvYJI7L3GAsbEBmrsGmsSNSZqNDH2lmfvAnZRY4saELD33gtWbKD0ct4fF64iF6UjuL+1Zj2dc87MnBvXu2xmZo+WoigIIYQQQgghhMh+tLN6AEIIIYQQQgghUkcCOiGEEEIIIYTIpiSgE0IIIYQQQohsSgI6IYQQQgghhMimJKATQgghhBBCiGxKAjohhBBCCCGEyKYkoBNCCCGEEEKIbEoCOiGEEEIIIYTIpiSgE0IIIYQQQohsSgI6IYQQQgghhMimdLN6AJoEBgYq9+7dy+phCCGEyGA1atR4Dlhk9TiyC/l+FEKIL0dyvyM/y4Du3r171KxZM6uHIYQQIoMpiiLRSQrI96MQQnw5kvsdKUsuhRBCCCGEECKbkoBOCCGEEEIIIbIpCeiEEEIIIYQQIpuSgE4IIYQQQgghsikJ6IQQQgghhBAim5KATgghhBBCCCGyKQnohBBCCCGEECKbkoBOCCGEEEIIIbIpCeiEEEIIIYQQIpuSgE4IIYQQQgghsikJ6IQQQgghhBAim8qRAV2JGlWxKl0yq4chhBDiy9AcuAbcBMZquN4TCATOxhW3D66bAgHA/IwbohBCiE8xMcuHUR7TrB5GiuXIgO6b3yfg9J1zVg9DCCFEzqcDLABaAOWBrnH/+yEPoEpcWfrBtUmAdwaOUQghRDL0WzKXUdvWY17EOquHkiLJCeiWA8+Ai+/VTQeuAueBrUDeJJ69C1xA9RvJk6keZQpFhIZhYGSUWd0JIYT4ctVCNTN3G4gE3IF2KXi+OlAQ2Jf+QxNCCBFPV1+fwuVKJ3m9eNVKFCpdEuO8efh+yVxMC1hk4ujSJjkB3UpUy0netx+wAyoB14EfP/J8A1S/kayRivGlSkRIKLmMJaATQgiR4QoDD977e0Bc3Yc6ovol6CagSFydNjADGJmRAxRCiC+BZckS6BoYJHm965Tx/OC5imKVK2q8XqdTe8LevGWR6yCM8pjSb/EcjPPm+WS/FRvXp6Frd4pXq4yuvn6qx58WyQnovIGXH9TtA6Lj/nwU+KzmJSNCQ9GXGTohhBCfBy+gGKpfgu4HVsXVDwB2owoCP6YvqlUuJ/Pnz59BQxRCZAe5zc1oNXwAY3a4U7hs0rNNX5oiFcoxaus6XOf9ibauTqLrNdq2pErzxsTGxNC4X89E1w1NTancrCGnd+3lzulzLBs8CvPChXBbNBODj0wS5StkSbdpE2k1rD+DVv3NZL99DFy5iBZDvv/oc+ktPfbQ9Qb+SeKagir4O4XqCylTRIaGZuqHKIQQ4ov1kHczbqD6BefDD+55AUTE/XkpqmWWAPbAIFTbE/4CugPTNPSxGNUqlxrPnz9Pl0ELIbKXfIUs+frnkfy8dwv1e3xDXsuCtB01JKuH9dloNXwAEaFhlLavRadxoxNcMy9iTYeffuDmidPsWbCEcvUcsC5fNsE9Ndo0R8/AAP+N2wC4ffIMq0b8TOEypek99090dHU19ttyaH9Q4K+O37F8yGh8N2xGR0+Pui4diY6IzJiX1SCtAd3PqGbq1iVx3RGohmqz+EDA6SNtpdtvIMNDQmUPnRBCiMxwAigFFAf0ARdgxwf3WL3357bAlbg/dwNsUM3ejQRWozlLphDiC1azfSt+3LWR2h3bctLrH6a1cWHnrAWUrFWdso51MqxfbR0dbCpqyvGUuXT09Og0YQzOv/2MlpZWoutlHetQqnYNds9ZyP7FK6jdsS0NXb8DQFtXh27TJhIbE8OGH3/lyPqNhAYH07hvzwRt1OnUjnvnLvL4+k113RVvXzwmTKFkreqqwO0DNhXLU61lUw6vWs/j6ze5dMgHrxnzmNvNjV+cWhATHZ3omYySloCuJ9Aa1ReSksQ98b+lfIYqeUqtj7SXbr+BjAgJxcDIME1tCCGEEMkQjWqWbS+qQM0TuAT8hip4AxgSV3cu7s89M32UQohsSS+XAa2GDeDBxSv83qIjm379gxcPAji6cRvP7wfQavhAtLQ1/zifVH1ytRjSj6Hrl1GhQb0k72ncrxf9lsxF3zBXqvspWKIYLpPHY12+TKJrRnlM6bdkDvad21OrQ2safRCIaWlr02r4QJ7fD8Dfcxt75i3m9O59tBo2gCrNGtG0vytFK1Vg469/8PrpMyJCQvFZ60nFRl9hVdoWUCVDsSxZgqObtifq/9TOPfi6b6Z+z28o51Q3wbW2o4YS/PwFh5avTfRcZgZzkPqArjkwGtWXVWgS9xgDud/7c1MSZsrMMJGhYbLkUgghRGbZDZQGbIEpcXUTeDdT9yNQAaiMKlHYVQ1trEQVGAohhFqdju3IbW7GzlkLCHoaqK6PiY7mn7l/U6h0Saq3Tpi7UNfAgJ6zpzFyy9oklwp+SokaVanfs5tqDJ00J+41MDaiYe9vKV2nJp0nfiw/YtIMjI3oNecParZrydANy+k4frT6HDjzItYMWbsEG7vyrBk1npNe/9BsgBtl6r6blazWqhmFSpfkn7l/q4Moj/FTuHP6HF1/n0Aj1+4c37qT8/sOqp/xWedJ+NsQGrn1iHs/VTKUs3sPaBzjjulzeXjlOl2njCdvwQIAVGrSgOJVK7Fn/mIiQpMKhTJPcgK6DYA/UAbVxm1XVIef5ka1ufss8HfcvYVQfbGBKg3zEVS/kTwO7AL2pNfAPyY8NBQ9AwONmyKFEEIIIYT43Ono6dGg17fcOnmGO6fPJbp+bt9B7l+8TPNBfdTZFQ2MjHBb8BcVG32FpW1xKjdrmOJ+c5kY03XKeF48eMh/a9wpW7eOOpB5X/XWzTEwMuLM7n1Ua9mUr7p3TXFfXX79CTPrQiwbNAqfdZ7U/roNY3d60vT73gxdtwSjPKb87TaEs3sOsOm3P3hy4xbf/vErZoWt0NXXp/mgPjy4dIVz7wVs0ZGRrBg6hlePnvD8fgBbp85M0GdY8BuObNhE5WaNKFrZTp0MJTIsXOMYoyMjWT3yZ3T0dPn2z9/QN8xF6x8G8vjGLY5v3Znid84QiqJ8duXEiRMKqmWcqSr1vnVWZlzwVwxNc6epHSlSpEiRkrFFUZSTWf2dk51KWr8fpUiRkn2KfecOyowL/kqpOjWTvKdkrerKjAv+Sv0e3yiGprmVwWsXK3+e8VGqt26ujNnhrgxzX57ifrtOmaD8ecZHsalYXslXyFKZfs5Xafp970T3jdyyVhnusVIBlO4zpijTzx5JNFYz60JKs4F9lAr1HRM97/hNZ2XGBX+lQa9u6jrLUrZK/+ULlBkX/JWxOz2V/DbWCZ4xL2KtTPbdpwzzWKE07ttTmXHBXylZq7rG99A1MFD0DQ01XjPOl1f5/dhB5Tfvf5QZF/wVq9IlP/m5VGneWJlxwV8ZuWWtMuOCv1LavlaG/xtI7ndk6uZhP3ORcVOfBkZGhAW/yeLRCCGEEEIIkXzaujo0dP2Oe+cucuPoiSTvu3n8FFeO+NOoTw+qtWpGwZLFWT1iHBcP/oe+oSGdJoymeLXKGmf4NKncrBE12rZg78Kl3L9wGYAb/sep9XUb9i9eiRIbC6j2nVmVssVz4lQA3MdNpkDxonz352/McumFvqEhDV2/o2qLJupln1d8/Ng2bRbP7wdgU6kCbUYO5tIhHw6vXK/u/8mNWyzqPRDbmtV4dO1Gop/jXzwIYP1Pv+E6fzpFypflyhF/bh4/pfFdoiMiNNYDhLx6jf/GrdTv8U2iZChJObvnACVrVce+c3uuHPHnuv/xTz6TWdLj2ILPTkRIXEAn++iEEEIIIUQ2U711c8wKW7H/fys+ee+uWQvJlduEAsWLsnzwaC4e/A+Ak167CXkdhNO3zome0dLSotkAN1wmj6PZADdqdWhDhfqOdBo/mnvnL3FgyUr1vUc37yCflSVl6tZW19l36UDYm7ec2b0PgMiwMFYOG4u2jg7DNixn9Lb1VGz0FT5rPZnctAPb/phNsSqVGLV1Ha2GD6D7X5MJehrIhnGTUBQl0fhunTid5KTM5f+OsHfhUqLCI9g1a+EnP5+kHF65nuDA5xxevSHZz2z7Yzb7Fi1j829/prrfjJAjZ+jC1TN0kulSCCGEEEKkXOGypXl84xaxMTGZ2q+2jg6N3HoQcPkaV3z8Pnn/4+s3WTNyHK8ePeHBpSvq+qjwCPw3bqNh728xK2zFy4eP1dea9nelaX9Xgp+/wMQsH9pxGTEjQsNY/+NEYqPfvfOlQz68efGSOh3bcdXHH+N8eanctCFHN21PsO/s+f0A1oyeQIexw/F136w6IiAoGACftR6c/Wc/rYYPpGHv74iOjGTed/1SvZJu36JlHF65nsiwsFQ9D/Dm+Qt+bdgmRc9ER0Swd+HSVPeZUXJkQBcZqvqPa2BsnMUjEUIIIYQQ2Ym5dWE6jhtJmbp1OLF9F+7jJmdq/5WbNcKiaBFWDkv+sZTn9x/SWO/rvpkGPbvh2K0LO/6cA4BdQyea9nfl+NadeEyYgrauDnkLFiCvlSWvHj3m1aMnCdqIiY7m5PbdOHV3IXd+c2q0aY6uvj5+nlsT9XfN9yjT2iSeEQR48+Il7uMm4eu+GV09XQIua0r4m3xpCeZympy95FIOFxdCCCGEEMmgo6tLQ9fujNq6jqKVK3L+wGFqtmtF4369MqV/LW1tqrduTtuRg3l84xYXD3qnuc3gZ4Gc3XuA2h3aYGBsRIHiRen6+wTuX7jM5snTAYiNjuHlw8fcPnkmUTAX7+iWHejo6lKrQ2vqdG7PrVNneHrrTqrG9ODiZe6cOZ/qdxKJ5cgZuvAQWXIphBBCCCFUCUbqOnfk7J4DvHnxUuM9FsVs6DHzd6xK2XJu30G2/TGb4GeBuEweT4tBfXkZ8JDTu/aluO/8NtaEBb8h5HXQR+8rW8+eVkP7U6hMKQIuX8Nz4u8a95alhvcad6q3bs5X3btStUUTosIjWDl8LNGRkclu4/m9B9w8cZrGfXqib5iLPfMWp8vYRPrIkQGdOsulJEURQgghhPiitRjUl4au3SlRvQqrfvhJ4z2dfxmLqUV+lg0axeX/jqjrN06cSj6rgjj/9jOvHj9NdrZIHV1dmg5wo2Hvb7nuf4Il/YdrvM/ASHWwdqk6NVR70EaN59zef9MtmAMIuHyNW6fO0GyAGzFR0fzdZ3CCQ8qT69jm7ZSsWY03L15y/sDhdBufSLscGdBFSEAnhBBCCPHFK21fk4au3Xn56DGVmjSgWJVK3D2bcLlfhfqO2NaoyubJ0xMEc6DaP7Zy+I8MWbuEXnP+YOPEqcTGxqKnr4+Ovj7RkZHcO3eR10+eqp+xKm1L1ykTKFy2NM/vB1Davia5zc00zg5Wa92MUnVqsH36HHzXbyImOjpDPodDy9ZiW70q26fP4faps6lq4/z+w7Qc+oSjm7YTExWVziMUaZEjA7rIsHBiY2NlD50QQgghxBfKxCwfXX//hSc3b7Ow1wBGbF5DmxGDmPddX/U92jo6tBo+kGd37nF083aN7YQFv2HJgB8YsnYJPWdP03jPy0ePuXP6HG9evMSxayfC3rxl+eBRBN57wJgd7lRp3hifdZ6JnqvZrhWPrt/Ee7V7+rx0Eq74+DHBqQUhr16nuo3oyEh+b9Ep07N+ik/LkQEdqDJdSkAnhBBCCPHl0dLSwmXKeAxNTPhf36GEvA5iz/wlOP/2E5WaNFBnhaz1dRsKlijGiqFjEqTq/9DLgEdMb/8N+YsWIToikqiICKKjojDMbULxqpUpUb0KperUxDS/Oef2HWTz5Onq4OnhletUbdk0UUBXoHhRilaqwPbpczLug3hPWoK5eBLMfZ5ybEAXERIqSy6FEEIIIXI40wIWmBWy4vn9B7x9+QoAp+9cKOdoz6ZJf/Lkxi0ATmzfhdN3zrQaNoBLh3zQ1den2QA3bp86m6yMkiGvgzQmN3l45TpH1m8EwNDUlLDg4ATXT+/eR5sRgzAvYs2LBwHq+prtWhITHc3pXXtT/e5CQE4O6EJDJculEEIIIUQWqdetC/pGhvy7ZFWG9tNr9jRsKpYHVMsjA+89oFDZUpw/cBj/985KU2Jj2TlzPn0WzcK+SweM8+bBNL85K4aMTrexfBjMAZz5Zx+thg+gWqum7P97ORB/REELrh45ytsXr9Ktf/FlyrEBXXhIiMzQCSGEEEJkgeJVK9F29FBioqI4sn6j+ozg9Jbb3AybiuU5tsWLR9duYFHMhgLFbLhz+hyev0xNdP/VI0e5fvQETfu7oquvz9k9B7h/4XKGjC1e0NNAbp86S7WW7wK60nVqkqegBdv+mJWhfYsvQ44N6GQPnRBCCCFE5tM3NMRlynjC377FyNSUCvUdk32GW25zM6KjojXOdGlSpm4dAHw3bOLh1evJembnjPkM81hBbEwMu+YsStYzaXV61166TPwR6/JlCLh8jZrtWhIaFMylw0c+/bAQn6Cd1QPIKBEhoRLQCSGEEEJksjYjB2NWuBArhozh9dNnVG7aMMl79Q0NKVfPgXajhzFq6zomHt7FL//uoO2oIZiY5/tkX2Ud6xAc+DzZwRzAw6vX2TVzAVt/n8HLgEfJfi4tzu8/THRUFFVbNiVXbhPsGn7F6d37JP2/SBc5doYuIlSSogghhBBCZKayjnVw6NKBQ8vXcvvUWc7vP4R95/YYGBslWnZpnDcPI7euwzS/OVHhEdw+fZaTO3ZToEQx6nXrgn3nDvhu2MShles0ZmjU1tGhjENtLh76dEKTDx1etT7V75gaYcHBXPXxo2rzJjy/F4BeLgNObt+dqWMQOVcODujCJKATQgghhMgkhqamdPn1Jx7fuMWeBUsAOLf3IE7fOlP+K0fO7E647NLxm86qpCRDx3L1iD/RkZHqa/8uXU2Tfr34qkdXan3dhuntv0l0MLeNXXmM8phy9cjRjH+5dHB6937sGn5Fi8F9eXLzNg8uXcnqIYkcIkcvudQ3lCyXQgghhBCZ4eufR2CSLx8bfvpNHZzdO3dB47JLfUND6nbtxMVD3lw8+F+CYA7g+b0HbPjpN+Z844ahaW7su3RI1F8ZxzrExsRw3f94xr1UOrp0+AjhISEY58vLyR0yOyfSTw4O6EIwMDJESzvHvqIQQgghxGehQPGiVGvZlH+XrU6wn01RFM7vP0RZxzoJchvU7tgW47x5OLhszUfbDbh8lSvefjg4f42uvn6Ca2Ud63Dv3EXCgt+k78tkkOiICC4cOExsTAyndsrZcyL95NhoJyI0DEDOohNCCCGEyGC12rcmJioa3w2bEl07v+8gegYGlK/vCIC2rg5fdXfh1qkz3Dt38ZNt+6zzJLe5GVVbNFbXmZjlw8auPFeO+KffS2SCnTMXsMh1EMGBz7N6KCIHycEBnWrjrb5kuhRCCCGEyDDaujpUb9uCy96+vH2Z+JDsu2cvEPQ0UL3ssmqLpuSzsuTQsrXJav/G0RM8vnGLet2c1XVlHGoDcM03e+yfi/f25Stunzqb1cMQOUzODejiMinlksQoQgghhBAZppyjPab5zTm+dafG64qicP5A3LJLYyMa9OrG4xu3uOLjl+w+fNZ6ULhcaUrUqApA2Xr2vHnxkodXkn9cgRA5Vc4N6NRLLiWgE0IIIYTIKLU6tCY48DlXP7L88VzcsstO40djVcqWg8s/vnfuQ6d27SPk1WucvnVGS1ubMg61uXrkKIqipHX4QmR7OTegCwkBQF/20AkhhBBCZAgT83yUq1eXU157iI2JSfK+u2fOE/QskGqtmvHy4WPO7jmQon6iIyLw37iNCg3qUaVZI4zz5vloACnElyTnBnShsuRSCCGEEDmbUR5T8hWyzLL+q7dqjo6eLse3aV5uGU9RFC4cOAzAf6vXExuddPCXFF+PLSgxsXQcN4rYmBiu+WWP4wqEyGg5+mBxQA4XF0IIIUSOY5THlPo9u+H4TSe0tLRZNmgkN4+fypC+dHR1KefkQFR4BNf8jiW4VqtDa+6evcCzO/c+2Y7POk909fU5tsUrVeMIfhbIuf0HqdayKXfPXiAsODhV7QiR0+TcGboQyXIphBBCiJzF0NSU5oP78vPeLTTo/S2XDx/hRcBDXOf/pU4Ykl4KlihGm5GDGX9gO73m/EHf/82m4/jR6vPgbCqWx7JkiU/OzsV7fj+Ajb9OIyo8ItVj8l7jAZCihCoi82hpaVGrVukUPaOrq4O9fdkMGtGXIefO0MVnuZSATgghhBA5gLaODsM2LCO/jTVn9/7LvkXLeHrrDibm+ei/bAFuC2awpP9w7pw+p35GS1sb63JleP7g4UdntAxNTSlU2har0iWxKmWLdfmyWJcvQ3RUFJcO+XB8205KVKtCI7fuWJcvw+offqZm+9ZEhoWneD9cWjy4eJm/3QZz7/ylTOtTJF+HDvZs2vwjNWsM59Spm8l6ZsYMVwYPaUOd2iM4flyylqZGjg3oIsNkyaUQQgghco5StWuQ38aadWN/4fSufer6ty9e8bfrIAasWEifRTNZNnAkOnq6VGzcgIqNviK3uRmhQcHsnvs3RzdtR4mNVT9rbl2Y5oP7UqV5Y7S1VQu33r58xePrt9gxfS6ndu5Rny131cefe+cv0nXKBIZ7rkRHT5fz+w+pf4meWW4cO5mp/Ynkq1bNFoCvvrJLVkDXsGElBg9pA0DXrk4S0KVSjg3oFEUhIjRUslwKIYQQIkeo1roZocHBnNt3KNG1Ny9essh1EAOWL2DAioWAKkHc5f98ueZ7lOptWtBp/GjqdGzHlt//4kXAQ5r06419p/bEREfz38r1XD96gsfXb/Lmxcskx3DpkA+znHvRc9bvFCpTimNbU7cfTuRMFexsALB3KAczt330XlNTI5avGMbVqwHcufOUzl0cGTFiObHv/cJBJE+ODehAtexSZuiEEEIIkd3pGxpSsVF9Tu/aS0xUlMZ7ggOfs9B1EE7fOnPnzDmu+R0nOkK1X+3E9t1UbtaItqOGMGTtEiLDwtHR0+XYFi/2/72c4MDnyR7LiwcBzP22D4XKlOLeuYvp8n4iZ6hQQRXQ1a1b7pP3zp7Tl0KFzKjrMIpixQri4TmGevXK899/8m8qpXJ2QBcaJnvohBBCCJHt2TVywsDIkFNe/3z0vuBngeycOV/jtXN7/+WKtx8Nen+LaX5zDq1cx/N7D1I1nqjwCAnmRAJGRgaUKGHJgweBFCliQfHiBblz56nGe9u2rU3Pno2YPMmdEyducOnSfUJCwnF2ricBXSrk2CyXED9DZ5zVwxBCCCGESJMarZvzIuARd89eSFM7kWFh7F2whI2/Tkt1MCeEJuXKFUFbW5vly/YDULdueY33WVjkYfGSQZw+fYtJk1RZS0NDI9ix4xgdO9VFV1cn08acEYyNc2FkZJCpfebsgE720AkhhBAim9DR0yO3uVmi+tz5zSlVpyand+1FUZQsGJkQn2ZnVxQADw8fgoJCcHDQfBTB1KndyZPHmB7dZxIVFa2u93D3wcIiDw0bVkr0zLr1I9m3fxI6Op9/6OLq2oSnz9ZSoEDeTOvz8/9U0iAiVPbQCSGEECJ76DhuFD/9sxnr8mUS1Fdt2QRtHR1O79qbRSMT4tPs7IoSHh7J9euPOHr0mioxygcMDQ3o3MWRtWsOcenS/QTX9uw5xevXb3F2rpeg3s2tKV27fkXjxlX48cfOGfoO6cHZxYnr1x/y7NnrTOszZwd0IaGyh04IIYQQn71cuU2o1rIp+oa56D13OqYFLNTXqrdqzv2Ll3l2514WjlCIj6tgZ8Plyw+IjY3F3+8KFSsWxdQ04c/hLVtWJ3duIzZs+C/R85GR0WzbdowOX9ujr69K81G8eEFmznLjwIGzbNjwH+MnuFC1qm2mvE9qFCtWEHv7sni4+2Rqvzk+oJMll0IIIYT43FVr2RS9XAa4j5uEgYkRvef+gV4uAwraFse6fBlOee3J6iEK8VEVKthw8aLqlw6+vlfQ1tamdu2Es81dnOvx5MkrDh/WnPjEw92bvHlNaNasGtra2qxYOYyYmFh695rDoIF/8+xZEKvX/ICBgV6Gv09qdOniCKiWnWamnB3QhYXJkkshhBBCfPZqf92Wh1euc2L7btaNmUjhcmXoOmUCNdq2ICY6mrN7DmT1EIVIUp48xhQpYsGluIDu2LHrxMTEJDi+wMTEkNata7J5k2+SZ839++85nj8PxtnFieHD2+HkZMeQwf8jIOA5r169xc11LhUq2DBp0reZ8l4p1cW5HkePXuXevWeZ2m/ODuhCQsllbIyWllZWD0UIIYQQOVjT73tTpXnjJK+bFrDA+befyVfIMtG1wuVKY12+DMe27ADg8n9H2DljPpWbNuSr7l255neMty9fZdjYhUir+PPn4vfFvX0bxvnzd7F/LzFK27a1MDQ0wP0jyxGjo2PYstmP9u3rMHnKd2zd6s+aNYfU1/fuPc3fi3bzw4j2ODpqzqKZVUqXLky1araZvtwScvo5dCGhAOjlykVkWFgWj0YIIYQQOVFZxzo0G9iH6MhInt25x6NrNxJc19LWptvUXyhZqzpmha34221wgmyVtb9uS1R4BKd371PX/bd6AwVLFKN2x7ay3PIL5+RkR8uW1fnxx9WfbZbT+AyX8UsuAfz9rvJd9wbo6GgTExOLs4sTDx4E4ud35aNtubt707dfc549e833/RYkuj5q1AqaNK2K58axnDlzW10fGRnF5EkenDp1M53eKmWcnesRGxvLxo1HMr3vnD1DF6oK6GTZpRBCCCEygo6eHu3HDCfw3gNCXgfR7Y9f0cuV8AyqBr26UbJWdS4e8qZkrerU6dxefU0vlwHVWjXj3P6DhAW/SfDcpsl/ssh1kCy3/ML16t2Y0WM6MXBgq6weSpLs7Gx48yaU+/cD1XW+vlfIndsIO7ui5M1rTLNmVfH0OPLJoNTb+xJr1hzi224zCAwMSnQ9JCQcF+c/uXo1ADMzE3WpW7dZnOLFAAAgAElEQVQ8nhvHYmKSNfkznF3q4e19iUePXmZ63xLQCSGEEEKkktN3zlgUs2Hr1Jm4j5uEpW1x2owYrL5epEI5mg/sy7l9B1kxZAzX/I7R+oeB5LNSLb2s1KQhhrlNOLbFK1HbsdEx3Dx+KtPeRXyebG2tAJj2R09Kly6cxaPRrIJdUS5eTHgMQfxMXN265ejQwR59fb1kJQuJjY2lR/eZHDhwNsl7Tp26ScMGP2FfZ6S6tGs7CRub/Myc6Zq2l0kFO7uilC9vg2cmJ0OJl7MDurgllwaS6VIIIYQQqVS8aiU6/TIGE7N8CepNC1jQpF8vLh7y5prvUa77n+DwyvXUdelIhfqO6Bsa0m3aRN68eMHGX6cBsHGi6n87TxwLQO2ObQi894DbJ89k7kuJbMPW1pLdu08SFhbBqtXDP8vDtStUsFEnRIl3794zHj58gb1DOZxdnLh16zEnT95IooW08/e/yvQ/t+DWpxktW9bIsH40cXFxIjo6hs2b/TK133if37+IdKQO6IyNs3gkQgghhMiu6vf8BvtO7RnuuZKile3U9a2HD0BbR4ftf85R1+2e+zcPr1yny68/4TJ5HOY21qz78Vf1cspXj5+wa9ZCyjjUptXwAdhWr6pOhiLEh4yMDLCyMsP3yGUGDlhE7dplGDOmU1YPKwELizwUKJA3wf65eH5+V2jSpAoNG1bKlGQhEyeu59y5OyxdNgRzc9MM7y+es0s9/v33nMYlopkhZwd0oapEKAZyuLgQQgghUkFbV4eStWpwzfco0ZGRDFyxCMdvOlG8aiWqt27O4ZXreRnwSH1/TFQUa8dMQN/QkMpNG3Jw2ZpEs2/+nlu5eeI0DXt/R0x0NCe3787s1xLZRIkSqqW5t249xtPzCBs2/McvE7tSpUqJLB7ZO5oSosTz871CgQJ50dXVyZSz2SIjo+nRfSZmZiYsWNg/w/sDqFGjFLa2Vlm23BJyfJbLEECWXAohhBAidYpVrkguE2P8PLdx6+Rpuk6ZQIcfRxARGsqrx0/4d+mqRM88u3OPDeMmUc7Rnr0LlyS6rigKnr9MZeTmNVz1PcqbF5mfROFL06hRZfz9rxIaGpGh/bRpU4v9+88SHh6ZLu3F75+7desJAIMG/s1XX9mxZu0I5s9LvO/yY169esvmzX7ExGg+A87GxgJLy3wcP349Re3GB3TxRxa8L34f3eXL97lw4W6K2k2t8+fvMvGX9fw+tQd3bj/h7t2nSd4bHa3KShkcHJrs9lu3rknhwubqvzdvUZ3IyCi2bvVP07jTImcHdJIURQghhBBpUMahNjHR0dw8fpLwtyGsGDKahq7dafJ9L7ZNm01UuOYA4fy+g5zfdzDJdl88CGCWc085Xy4T9OzZiOUrhuHh4UNXlz8zrJ/Gjauwfcd4Fv9vD99/nzjdfmrY2r6boQNVUObaey7bto9j0d8DU9zepN/c+eWXdYnqDQ0N2LvvNwoVMsOyYHfCwpIf+NrZ2fD8eTBPn75OdO3Mmds8evSCpUv2aXgy40yfvoXGTaowZuynl6cWLWrBhAmJPxNNevduwtJlQxLVe3j48Pp1SIrHmV5yeEAXt+RSAjohhBBCpEKZurW5d+4i4W9VP6wpisK/S1dxaOVaYqNj0tT2szuJl6iJ9GVjY8HsOX158SIYZ+d6bNvqn2FL/1xc6gHQt19ztm07yp49ac9QamtrxcuXbxIEC3v3nqaARTeMjAw+8mRif/zZix9/6szOncc5cSJhcpKpU7tTpow1AC1bVk9Rcg9VhkvN/5ajo2MoatM7yVnBjBITE0uTxuMpUCDPR+9bv2EUzi5OyQroihUryKzZbhw6dJ5u3/yV4PiFwMDgNI85LXL0HrpI2UMnhBBCiFQyMctHkQrluOZ3LNG1tAZzIuNpaWmxYuUwAGrXGoG//1UWLOyPlZVZuvelp6dLh68d8PQ8woULd1m6bDD58pmkuV3bklbq5Zbve/MmjKdPX6eoDB2ymEePXrJq9Q8YGr4LBhs2rMSQoW1ZuGAXT568ootzvRSNUVOGy/dldjAXT1GUT34m69f9R6lShaha1fajbWlpabF8xVAUBXr1nM2TJ68StBMbmzXvGC9HB3SxMTFEhoVLQCeEEEIIjXKZGNNq+ADyFLRIdK20fU0ArvkmDujE52/w4NY0aFCJH4Yv5fbtJ/TsMYtcufQ1LplLq6ZNq5IvnwmrV/1Lj+6zsLDIw/wF36e5XVtbS/Vyy7QKCgqhd6/ZlC1rzdSp3QEwNTVi+YphXLsWwKhRK9i08QitW9dM9uHc1tb5yZPHOMkZus/dli1+REVFq2dXkzJ0aFvq16/IsKGLExye/rnI0QEdqPbRyZJLIYQQQmjy9c8jadj7O1oNG5DoWhmHOoS8ek3AlWtZMDKRFmXKWDN1Wg+8vI6zfPl+AG7ceMSY0Sto0aI6ffo0S9f+nF3q8fLlG/bvP8vZs7f5deIGunb9ii5dHFPdpq6uDkWLFuC2hhm61Dp48Dzz5noxZGhbGjSoxKzZfShc2Iwe3WcRFhaBh8cRDA0NaNu2VrLae5fhMnFClOzg1au37Nt35qOzkuXKFeH3qd3ZseMYK1f+m4mjS77kBHTLgWfAxffqpgNXgfPAViDvR57XAc4AO1M5xjSJCA2VLJdCCCGESKRaq6ZUb92c5/cDqNqyKQVLFFNf09LSorRDLa77H0fJ4uVUXxJtbW0sLfMlKJ+aLdLV1Ulwf6FCZqxaPZyQkAj69pmX4N6FC3dz4MBZZsx0pXjxguky5ly59GnXrjZbt/gTFRUNwB9/bOLo0bQt8bSxsUBXVyfdZujijR27imvXAti4aSy9ejVm6u8b1Zkt/fyu8OBBYJIBjoGBXoLPuk6dMgBcupQ9Z+gAPNx9KFq0gPpd3qerq8Oq1cN58yaMvn3mZ8Hokic5Ad1KoPkHdfsBO6AScB348SPPDwWupGZw6SEiRGbohBBCCJGQWWErOo4bzZ3T55jXvS+RYWE07e+qvm5VuiSm+c017p8TGWfd+pE8erw6QXnydA2WlvmSfGb7jvEJ7g94uIpatUozoP/CRJkXFUWhd685REfHsHLVcLS1075YrWXLGuTObYS7u7e6LiYmlp49ZmNoaMCSpYNT1e6HRxakl7CwCHr2mE2ePEacPn2LSZM81NcURcHT4wjNm1cjb17jBM9ZWubj5q0lCT7rCb905cGDwCzN8JhW27cfJTw8EhcXp0TXfv65CzVqlKL/9wt59ixxFs/PRXKyXHoDxT6oez/36FEgqZyg1kArYArwQ0oHlx4iQ8NkD50QQggh1LR1dOg27VcURWHdjxN5++IVPus8adK3F/sXr+TJjVuUqVsbgGt+x7N4tF8OLS0tmjWryoEDZ9m00RcAU1ND/pzemy5dHJk7N/G5a1ZWZjRrVhUPDx8OHTyvrr979yn79p1JdD9AQMBzhg5ZzKrVPzB8eDtmzNiapnE7u9Tj6dNXHD58IUH99esPGTtmJXPn9cPNrSlLl6Ysdf+HRxakp2PHruFYdwy3bz9RzyrG8/DwYcTIDnToYM+KFQfU9UuWDsbMzIShQxYTERGlrj9z5la6jy8zvXkTxu7dJ+ncxZEfflimTnBSvXpJfh7nzJo1h9iyJflZP7OEoijJKcUURbmYxDUvRVG+TeLaJkVRqiuKUl9RlJ3J7IsTJ04oQLoUt0UzlaHrl6Vbe1KkSJEiJf2Koignk/vd8JmW5oqiXFMU5aaiKGM1XO+pKEqgoihn44pbXH0VRVH8FUW5pCjKeUVRnDP7+/FLLk37uyozLvgrVVs0UdcZmuZWJvvtV3rMmqoAyvdL5ykjNq/J8rF+SaVCBRslVvFSundvmKD+9Jk5yhHfPzU+M2RIGyVW8VLKlLFOcX+bNv+ohIVvUSpUsEn1mE1MDJW3IZuUefP6abyupaWl7D8wWQl+46kUL14wRW1Pn95bCQ3brGhpaWX6f4sbNxcre/b+pv67m1tTJVbxUgYNap3l/04yonTu7KjEKl5K/foVFUDJlUtfuXhpgXL/wQolTx7jLBtXcr8j0zrP/DMQDWg6vKE1qr13yT2Eoy9wEjiZP3/+NA7rHVlyKYQQIoPoAAuAFkB5oGvc/37IA6gSV5bG1YUC3YEKqLY1zObj+9FFOqnSvDFN+vXi5I5/OPPPfnV9WPAbvFdvoFLj+hSvVpni1SpLdstM5uBQDgBf34Q7dTw9fHBwKEfRogUSPePs4sSZM7e4di0gxf19328BQUEhrFr9A3p6qTuauU2bWhgZGSR5tl38Es+YmFhWrByWoiWeJWwtuXXrSYLzzjKLh7sPDRtWwsIiD8WLF2TGTFcOHDjLggW7Mn0smWHXrhO8fRuGc9zewcmTv6V8eRt695pDUNDnv5w0LQFdT1RBWzdUUeSH6gJtgbuAO9AQWPuR9hYDNYAaz58/T8OwEooMDZOATgghREaoBdwEbgORqL7r2iXz2etA/Mm+j1D9AjRx3nyRbkrUqMrgtYv5bvokHt+4xZbf/0p0j/daD0KDg/lu+iR09fS45ns0C0b65bJ3KMezZ68TLTGMD5Y+zBhZtGgB7O3L4ulxJFX9PX8eTN8+86lWzZbx451T1YazSz0CAp4nCkLf9+BBIEOHLMbJyY7hw5P7fxHpe2RBSnl4+KCrq0OXLo6sWDmM2FhVYJoVwWVmCA2NwMvrOB071aVRo8oMG96OhQt2ceDA2aweWrKkNqBrDoxGFbCFJnHPj6j20BUDXICDwLep7C/VwkNCMDCULJdCCCHSXWHgwXt/D4ir+1BHVFmhNwFFNFyvBegDSW1EyZAVLF+KAsWL4rrgLwauWEhey4J4TPid2S69iQhJ/ONL+NsQDq9cT54CFkSEhnH79LksGPGXq27dchoDozt3nnLs2LVEmRfjA7ykZseSIz4V/dgfO1OrVukUPZs3rzHNm1fD08Pnk4HO6tUH2bbtKJOnfEeFCjbJar9ECct0PbIgJS5cuMvly/eZOq0HTk52DBn8PwIC0m/C5XPk4e5D/vymbN32M7duPWH06BVZPaRkS05AtwHwB8qg+rJyBeYDuVFluzwL/B13byFgd/oPM/XkHDohhBBZyAvVLzYrofrOXPXBdStgDdALSCo3foasYPkS5MptwoAVCylWuSJeM+YztVUXjm/1IjYmJslnjqzbyNuXr7h57CQxUVFJ3ifSl4VFHkqVKoS/n+aZLk8PH6pXL0mpUoXUdc4uThw9epW7d5+mqe9hQxfz6NFLVq4ajo5O8uc6unSph76+Hu7uyQso+/WdT1BQCCtWDvvkvQUL5sXExDDLZuhAFeCYmBiydas/a9YcyrJxZJY9e07x+vVbDA316dF9JqGhEVk9pOT7DDaUZ+im74au3ykzLvgrerkMsnzDpRQpUqRISViSu+H7My32iqLsfe/vP8aVpO7XURQl6L2/myqKclpRlE5Z8f34JZT2Y4cr088eUQqXK52i58ytCysm5vmyfPxfUmnXro4Sq3gp9vZlNV4vXNhciVW8lHHjnBVAKVWqkBKreClDh7ZNl/7jk2I0bFgpWfdbWuZTAp+vU3z9pqeonz59mimxipdSvXrJj97n4FBOiVW8lObNq2fZfxNLy3zK0qWDlfz5TbP830dmFTe3pkqfPs2yfBzxJbnfkWk/fOMzF7+kQo4uEEIIkc5OAKWA4qiWTLoAOz64x+q9P7fl3bms+sBWYDWqpZginVmVLkldl474b9zGwyvXU/Tsi4CHvH3xKoNGJjRxcChLREQUp09rXnn88OELvL0v4hx3Vpizcz1iY2PZuDF1++c+5OV1nODgUI1nkWmyZOlgDA0N6NljVor62bjxCJGRUZ/sp2TJ+DPosm6G7smTV7i5zeP58+AsG0NmW7p0H0uW7M3qYaRYjg/owiWgE0IIkTGigUHAXlSBmidwCfgNVfAGMCSu7lzcn3vG1XcBnOL+fjauVMmcYX8ZOv48ktCgYP6Z97+sHopIBoe65Th58kaC880+5OHuQ4UKNtjZFcWlqxM+Ppd59OhluvQfHh7J9u3H+LqjwyczXrq5NaVVq5qMHbOSGzcepaif169D2Lv3DF2cHdHS0kryPltbK2JiYrh791mK2hdfphwf0EWGxgV0so9OCCFE+tsNlAZsgSlxdRN4N1P3I6qjCSoDDYCrcfVrAT3eHWdQBVVQJz6i6+8T+O6vyeTKbfLR+6q3aUHxapXZNWshYcFvMml0IrX09XWpUaMU/n5XP3rf5s1+xMTEMCkupbyHu3e6jsPD3Rszs9w0blw5yXvSI4W/h7s3RYpYYG9fNsl7Stha8uDB80SHfguhSY4P6CLiAzojyXQphBBCZFf2nTtQo00LqjRrxLD1yyhQvKjG+3LlNqHNiEHcPXeBE9tz5plZOU316iUxMNDDL4mEKPGePXvNwYPnadeuDtHRMWze7Jeu49i//ywvX75RL+v8kLa2drqk8N+x4zhhYRG4uNRL8h7buDPohEiO1J2imI2ol1zKDJ0QQgiRLZkXsabNyMFc8zvG/v+toPuMKQxdv4z1P/3KpUMJMww2H9gH43x5WdJ/OIqiYG9flrdvw7lw4W6q+y9c2JyqVUuwc+eJNL7Jx5UoYUmpUoXYu/d0qtvIlUuf/v1bYGho8NH7Tpy4wf79Z1Ldjybt29fh6NFrPHmSsv2H8QeK+31ihg5Uyy6bNKnKv/+eIzAwKFXjTEpUVDRbt/jTuYsjuXLpEx4emeD6sGFtcXKyo2ePWWlK4f/2bRi7dp2kU2dHhg1bSmxs4gS3trZWbNvqn+o+xJclxwd0kaFhABgYG2fxSIQQQgiRUlra2nSdPI7Y6Gg8Jkwh6Gkgs1160XP2NHrP/ZPjW3eio6eLRTEbLIraYJjbBF/3zepEKMuWDyU8PJJqVYemegwjR3Zg6LB22BTplaFncc2a3YdmzapiZdmdV6/epqqNHj0aMmOm2yfve/nyDRb5u6XbQdFdujji7jGGY8eu4Vh3NDExSZ3CkZhD3XLcvPmIZ89ef/LeLVv8mPBLV/739z9pGW6S3N29cXVrSosW1dn6XkBVvrwNU37vzrZtR1m9+mCa+/Fw96ZTp7p89ZUdhw6dT3Atd25DLCzyyAydSLacv+QyfoZODhcXQgghsp36PbpSvFpltkydQdDTQACCngayoEd/jm3xonqb5hSvWpmw4Dec2rmHzZOns+OveYBqiVyJEgWpUqUEZcpYp3oMdhWLAe8Oss4I+fKZ0KxZVfT19ejQwT7V7Ti7OHHlygMM9Dugr9deY3FznYuZWW7Klk39Z/I+KyszFizsz4MHgdSuXYaxYzul6HkHh7IaDxTX5PXrEIoV7c22bUdTM9RPOnz4Ak+fvkpwiLmeni6r1/xAUFAI/frOT5d+du8+xZs3oRqXXdraZn2GS5G95PiATpZcCiGEENmTZSlbmg/qy/n9hzi9M2Eq8ejISDx/+Z0x1b9iSvOvWdxvGFt/n4GfxxaiI1QHAhcpkh99fT0AnJ1TH4zZ2dkAJPghP7116GCPvr4eQUEhOH9kb9XHWFmZ4eRUAQ93b6KioomOjtFYfHwuAe+WOqZVfAr/xo3GsWHDf0z4pStVq9om69kSJSwpWDDfJxOiZJaYmFg2b/KjTZtaGBvnAmDcOGeqVbPl+34L0m2ZZ1hYBDt2HOfrjg7o6uokuGZrawkgM3Qi2XJ8QCdZLoUQQojsR1tXh2+mTCDszVs2TfozyfsUDfuP4sX/YKwKkpJ3vtiHLCzyULBgPu7ceUqtWqUpUcIyVe18Shfnety8+Yj583bSsGElLCzypLiNzp3roq2tjYfHx89mu3HjEYGBQTjUTXtA5+bWlJYta6hT+A8a+DeBgUGsWj0cAwO9Tz5fN24Mvr6X0zyW9OLu7o2RkQFt2tSiZs1S/PhTZ1at+jfdZwU93L0xNzelceOEJ5bIDJ1IqRwf0MVERxMdGSlZLoUQQohspH6PbyhcrjSbfvuTkFef3lulSfwPxgsX7KJcuSJUjFs6mRIVKqhm56ZM9gBUB1qnNwuLPDRqVAkPdx82bPBGR0eHTp3qprgdZxcnzpy5xbVrAZ+818/vCg4OSafNT47ixQsyc5ZbghT+r169xc11HnZ2Rfntt26fbMPBoRxBQSFcvvwgTWNJT76+VwgIeE7PXo1ZtfoHHj16ybChS9K9n337zvDq1dtEM7K2tpY8e/aaN2/C0r1PkTPl+IAOVPvoJCmKEEIIkT2YFbaiSb/enD9wmIsH/0t1O7a2lkRERDFnjhfR0TFJpok3NzdNsg07O9XxCLt3n8TX93KSyy5NTAypUMEmQcmbN3k/e3TqVBcdHR3c3b25fPk+Fy/eS3LZpYmJocYDqYsVK4i9fVk8PzE7F8/f7yplyliTP3/S7/6+XLn0E73fipXDiImJTZTCf8+eUyz+3x5GjOxA3brlP9quQ91y+PtfTbfkLOlBURQ8PXxo2rQqZcta07vXbIKCQtK9n8jIaLZu8aNDB3sqVSr27rO1KyrLLUWKfBkBXWgYBkay5FIIIYTIDr7+eSSxsTFsmzYzTe2UsLXizp2nPHv2mgMHzmpcdtm0aVWePF1N69Y1NbZhZ1eUFy+CefLkFR7uPlSuXJxy5YokuMfc3JTLVxZy4eKCBOXmrSXY2Fh8cpxdnOtx6dJ9Ll26D6iW4jk52VG4sHmC+woVMuPW7SW4e4xO3EZcwhYPD59E1zSJT0LyscOt37di5bBE7+fkZMfQIYs1Zv4cMWIZd+8+Y978fkm2mT+/KRUq2OCXzIQomWnDBtWh5fPneXHw4PlP3J22fkxNjTh7bp76c3VwKMfVq5+eZRUi3hcS0IWiL0suhRBCiM9epaYNKVfPgT3zl6izWqaW6nBm1T4kTw8fSpSwpEaNUurrefMas2z5UHR0dBLtY4pXwc6GixdVgdamTb7ExsYmWna5cFF/ChTIg5vrXDp3mkrnTlP5tttf6OnpsHzFUI0zavEKFzanXr3yeLh7q+vig7LOnRMmclm2fCgWFnno3NmRb79tkOBaF+d6HD16lbt3n37qYwHg1KmbREZGqfewfYypqRHt29dh0yZf9ft17jSVeo5jkkzhHxISzuxZ26lSpUSiADhep06qPX8ZlbEyLU6duol9nZH88MOyDO3n33/P0azphASfa+dOUxkzekWG9ityGEVRPrty4sQJBUi3MnjNYqXf4jnp2qYUKVKkSEl7URTlZFZ/52Snkt7fj59byWVirPxy0EsZ7rFS0dbRSXN7QcEeypw5fRVAyZPHWAmP2KL89Vdv9fU1a0coEZFblRs3FyvHT8zU2MbLVxuU+fO/V//9wL+TlctXFqn/3rXrV0qs4qWMHdsp0bOurk2VWMVLGTy4TZJjHDasnRKreCmlShVKUH/i5CzF/+hf6r9//30LJVbxUgYMaKn85z1Neflqg2JtnV8BlNKlCyuxipcybFi7FH0+vn7Tlf+8p33yvu++a6DEKl5K7dplUtR+wYJ5lajobcrEid9ovH7w0O/KhYsLsvzfnRQpn2tJ7nfklzFDFxIiSy6FEEKIz1yLId9jYm7Gpt/+IDYmJk1tWVjkIXduI27eVM3QBQWF8M8/p+jiXA8tLS06dapLt271mTzJA/cN3lSpUgIjI4MEbRQubE7evCZcvHhPXefh7kPZstZUrlycQoXMmL/ge/z8rjB9+pZEY1i2bB87d55g2h89kjwHr4uzI6dO3eTGjUcJ6j3cvalduwzFixfE1taK6X/1Zu/e0yxcuJtePWejq6vDsuVD0NLSwtm5HrGxsXh6Jm+5ZTx/vyvUrFkKPT3dj97n7OLE3btPOXbsWoraf/r0NYcPX9S4H7BQIdURC57JXCIqhEjalxHQhYbJkkshhBDiM2ZdviwOzl/ju2ETDy6lfU/Vu7O83qV+93D3wdo6Px07OrBw0QBOnLjB1Kkb8fO7gq6uDrVqlU7QRnxClPcDui1b/ImKiqZrVyeWLhuCvr4uPXvMIiZG8/EJffvMIzQ0gpWrhqGjk/DHrmLFClKnTtkEyy3jeXqqkpt8881XrFw1jMjIaNxc5wJw+/YTRo5YTpMmVenfvwXOLvXw8bnMo0cvU/QZ+fpeIVcufapVS/rMOHNzU5o0qYKHe+oCLw93b8qUsaZKlRIJ6jt3dow7YkECOiHS6gsJ6ELlHDohhBDiM9agVzfC37zln3n/S5f23p3l9S5boJfXcUJDI1izdgTGxgb06D6T6OgY/P1Vh1p/eNB2fEAXn6wE4MWLYA4cOMfQYe1o3rw6o0etUM8CavLkySv6f7+Q2rXLMHZspwTX4vfixQdv77t/PxA/vyuMG+9C3brlGTzobx4+fKG+vnjxHv755xQzZrpRvrxNqma6/PxUgfPHji/4+mt79PR0cdcQdCZHfAD8YYZRZ5d6nD59i+vXH6aqXSHEO19GQBcSSi4TObZACCGE+BzlNjejYqP6nNi+i4iQUAAMDQ044vsnN24uTlDGj3dJVpu2tpbExsZy5867gC4kJJydO09gYKDHTz+uVmcSfP06hEuX7ic6aLuCXVEePXrBq1dvE9R7uHtjYKDHvn1nWLRo9yfHsmmTL+vX/8cvE79J8C7jxjvj53eF+/c1J3/xcPfBwEBP/fyH3FznEhISTkxMDJs2+X5yHB96+vQ1t249xuEjRwt0ca7HtWsBnDt3J8XtA7x8+Yb9+88mOO7hYzOTQoiU+/ii6RzizYuXGJmaomtgQHRERFYPRwghhBDvqfV1G3T0dPHz3Kqus7cvg4NDOXbvPsmLF28AqFKlOCNGduDPPzcTERH10TZL2FoREPCCyMjoBPWTJ7lz6eI95s71SlDv53uZTp0d0dLSUp+JZvdehsv3bdzoS/nyRZg9e0ey33HQwEU8Dwwin1nuBPXLlu5L8plVq/7F2tqcP/7YrPH648cv6dxpGmXLWjAO/LoAACAASURBVBMYGJTssbzPz+8qjRtX1njN0jIfDRpUZPIkj1S1Hc/D3ZtVq3+gdu0yHDt2TX3EgqaZSSFEKmR1xq7MyOJVo21LZcYFfyW/jXWWZ6uRIkWKFCnvimS5lCyXWtrays97tyj9lsxNUD9+vIsSHbNdMTU1Utc1aVJViVW8lHbt6nyyXZ8jfyj/HpyS7HH06NFIiVW8lHLliiiAoq2trbwN2aTMmOGa5Z9RRpZ+/ZorsYqXUrx4wUTXBg1qneAzSW0xNTVSwsK3KLNmuSmAcur0bMXXb3qWv7sUKZ97kSyX73n1WLXcIq9lwSweiRBCCCHeV66eA2aFrPB/b3YOwN6hLBcv3iM4OFRdd/DgOQIDgzRmTfyQra0Vt9/bP/cpvr6XAdTnshUvXhAjI4MECVFyIj8/zfsHQbXc8ty5O1y58iBNfQQHh7J790k6d3GkbFlrqla1leWWQqSjLyKge/3kGQD5rCSgE0IIIT4n9l3aE/QskIuH3v2Ar62tjb19Wfx8E2a7jImJZdNGX9q0qZXoiIH3GRvnwtIyX4IMl59y8+ZjAgODsI8LbCpUsAHI8QHdpUv3CQoKSXTAeJEiFjg6lk+3wMvD3ZtChcyZO68fsbGxbNyY8j1/QgjNvoiALuipKqDLIzN0QgghxGfDrLAVZR3tObbFi9jod+fOlS9fhDx5jNWzR+9zd/fG2DgXrVvXTLLdd0cWJH+GDlRZH+MDm/gMl5cvp2126nMXGxvL0aPX1IFsvPh9bul1rMDOnScICQmnceMqeHtf4vHjlB2xIIRI2hcR0EVHRvLmxUvyWhbI6qEIIYQQIk6dTu1BUTi2aXuC+vigKn4Z5PuOHLnMw4cvcHZxSrLdd0cWJH+GDsDP9wqlSxcmf35TKtjZcPv2E0JCwlPURnbk53uFihWLMnOmGzNmuDJjhiv9B7Tk+PHr3LnzNF36CA2NwMvrOIAstxQinX0RWS5BtY8un6VlVg9DCCGEEICOnh61OrTm8n9HeB23kiaevUM5njx5pTGYUBSFjZ5H6D+gJaamRgn22MVL/Qzdu/1kdnZFc/xyy3heXscZOKgVvV2bqOtiY2P5deL6dO1n4YJdVKhgI8sthUhnX0xA9/rJMwoUs8nqYQghhBACqNS4PrnNzfDz2JroWt265fD9YP/c+9zdvRk2vB3t29dh9eqDia7b2lrx/HkwQUEh/2fvzsOiKv83jr9Zh0UQBcVdFBG3NPc9y9IW2y0zKzMr282yzLKy0sqyxcp+lZXfsiy11FJbzEzLXMF9RwUXVFQWRdmX8/tjBkIBRWA4zHC/rutcDuc8Z+YGp8YPz3ZRmSIj95CZmcXll19CixYN+GVRxEXd76g2bYqmTvA9dn+df//dQbu2T9j9dUSqmiox5BLg5NFjBGhRFBERkUqh+6BbiD8US9TqdWedDw4OIDS0LqtXFV/Q5Q0FLLhZdUFNQ+tc9HBLgIyMLNav38edQy7Dw8O9yvTQiYhjqzoFXdwxvHx98fKrZnYUERGRKi24aQihndqz5oef8jfxzpO3fP75eugA5sxeQb9+lxIY6F/oWmho3Ysebpln1cqdBAfXAJx/hUsRcQ5VpqBLirOOw9fWBSIiIubqdtvNZGdlse6nXwpd69GjJenpmWzcuO+8zzFr1j94eLhz663dzzrv7u5Go0a1iC5FDx1YV7oEyM7OYffuw6V6DhGRilRlCrqTtoIuIFgFnYiIiFncLRY63XQtW5csIyXpZKHrPXq2JCJiD5mZ2ed9ns2bY9i1K7bQsMvGjWvj7u5W+h46W0G3Z88RMjKySvUcIiIVqQoVdNYVtDSPTkRExDyXXn0lPv7+rPrhp0LXLBYPOnYMPe/8uYJmff83V1xxCV26NM8/998Kl6XroTt27CTbtx8kImJPqe4XEaloVaagOx2fQE5WNgHaXFxERMQ03QfdzLHo/URHbix0rVOnMDw9PS44fy7P++//zKFD8Xw94ym8vS0ANGuWtwdd6XroAPpe8QJPPP5pqe8XEalIVaagM3JzOXX8hObQiYiImKRu82aEtLuE1UX0zgH06NECgNWrd5Xo+U6fTuO+YVMID2/ApEn3AtYFUVJTMzh6NLHUOU+cOMXp02mlvl9EpCJVmYIOICkuTj10IiIiJul++81kpWcQueC3Iq/36NmS3btjiY9PLvFzLl++lQ+m/MwTI2+gb9+2NA2tQ3R06XvnREQcTZUq6E7FHSegTm2zY4iIiFQ5nt7edLz+GjYtXkpactEFW48eLVm1qmS9cwU9//wMdu2KZfr/RnHJJSGlnj8nIuKIqlRBl3T0GNWDa+Pi4mJ2FBERkSql/XX98Krmy+of5hd5PSysHrVqVWdVCefPFZSensm9Q9+jXr2aNG1ah+gyzJ8TEXE0VaqgOxl3DHcPD6oF1jQ7ioiIiNPq2bMVEZHvU726b/657rffzJGovRzYvK3Ie/I2FF9VwhUuzxURsYc33/gBKP0KlyIijqhKFXRJR7W5uIiIiL3163cpHTs245ZbugHQ8rKeNGzdklWz5hV7T8+eLUlMPM2uXbGlft0JE2Yx+ukvmD3731I/h4iIo6lSBV3+5uJaGEVERMRumtr2grtj8GW4e3py83OjiNsXw9r5C4q9p3uPlqxevQvDMEr9utnZObz//s8kJJR8URUREUdXNQs69dCJiIjYTWiodS+4K69sx02PDSOoUQN+fut9crNzimxfo0Y1WrduVKr5cyIiVV2VKujSkk+TkZqqHjoRERE7Cg2tw9q1u3F3d+OBkQPZ8udyolZHFNu+W7dwoPTz50REqrIqVdABnIw7Tg0VdCIiInZRrZo3tWsH8NP8NRyOT6NlYA4LJn9w3nt69mxFdnYO69btqaCUIiLOo+oVdEe1ubiIiIi9hNrmz53BiwM5gTTwzcabzPPe071HCzZujCYtLaMiIoqIOJWqV9Bpc3EREZFy5VXNlwatwqnTrCmXdm0DQGj/64iIOoWrqwu3396r2Hvd3d3o2jWcVSt3VFRcERGn4m52gIqWFHcM/1pBuHl4kJOVZXYcERERh3f35Am07NUdgE5BqUAqHnUaM23kC/R++xbuGNybDz4oeoXLdu2a4ONjYdWqXRWYWETEeVTBHjrrSpfVg2uZnERERMTxeXhZCOvSkU2//8nXT7/AiU3rSDqVxv89/Cxbl/7NnNkr6NatBSEhRU93KOuG4iIiVV3VK+jyNhfXPDoREZEya9K+Le6enkT8/AtblizD3z2b3TsPsHPFKgBmz14BwKBBRQ+77NGzJQcOHOfw4YQKyywi4kyqXEGXpM3FRUREyk1Y105kZ2URvX4zAM2a1WXfvrj86wcOHGf16l3cMbh3kff36NGCldp/TkSk1KpcQXcy7jigzcVFRETKQ1i3zhzYso3MtDQ8PNxp2DCI6AIFHcDsWf/Qvn0o4eENzjrfsGEtGjasxWoNtxQRKbUqV9BlZ2RwJjFJPXQiIiJl5O3vT/2W4exZEwlASEht3Nzc2Lfv6FntfvhhJdnZObwwbtBZ53v0aAGgHjoRkTKocgUdWIdd1lAPnYiISJk069IBV1dX9qyOACA0tC7AWUMuAY4eTeSN1+dwzz1XcMst3fPP9+zZkjNn0ti6dX+FZRYRcTYlKeimA8eBbQXOTQZ2AVuA+UBAEfd5AeuAzcB24NUyJS1Hp+KOqYdORESkjJp360x6SgoHt1v3kMvbVPzcHjqAiRNns379Xj797DFq17b+s6F7j5asWbObnJzcigstIuJkSlLQfQVcc865JUAboC0QBTxfxH0ZQF+gHXCp7Tm6lTZoebJuLq6CTkREpCzCunVmX8RGcrNzAGsPXUpKOnFxSYXaZmfnMPSe9/Dz8+azaY/h6+tFu3ZNWK3950REyqQkBd0/QOI55/4Asm2P1wANKMwAztgee9gOoxQZy13S0WN4+1XD4utjdhQRERGHVKNuHWo1bsieNRH555qG1iE6Oq7Ye3buPMS4F2Zw003dmPrxw7i7u7Fy5Y6KiCsi4rTKYw7dcOC3Yq65AZuwDtlcAqwth9crs5PaukBERKRMwrp2AmDP2sj8c6GhdQvNnzvXlCkL+Pvvbdx775Xk5uaydm2UXXOKiDi7shZ047D21M0s5noO1uGWDYAuWIdpFmcEEAlEBgUFlTHW+SUesY7tD2pUVMeiiIiIFMfDw52duz7hnvv6kxyfQNzeaABcXFxo2jSY6CLmzxVkGAb3DZvC6dOpbN9+kFOnUioitoiI03Ivw73DgOuBK7nwUMqTwDKs8+i2FdNmmu0gPj7erkMz4/ZEk5ubS73wMLYvW2HPlxIREXEqTZvWITy8Ad514IVP/htuWbduDby9LRfsoQPYv/8Y11w9XouhiIiUg9IWdNcAY4A+QGoxbWoBWViLOW+gH/BWKV+vXGWmpRF/4BD1WzQ3O4qIiIhDyVvJslF1OLV3Z4Hz1i0L9u49fw9dntWrtRiKiEh5KMmQy++B1UA4EAvcD0wF/LDOi9sEfGprWw/41fa4LtZeuS1AhK3tovIKXlaHd0VRLzzM7BgiIuL4rgF2A3uBsUVcHwacwPp5uQl4oMC1e4E9tuNeu6YsJ3mFG0C7Bp4Fzhe/ZYGIiNhPSXro7izi3JfFtD0CXGd7vAVoX5pQFeHI7j20v7Yf3v5+pCWfNjuOiIg4JjfgY6yjUGKx/gJzAXDu0o2zgcfPOVcTGA90wjp1Yb3t3sJr/lcizZrVJS0zh8Q0uPG69rzx6re28/XIzs7h4METJicUEalaymOVS4d0eKd1Va16zZuZnERERBxYF6w9c9FAJjALuKmE916NdfRKItYibgmF932tdEKb1eVUljtLV++nS5fmNGliXTG6aWgdDhw4TrZtTzoREakYVbeg220t6Oq3DDc5iYiIOLD6wKECX8fazp1rINaRKz8CDS/y3kqlResQTmd78O3XSwC4447eQMm2LBARkfJXZQu6MwlJnDp+QvPoRETE3hYCIUBbrL1wX1/k/RW2rc+FuLq60rB+TRJT4e+f/mTVqp3cMfgywDqH7kJbFoiISPmrsgUdWBdGqd9SK12KiEipHea/Hjew7rt6+Jw2CUCG7fEXQMeLuBesW/p0AjrFx8eXNW+Z1K8fiIe7K7t3HSArPYPZs1bQrl0TunULp2ZNP/XQiYiYoEoXdEd27SG4SQjunp4XbCsiIlKECCAMaAJ4AoOxLmxSUN0Cj28E8tb6Xwz0B2rYjv62c5VW58usa51F/LMRgB9++Jfc3FxeGHcHoBUuRUTMUJaNxR3e4V1RuHm4U6dZE2J37DY7joiIOJ5srKtXLsa64uV0YDvwGtZhkguAkVgLuWysC6AMs92bCEzAWhRiuyexgnKXSo9+3QH4e8FfAMTFJbF8+Vauv74zgHroRERMUMULuj0A1AtvroJORERK61f+24M1z8sFHj9vO4oy3XY4hEs6tSA7x2DHpj355+bM/pe+fdsBEB2tgk5EpKJV6SGXibGHST+Tonl0IiJSZXXv3gJ/f58LtqsWWING9WsSF3+G3Nzc/PNz564iOzuHuLgkUlLS7RlVRESKUKULOsMwOLJ7D/VbqKATEZGqx2LxYNnyNxg9+pYLtm3VuycBlhz27I4963xCQjLz5q1m7VqNdBERMUOVHnIJ1nl0XW65HhcXFwzDMDuOiIhIhQkM9MPT04O27UIu2Lb1Fb2o7p7N9s17C127a8hkcnP1GSoiYoYq3UMH1pUuLT4+BDZqYHYUERGRChUU5A9AmzaNz9vO3WKhba/OeHm4FLnwSU5Orn4pKiJikipf0B3eFQWgYZciIlLlBAZaC7rQ0Lr4+FiKbRfWpSO1q3sA2ppARKSyqfIFXdzeaLKzslTQiYhIlZPXQwfQqlWjYtu1urwXPkYqoK0JREQqmypf0OVkZ3NsXwz1WoSZHUVERKRCBQb65T9u06b4gq51n17kHLcuhhITc8zuuUREpOSqfEEH1mGX6qETEZGqJq+HLj09s9h5dA1ahVM9uBaeaUnExsaTnp5ZkRFFROQCVNBhXRjFPygQv8CaZkcRERGpMIGBfiQnp7J9+0FatS66h65px/YA1PQtekEUERExlwo6CiyMog3GRUSkCgkM8ic+Ppnt2w8W20NXt3koyfEJhDQKIloLooiIVDoq6IAju/cAUL9FuMlJREREKk5goD8JCafZvu0ADRoEERDgW6hN3ebNiN+3j3r1AtVDJyJSCamgA9LPpHAsej+hnS41O4qIiEiFCbL10G3bdgCA1q3P7qVzdXOjTmgTjERrz5y2LBARqXxU0NnsWP4voV06YvH1MTuKiIhIhQgM9CMhIZlt2w4ChVe6DGrUAA+LBa/MU4C2LBARqYxU0Nls//tf3D08CO/ZzewoIiIiFSIoyJ+E+GQOHTpBcnJqoXl09cKtW/r4u2UBsHfvkQrPKCIi56eCzubA5m2kJJ2kdZ9eZkcRERGxOw8Pd/z9fUhIOA3Atm0HaH1OQVe3eTNysrKpHWAhMfE0J0+mmBFVRETOQwWdTW5ODjv+WUXLy3rg6uZmdhwRERG7yttUPD4+GYDt2w4U7qFr3ozj+w/QpEltDbcUEamkVNAVsH35CnwDqhNy6SVmRxEREbGrvIIur4du+/aDBAX5U7t2QH6bus1DObJ7D6GhdbQgiohIJaWCroCoVevIzsyk9eW9zY4iIiJiV4GB/sB/PXR5K13mLYzi7e9Pjbp1OLZ3H40b1yZaPXQiIpWSCroCMlJT2btuA60v1zw6ERFxbkFB1oIuISGvoMtb6dI67LJu81Brw+R4PDzc1UMnIlJJqaA7x/blK6gV0ohaIY0u3FhERMRBnTuH7vjxk5w4cSq/oKtnK+h8jVRAWxaIiFRWKujOsWP5vwAadikiIk7tvx660/nnCq50WTesGSlJSQy9sycZGVns2HHQlJwiInJ+KujOcfLYcQ7vjNKwSxERcWqBgX6cOZNGRkZW/jnrSpfWESr1wsNokBPHrbf24OWXvs3vyRMRkcpFBV0Rti9fQcill+AbUN3sKCIiInYRGOR/Vu8cWFe69PPzoXFIMGGtQri9awD//ruDd9/9yaSUIiJyISroirB9+Qpc3dxo0buH2VFERETsIjDQv1CvW97CKD2u6MCA0CxcgGH3vk9ubq4JCUVEpCRU0BUhdsduTh07oWGXIiLitIKC/IroobNuXfDKS4NoXC2LCW/9THS0FkMREanMVNAVY8c/Kwnv0RVXNzezo4iIiJS7onroTp5MITY2nrAmQcQku/PupFkmpRMRkZJSQVeMfREb8Krm+98+PCIiIk4kKMifxITCC51s2hRDSnoO360+SXZGhgnJRETkYqigK0bMhs0ANO1wqclJREREypebmys1alQrcuXKhx+ayrR1OezesseEZCIicrFU0BXj5LHjJB4+SpMO7cyOIiIiUq5q1rRuKn7uHDqAxOQMqFmPo1H7KjqWiIiUggq684jesImmHdVDJyIiziUw0FrQFdVDV6eZdarBkai9FZpJRERKRwXdecRs2IxfYE2CGjc0O4qIiEi5CQryB4ruoasX3gyAoyroREQcggq684hevwnQPDoREXEugYHWgq6oHrq6zZuRmpzMybhjFR1LRERKQQXdeRyPOcCZxCSadtQ8OhERcR7n66Gr1bghx2MOVHQkEREpJRV0FxCzcYsWRhEREadyvjl0AXWCOXlUvXMiIo5CBd0FRG/YRFDDBvjXCjI7ioiISLkICvInLS2DtLTC+8wF1KnNybjjJqQSEZHSUEF3ATHr8/ajUy+diIg4h8BAvyJ756rVrIGHxULS0TgTUomISGmooLuAw7uiyEhNpYm2LxAREScRGORf5Py5gDq1AdRDJyLiQFTQXUBuTg4HNm9TD52IiDiNwEC/Ygq6YACtcCki4kBU0JVA9IbN1AkLxcuvmtlRREREyiwoyL+YBVFsPXTHVNCJiDgKFXQlEL1+E66urjS5tK3ZUURERMosMNCfxCJ76OqQlZFBSuJJE1KJiEhpqKArgYNbt5OdlaX96ERExOG5urpSs2a1YnvoTh07gWEYJiQTEZHSUEFXAlnpGcTu2EWTDloYRUREHFtAgC+urq4kJBSzB53mz4mIOBQVdCUUs34zDdu0xN1iMTuKiIhIqQUF+QPFbSquPehERBxNSQq66cBxYFuBc5OBXcAWYD4QUMR9DYFlwA5gO/BkmZKabG/kBtw9PLjivrvMjiIiIlJqgYF+AIVWuXR1c6N67VokxWkPOhERR1KSgu4r4Jpzzi0B2gBtgSjg+SLuywZGA62AbsBjtscOafe/a4hc+BvXPPYgve8aZHYcERGRUimuh84/KBBXNzf10ImIOBj3ErT5Bwg559wfBR6vAW4r4r6jtgPgNLATqI+1x87hGIbB7Jdex9Pbm5vHPkVmWhpr5y00O5aIiMhFKa6HTnvQiYg4pvKYQzcc+O0CbUKA9sDacng90+Tm5PDtmJfZ+e9qbhs/lvbX9jM7koiIyEUprocufw869dCJiDiUshZ047AOrZx5njbVgLnAKKDwDOz/jAAigcigoKAyxrKfnKwsvn7qeWI2bObON16mRa9uZkcSEREpscBAfzIzszhzJu2s8wF16wBw8qjm0ImIOJKyFHTDgOuBu4DiNqzxwFrMzQTmXeD5pgGdgE7x8fFliGV/WekZfPnYM8TtjWbwxJeoVrOG2ZFERERKJCjIv9gVLtPPpJB+JsWEVCIiUlqlLeiuAcYANwKpxbRxAb7EOnfuvVK+TqWVkZrKzLGv4FXNl9vHP2d2HBERkRKpGehXaP4caA86ERFHVZKC7ntgNRAOxAL3A1MBP6yrXW4CPrW1rQf8anvcE7gH6Gtrswm4rryCVwbH9sXw64ef0qZvHzrf5FTfmoiIOKnz9dAlqaATEXE4JVnl8s4izn1ZTNsj/Fe0/Yu1l86prfhmNq0v783NY59m77oNJGnugYiIVGKBgX7s2HGo0PmAOsHE7txtQiIRESmL8ljlskozDINZL04AFxg88UVcXJy+hhUREQcWFORPwjk9dO6envgF1tQKlyIiDkgFXTlIOhLHz5Om0KxLR3pp03EREanEatYsPIeuerB1y4JTGnIpIuJwVNCVk3U/LWL7shUMGPUIdcJCzY4jIiJSSPXqvri7uxW7B13SURV0IiKORgVdOZrzypuknT7D3W+9irvFYnYcERGRs+RtKp6QcG5BFwygVS5FRByQCrpydCYxiVnjJlA3LJQbnn7M7DgiIlIxrgF2A3uBsedpNxDrvq2dbF97AF8DW7Fu8fO8HTMC1gVRgEI9dDXq2gq6YyfsHUFERMqZCrpytnvVWv7+Zha9htxOy949zI4jIiL25QZ8DFwLtMK6MnSrItr5AU8Cawucux2wAJcAHYGHgBA7Zi3QQ3f2HLqAOsGcSUwiOyPDni8vIiJ2oILODn55//84vCuKOyaMwy+wptlxRETEfrpg7ZmLBjKBWcBNRbSbALwFpBc4ZwC+WLcQ8rbdX3iDuHLk5uZKbGw8J06cOuu89qATEXFcKujsICcri5nPjcfL15fBE1/SVgYiIs6rPlBwU7dY27mCOgANgV/OOf8jkAIcBQ4C7wCJ9olptXDhOho1vI+YmLOLt4A6wVrhUkTEQamgs5Nj0ftZ8M6HtOjVjUuuutzsOCIiYg5X4D1gdBHXugA5QD2gia1N0yLajQAigcigoCC7hAyoE6w96EREHJQKOjta/cNPHI85wJUP3Gt2FBERsY/DWHvf8jSwncvjB7QBlgP7gW7AAqwLowwBfgeygOPASv5bMKWgabbzneLj48s1PIDF1wdvv2pa4VJExEGpoLMjIzeXv6Z/Q4NW4bTo3d3sOCIiUv4igDCsPWyewGCsBVueU0AQ1sVOQoA1wI1Ye9wOAn1t7XyxFnu7KiDzWfK2LNAedCIijkkFnZ2tX/Q7iUeOctWDw8yOIiIi5S8beBxYjHXrgTnAduA1rIXb+XwMVLO1jwD+B2yxW9Ji/LcHnYZciog4InezAzi73Owclv9vJreOe4amndoTHbnR7EgiIlK+frUdBb1cTNvLCzw+g3XrAlMF1KkNaFNxERFHpR66CrB2/iKS4xO46kHNpRMRkcqlRt065ObkkHyi/OfniYiI/amgqwDZGRn8M+N7wnt0pWHrlmbHERERyRdQpzbJJ+LJzckxO4qIiJSCCroKsmr2fFJPJXPVCPXSiYhI5REQrC0LREQcmQq6CpKRmsqKmXNo07cPdcJCzY4jIiICWHvoNH9ORMRxqaCrQCtm/kB6SgpXDLvL7CgiIiKANhUXEXF0KugqUFpyMuvmLaL9tf3wr13L7DgiIlLFeXp74eFl4UxSktlRRESklFTQVbAVM2fj4upC7yG3mR1FRESqOE8fbwAyUlJNTiIiIqWlgq6CJR4+ytalf9Pt9pvx9PY2O46IiFRhFm8fQAWdiIgjU0FnguVff4ePvz9dbrne7CgiIlKFWXxtBV1qmslJRESktFTQmeDglu3EbNzCZffcgYur/gpERMQcFtuQy8w09dCJiDgqVRMm+fvr7whsUJ9LruxjdhQREamiPH3UQyci4uhU0Jlk27IVxB+Mpc+9dxa65u3vb0IiERGpavKHXGoOnYiIw3I3O0BVZeTm8s+3s7n1hdGEXNqWnKws2l19JW37XUFAndpMe2gUe9etNzumiIg4sbwhlxmpKuhERByVeuhMFPHTIlJPJfPI9KmMmjWd3ncP4lh0DElH4xj06gtaBVNEROwqfw6dhlyKiDgs9dCZKDMtnUXvTaVVn55s++sfti1bQVryaZq0b8ujX33CgKceZf4b75odU0REnJSnbduCdA25FBFxWCroTLZ23kLWzlt41rmYjVtYMXMOfe4ZzOY//iI6cqNJ6URExJlZfH3IycomJyvL7CgiIlJKGnJZQIB/7wAAIABJREFUSf324afEH4zljtdewNPby+w4IiLihCw+3mRoywIREYemgq6SykrPYPbLrxPUsAHXjnzY7DgiIuKELD4+WuFSRMTBachlJRa9fhMrZs7hsrvvwMvXl/QzKWRnZpCVkUn0+k1aBVNERMrE4uujPehERBycCrpK7tcPPqF2k8a06NUND4sFd4snHhYLOdnZfPn4s+xeucbsiCIi4qA8fby1wqWIiINTQVfJZaalM+2hUWeds/j68NhXnzD0nYlMvfdhjkbtNSmdiIg4Mou3t/agExFxcJpD54AyUlL54rFnSE9J4YH/exf/2rXMjiQiIg7IOuRSBZ2IiCNTQeegko+f4ItHR+NVzZcHpr6DxcfH7EgiIuJgLD4+GnIpIuLgVNA5sKNRe5kx+kXqhDXlnncn4uFlMTuSiIg4EE8fby2KIiLi4FTQObjdK9cwd+JkWvbqztNzvqZhm1ZmRxIREQdh8fHWtgUiIg5OBZ0TWDt3AZ/c/zgeXhae+OYzrn7sQVzd3cyOJSIilZiLi4t1HzrNoRMRcWgq6JzE3nXreefWu9nwyx/0f3g4I2d+od46EREplqe3N4Dm0ImIODgVdE4k/UwKs16cwFejxhIQXJtR33/JA//3Lo0uKVzY+dYIIKhxQxNSiohIZeDpYy3oNIdORMSxaR86J7R16d9ErY6g550DufzeITz53Zfs/Hc10ZGbaNAqnIatW1Kzfl1yc3P5cMgDHNq+0+zIIiJSwSy+1tWRM1JTTE4iIiJloR46J5WRmspfX37D69cM5Jcp/0ej1i0ZMOoR6rdszoEt21j47lRST57i2idGmB1VRERMYFEPnYiIU1APnZPLK+z++WY2Hl4W0pJP518zcnO58dmRNO14KdHrN5mYUkREKlre/qWaQyci4tjUQ1dFZGdmnlXMAaycPY9Tx09wjXrpRESqnLw5dOnatkBExKGpoKvCsjMy+HPaV4R2bE94j65mxxERkQrkld9Dp4JORMSRqaCr4tbOXUBC7BH10omIVDGePnmLomjIpYiII1NBV8XlZGez5NMvadSmFW369jE7joiIVBAtiiIi4hxU0AnrFy3meMwBrnn8QVxc9ZYQEakK8rYt0JBLERHHpn+9C7k5OSz++HPqhoXyxDfTuOm5UXS84VrqNGuqAk9ExElZfLzJzswkJzvb7CgiIlIG2rZAANj8x1/U+vhzwnt0peutN3LZ3dahOHH7Yvh2zEscjdpnckIRESlPnj4+Gm4pIuIEStL9Mh04DmwrcG4ysAvYAswHAi7iXqmEDMNgyafTmTr0IcZ1v4q3bhzM7Jdex8ffjye/+5Lut99idkQRESlHFh8fMrRlgYiIwytJQfcVcM0555YAbYC2QBTw/EXcK5WckZvL8ZgDrPtpEe/eNpR9ERu57eUxDH33dbz8qpkdT0REyoHFx5sMzZ8TEXF4JRly+Q8Qcs65Pwo8XgPcdhH3igM5k5jEF48+TZ+hd3Ldk4/QvFtnUpNPYxi5YFjn38Xu2MWulWuJWrWW0wmJZkcWEZESsPh4k6khlyIiDq885tANB2aXw/NIJWUYBsu//o596zfRbeCNuHl44OLqgqurK+4WC2HdOtNhwNUAHN4VRdTqCPZGrCdmw2YN5xERqaQ8fXzISFNBJyLi6Mpa0I0DsoGZ5ZBlhO0gKCioHJ5OytuhbTs4tG1HofMuLi7UCw8jvGdXwnt0pfddt3PFfXfZeu92s3Xp3/z15QwTEouISHEsvj6cSUwyO4aIiJRRWQq6YcD1wJWAUQ5ZptkO4uPjy+P5pIIYhsHhXVEc3hXFX19+g4eXhcZt2xDauQPNu3dmwKhHOHX8BOsX/lbk/e4WC0ZOjpbOFhGpQJpDJyLiHEq7ydg1wBjgRkCfBnKWrPQM9q5bz+KPP2fqPQ+xf9NWbhj9eJELqnj5VeOZud/wyJdTteediEgF8vTWHDoREWdQkn9Bfw+sBsKBWOB+YCrgh3W1y03Ap7a29YBfL3CvVCGGYTB34mR8A6pz7RMPFbp+x2vjCGxYnyYd2tH7rkEmJBQRqZq8fH01z1lExAmUZMjlnUWc+7KYtkeA6y5wr1QxR3bvYeWsufQcPJB18xdyeGcUAL2G3E7bqy5nweQPCe3cgWufeIjty1aQEHvY5MQiIs7N1c0NDy+LFkUREXECGuMmFeL3qdM4k3SSgeOexcXFhYZtWnHDM0+wfdkK/p7xPXMnvk1Odja3vzLW7KgiIk7P09sLQHPoREScgAo6qRDpZ1JY+O5HNG7XhsuHDeGeyRNIPh7P9y9OBODUsRMsem8qYV070XXgjSanFRFxbp4+PgAaciki4gRU0EmF2bBoMXsjNnD9048TEFybb559kbTk5Pzra378mT1rI7lh9BNUD65lYlIREefm5Wst6LQoioiI41NBJxVq3uvvkHLyFD9P/oCDWwvvaffDK5Nwc3dnyBvjaXlZT/yCAs+67lsjgJa9e9D/4eGEdetcUbFFRJyKp7c3ABkq6EREHF5ZNxYXuSjH9sXwyuUDyM3JKfJ6Quxhfn57CgNffJZmXToCcOr4CY7tiyGwYQMCG9TLb5uRmsoHd97Psej9FRFdRMRpWHzyCjoNuRQRcXQq6KTCFVfM5Vnz489s+OUP6rcIo0GrFjRo1YLg0BAObd/JqllzObhtB8kn4nl8xmcMfe8NPrjzfjKLWKnN3dOT7MxMe30bIiIOy+LrC2gOnYiIM1BBJ5VSZloaMRu3ELNxS7Ftvnn2JR7+/EMGvTKWb58bn3/exdWVKx8YSv+H7ycrM4MTBw5xYv9BTsQcYMNvS4g/cKgivgURkUorr4euqF+GiYiIY9EcOnFY+yI28PvUz2l/XX96Dh4IQPXgWjz8xUdc+8RDbFv2DxE//UJK4kkat21Dv0fu57GvPiEguLbJyUVEzOWpIZciIk5DPXTi0P76cgaN27XhxjFP4unjzRX33Y27pwffj5tA5IJfz2ob3DSEJ779nOEfTWbqvQ+RmZZuUmoRcTLXAB8AbsAXwKRi2g0EfgQ6A5G2c22BzwB/INd2ze7/c7L4atsCERFnoR46cWiGYfD9uAmcOnaC6596jMQjR3lv0LBCxRzAsej9fDvmJeo2D+XON8bj4uJiQmIRcTJuwMfAtUAr4E7bn+fyA54E1hY45w58CzwMtAYuB7LsmDWfxbYPnX6xJSLi+FTQicNLS07mi0efZv6b7/HR3SPOO0du179rWPDOR7S96nKufvzBCz73lQ/ey2X3DM5f4ltE5BxdgL1ANJAJzAJuKqLdBOAtzu596w9sATbbvk4Azr9qVDmxeHuTlZ5xwUWqRESk8tOQS3EKx2MOcDzmQInarvh2NsGhIfQbcR/Ho/ez4Zc/imw3YNQj9L1/KABXjRjGiplz+Pe7H8/aDF1Eqrz6QMHfIsUCXc9p0wFoCPwCPFvgfHPAABYDtbAWg2/bLWkBnj7emj8nIuIk1EMnVdL8199l77r1DJ74EpcNHVzoeu+776Dv/UNZNXseH9z1ADEbN3PNYw/y4h/zuPLBe8/73N1uu4nON12Hu8Vir/gi4jhcgfeA0UVccwd6AXfZ/rwFuLKIdiOwzrmLDAoKKpdQFl8fFXQiIk5CBZ1USTnZ2UwfOYbty1Zw07NPMvTd1/MXCWh/bT9ufm4UW5YsY94b73Jwy3b+N/I5Jt96N3vXRnLdyIdp07dPkc/bsncPbh8/lsETX2L80gXcMPoJAhs2qMhvTUQq1mGsvW95GtjO5fED2gDLgf1AN2AB0Alrb94/QDyQCvyKtTfvXNNs7TvFx8eXS2iLjw8ZqdqyQETEGaigkyorIyWVr59+gYXvfESbvpcx6vvpdB90C4Nff4m9ERuYOfYVjNzc/PZxe/bx9ehxHN4ZxcAXn8Hb3++s5/Oq5stt45/j6J59fPrgSKLWRND7rkG88OsPDHlTi7CIOKkIIAxoAngCg7EWbHlOAUFAiO1YA9yItcdtMXAJ4IO1t64PsKMiQlt8vMlUQSci4hRU0EmVt/zr7/j0wZF4+VXjtpfGcDx6P/8bOYbszMxCbXOzc5j98uv41gjgxmdHnnXt+qcfxz8okNkvv8GeNRF888yLTOh/M8u//o6O119Dr7sGVdS3JCIVJxt4HGtxthOYA2wHXsNauJ1PEtbhmBHAJmAD1nl2dmfx8SEjJaUiXkpEROxMi6KIANGRG3l/0DB63HErK2fNJf1M8f/QObwrimXTv+WqEcPY9PtSdq9cQ1jXTnS//WaWTf+WQ9v++wX76fgEFr7zEbUaNWTAqEfYvXJNocVbXFxc6H33HSQdOcrWpX/b7XsUEbv51XYU9HIxbS8/5+tvbUeF8vTx5tTxExX9siIiYgfqoROxST4Rz+9Tp3E6PuGCbZd89j/i9sVw+/jn8AsK5PZXxnJi/0F+/78vimz/w6uTyExN4843XsbV3S3/vKubG4MnvsRNY55k2JRJjJo1nebdu5Tb9yQiUhTNoRMRcR4q6ERKITszkznj36B6cG2e/uFrAhvUZ/b4N8jOyCiy/emERH6c8DaN2rTiygesq2S6eXhwz+QJdLrxWn776DO+HzcB34AAHpr2AY9M/5hGlxS1N7GISNlZfLzJTFNBJyLiDFTQiZTSgc3bWDFzDv5Bgfz73Q/EbNh83vZblixjwy+L6TfiPpq0b8t9H75F235X8NOk9/lz2ldELviVSTfcwbw33qV2k8Y88c00+j88HBdX/WcqIuXL4qs5dCIizkJz6ETK4NcPPuXIrj1s/mNpidrPe+NdQjt14NH//R+4uDD75TdYN39h/vWcrCxWfv8jkT//yq3jnuHqxx4ktEtHvnv+FU4d03wXESk7N3d33D09NeRSRMRJ6Ff/ImWQnZFB5IJfyUoveqjludKSTzPrpQmcSTrJzOfGn1XMFZSRmsr3417juxdeo2HrFoz+8Rta9elVntFFpIry9PEGUEEnIuIkVNCJVLCo1RG8esX1bPr9zwu2Xb/wN94bNIykI3HcP3UyIz59n8bt2lRAShFxVhYfH8C6F6eIiDg+FXQilVz8gUN8ePeDLHpvKvVbhjPy288Z8dkUQtpdYkoei68P3v7+pry2iJSdxdZDl5mqgk5ExBloDp2IA8jJymLZ/2ayctZcegy6lcvvu4snvp3G4V1RbFv6N1v/+pujUfvsnqNZl47cNekVvKpVY9XseSz76lvOJCQVaufq7kZudo7d84jIxfPM66HTkEsREaeggk7EgWSmpbP86+9YNWceXW65nnb9r6TfI/dz9WMPEn8oll0rVhOzYTMxm7bkL6Li5uFB0w7tCO/RlSYd2rFzxSr++vIbcnNKXnC5urnR7+HhXDViGCf2HyRqTQSX3XMHPe64lVWz57F23gLqhIUS2qk9oZ3aE9w0hLXzFjJ34mSM3Fx7/ThEpBS8fG0FnbYtEJFKrEaNGowaNYqQkBBcXFzMjmM3hmGwf/9+pkyZQlJS4V+Sl4SLYRjlHKvsIiMjjc6dO5sdQ8QhVAusQevLe3PJlX1o2rF9/nCqpKNxxB+MpdElrbH4eJOdmcnx/Qep17wZMRs2M3PsKyQdjbvg81cPrsVdk14ltFN71v20iPlvvEtmWjpBjRpw1Yj76Hj91bi6WTdLz0hNZf/GLaScPEWHAVezafFSvnv+VXKysuz6MxDHZRjGeqCT2TkcRXl8Pra+ojfDP3yb9wbdy+GdUeWUTESkfL366qtERkby66+/knMRv4R2NG5ubgwYMICOHTsyfvz4s66V9DNSPXQiDu5MQhJr5y5g7dwFuLq7Ua95GCGXXkKT9m0JbNSAiJ9/Yde/a9gXsYHMtDQ6XH81A8c9y+gfZ/DjhLfZ9Puf1G7SmGZdOtKsS0catArHw8sLD09P3D098fCykJGaynfPv8r6Rb/nv278wVhmvTiBPz//irCunTi8czexO3fnD7WM3bGbG58diXc1X7566nky09LN+hGJSAEWrXIpIg4gJCSE1157zamLOYCcnBx++eUXBg4cWOrnUEEn4kRys3OI3bGL2B27+Pe7H4pss2HRYvZv2spdb77CPZMncOu4Z/ANqA5A4pGjHNi0lbQzKWRnZpKdmUlWWjobfltC/IFDRT5f/IFDRV77e8b3pCWf5vZXxjLisw/48vFnSEs+XX7frIiUSt4cukwVdCJSibm4uDh9MZcnJyenTMNKVdCJVEGJsUf4eNgj9Ln3TuqENiV6/Ub2rFtPYuyRcn2ddT8tIu3MGe5+61WemfsNf077mnU/LdIQTBETeWnbAhGRC6pevTpDhgzhk08+uaj7fvnlF4YMGcKpU6fslKwwbVsgUkXl5uSwbPq3fD/uNdbOW1juxVyerX8u55Phj5N09Bi3vTyG53+ZQ/fbb8HNw8Muryci55e3sXhmuoZBi4gUJyAggEcffbTQeTfbugHFGTBgQIUWc6CCTkQqwP7NW5k69CE+GzGSk3HHue3lMYz56Tt8qle+/excXF3xrRFgdgwRu7H4+JCRmqYVaEVEzmPSpEmEhoayceNG1q1bxz///MPPP//Mjh07AJg/fz6RkZFs27aNBx98MP++mJgYAgMDady4MTt27GDatGls27aNxYsX4+XlZZesGnIpIhUmanUEUasjaNm7B/d9+BbXPfkIP772Vonudff0pNElrUg7fYYziUmknDxJbnYOFh8fgho3oFbjRgQ1bkj8/oNsXrKsVP9YdfPw4P6P3qZJh0v54K4HiNtj/739RCqaxdeHDG0qLiIO5KYxo6jXIqxcn/PIrj38/PaUYq+PHTuWNm3a0L59e/r06cMvv/xCmzZt2L9/PwDDhw8nKSkJLy8vIiIimDt3LomJiWc9R1hYGHfeeScjRoxg9uzZDBw4kJkzZ5br9wEq6ETEBDtXrGLFzDlcds9g1s5byKFtO87bvnpwLYZ/OJkGrcLPOp9+JgWvar6F2l+1Zx+LP/6crUv/LnEmF1dXhrw5nvCe3Ug7fYah70xkyuD7TFud079WEIMnvsjRqH38+sEn5GRnm5JDnI/Fx1sLooiIXKR169blF3MAI0eO5JZbbgGgYcOGhIWFsXbt2rPuiYmJYfPmzQCsX7+ekJAQu2RTQScipvjjky9pf20/Br74DB8MeaDYHrVGl7Tivg/ewtPbm1kvTSQjJRXfGgH41ayBT0B1kk/Ec2L/QU4cOERC7GFa9enFNY89yLApkzi0Yxdrf1yAX1BNajVuSFDjhlSvVYuNvy3hz8+/OmvVzYEvPsulV1/JgskfcmT3HkZM+4Bbnh/N7Jdfr6gfSb564WHc//E7+AZUJ7xHVxq3a8OMZ14k+fiJCs8izsfTx1s9dCLiUM7Xk1ZRUlJS8h/36dOHq666iu7du5OWlsayZcuKHE6ZkZGR/zgnJwdvb2+7ZFNBJyKmyEhJZcHkD7ln8gS6DbyJ1T/ML9Sm/XX9ueO1F0g+Ec+nD47k2L6YCz7v5sVL2frnctpf15+rH72f214eQ25uLiePHuPEgYMkn4jnsqGD6XzzAJZ8Op1Vs+fR/9EH6H77zfz5+df8PeN7AJZ+/jX9HrqPPesi2bBocf7zu7i60qbvZTS6pBWBDerbjnpkZ2Xx94zvWfn9XDLTzu798K0RwCVXXc6JmAPsi9x43vwte/fgnncmkHoqmQ+G3E9wkxAGvfYCT8/5im+efYl9ERtK8NMVKZ7Fx0c9dCIiF3D69Gn8/PyKvFa9enWSkpJIS0sjPDycbt26VXC6s6mgExHTbPr9T7rddhPXPfkwW/5cRkrSSQACgmtzxf330OvO29gXuZGvn3qelJMlXzEqNyeH9Qt/Y+Nvf1CzXl1Oxh0nOzMz/3rd5s248ZknuHnsU/S9/x78awWxas58fvvw0/w2f3zyJU07XcptL43h4NYdJB4+Qsfrr+HK+4dSK6QR2ZmZJB4+SkLsYfZv3kpQwwZc/9RjXH7vEJZ/NZNVc+YTcmlbut56A62v6I27bVXPzX/8xYLJH3Iy7thZmV1cXel1523c+OxIDu+K4svHn+V0fAJHo/ZxJGovw6ZM4uHPP2T9ot9xcXHF298PH38/srOy+OGVSSTEHi7LX4VUIRYfH1IqeAU2ERFHk5iYyMqVK9m6dStpaWkcO/bf5/bvv//Oww8/zI4dO9i9ezdr1qwxMSm4GIZhaoCiREZGGp07dzY7hohUgOCmIYz+8RsiF/zKyllz6XPvnVx69VXgAmt+/Jmf35pit/lj4T27ce3IhzgatZc5498sNOyzenAtRv8wg5STp3C3eFKzXl1id+zmz8+/Yttf/xRq36hta65+5AFa9OpGbk4Orm5upCSdJHLhb6xf+DstL+vBlQ/cC8Cfn3/F+gW/Edq5Ay16dye8R1d8A6qzdenffPf8K4Xm7ll8fLj1xWdo1acn6adTSD2VTFryaRq0bkHc3mg+HvaIQ65aaBjGeqCT2TkcRXl8Pj7703cc2xfDjNHjyimViEj5mzFjBkOHDjU7RoUp6vst6WekeuhExFTHovfzzzezuGL43XQdeCPpKSn8O+tHVnw7m6QjcXZ97d0r17B7ZfG/VTt17ATfvziR4R++xYEt25k74W12/Vt8+4NbtvP5I0/RuF0b2l/bj+gNm9m+bEX+RuqHd0WxfuHv3PDME1w38mGuG/kwAKcTEtnx90p2/LOSrX8uL7Iwy0hN5fsXXit0vv11/bn7rVe54r67+OvLbwpdb9WnFzXqBrNy1twL/jykarBoDp2IiFNRQScipvvj0+kE1A3m8M7drP7xZ9JPnzE7Ur6d/6zk5cuuPWsBlQs5sHkbBzZvK/Ja0tE4ZoweR1jXTtRvGc7edZEc3hlFaUdLbPz1D9r0vYyrH3uQnStWczRqb/61rgNv5LaXn8PV1ZX0lFTWL/ztgs8XUCeY4KYhJB2N43jMgVJlksrN4utDRooKOhERZ6GCTkRMl5mWxrdjXjY7RrEuppgrqT1rI9mzNrJcnmvexMk07dCOIW+OZ8rg4eRkZXHZ0MHc9OyT7PhnJZ5eXtz20hiO7I7iaNTZe+t5entxzRMP0bRDO2o3aYzFxweArIwMPn/4qQsu4iKOx+Jt3VhcREScg6vZAUREpGxSTp5izvg3qde8Gdc+PoL+j9zPTc8+yabFS/nqybF8M+Yl0k6fZtj7k/Dyq5Z/n3/tWjz61Sf0vmsQacmnWTtvIT++9jafjRhJQuwRhn9UeO8/cWxuHh64ebhrlUsRESeiHjoRESewc8UqVv/4E1cMvxuAdfMXMecV60IvZxKSmDH6RR6d/jF3TnyR/z05lgatWjD8o7ex+Pow/fFn2bli1VnPN+2hJ3l8xmc8+Mn7fDzsEQ2/dBJevtYeWM2hExFxHuqhExFxEgve/pCYjVtYNv1b5ox/46zFVfZv2sLC96bSpm8f7n77NR7/+lNysrL56O4RhYo5sC4I89mIJzEMgxGfTSGgTnBFfitiJ54+1k1tVdCJiDgPFXQiIk4iMy2NqUMfYtH7Hxe5yMqKb2ezafFS2l/bj9idu/lgyP3E7Y0u9vniDxxi2kOj8KpWjYemfUDtJo3tGV8qQN4cSc2hExE5v+rVq/PII4+U6t4nn3wSb2/vck5UPBV0IiJVyOyXJvLtmJf55P7HOZOYdMH2R3bv4cvHRuNbI4Cn53zNZUMH4+Kqjw5HlddDl6keOhGR8woICODRRx8t1b2jRo3Cx/YLtIqgOXQiIlVIZlo6G39bclH3xGzcwuSbh3Db+Oe46dknuaRvH2a9OJGE2MN2Sin2kj+HTtsWiIic16RJkwgNDWXjxo0sWbKE48ePM2jQICwWC/Pnz+eVV17Bx8eHOXPm0KBBA9zc3JgwYQLBwcHUq1ePZcuWER8fT9++fe2eVQWdiIhc0OmERP438jk63XgdNz83itFzv2HuxMkl2ttOKg9Pbw25FBHH8/77D9Du0qbl+pybN0Xz1FNfFHt97NixtGnThvbt29OvXz9uu+02unTpgouLCwsWLKB3797UqlWLI0eOcP311wPg7+9PcnIyTz/9NFdccQUJCQnlmrk4GjcjIiIlFrngVybfehfRGzaRknThIZtSuaQlJ7N71VpST54yO4qIiMPo378//fv3Z+PGjWzYsIEWLVoQFhbG1q1b6devH5MmTaJXr14kJyebkk89dCIiclFOHTvBF488bXYMKYV9kRu1WbyIOJzz9aRVBBcXF958802mTZtW6FqHDh247rrrmDhxIkuXLmXChAkVnk89dCIiIiIiIgWcPn0aPz8/ABYvXszw4cPx9fUFoF69etSqVYu6deuSmprKzJkzmTx5Mh06dCh0b0UoSQ/ddOB64DjQxnZuMnADkAnsA+4DThZx7zXAB4Ab8AUwqYx5RURERERE7CoxMZGVK1eydetWfvvtN7777jtWr14NwJkzZ7j77rtp1qwZkydPJjc3l6ysrPxtDqZNm8bvv//OkSNHKmRRFAzDuNBxmWEYHQzD2FbgXH/DMNxtj9+yHefe52YYxj7DMJoahuFpGMZmwzBaleD1iIiIMAAdOnTo0OHkh2EYkSX5XNChz0cdOnRUrWPGjBmmZzD7+y3pZ2RJhlz+AySec+4PINv2eA3QoIj7ugB7gWisPXmzgJtK8HoiIiIiIiJSAuUxh244UNS61fWBQwW+jrWdExERERERkXJQ1lUux2HtqZtZDllG2A6CgoLK4elEREREREScW1kKumFYF0u5Eus4z3MdBhoW+LqB7VxxptkO4uPji3o+ERERERGpAgzDwM3NjZycHLOj2J2bmxuGUfryp7RDLq8BxgA3AqnFtIkAwoAmgCcwGFhQytcTEREREZEqYv/+/QwYMAA3Nzezo9iVm5sbAwYMYP/+/aV+jpL00H0PXA4EYZ0HNx54HrAAS2xt1gAPA/Wwbk9wHdahmI8Di7FuWzAd2F7qpCIiIiIiUiVMmTKFUaPO7fSRAAAGT0lEQVRGMXDgQFxcXMyOYzeGYbB//36mTJlS6udwKUv3nr1ERkYanTt3NjuGiIjYmWEY64FOZudwFPp8FBGpOkr6GVkeq1yKiIiIiIiICVTQiYiIiIiIOCgVdCIiIiIiIg6qUs6hA04AB8r4HEFAfDlkqQjKaj+OlFdZ7cORsoJj5S2PrI2BWuWQpaqoap+P4Fh5ldU+HCkrOFZeZbWP8spaos/IylrQlYdIHGeivbLajyPlVVb7cKSs4Fh5HSmr/MfR/t4cKa+y2ocjZQXHyqus9lGhWTXkUkRERERExEGpoBMREREREXFQbq+88orZGexpvdkBLoKy2o8j5VVW+3CkrOBYeR0pq/zH0f7eHCmvstqHI2UFx8qrrPZRYVmdeQ6diIiIiIiIU9OQSxEREREREQfljAXdNcBuYC8w1uQsRZkOHAe2FThXE1gC7LH9WcOEXEVpCCwDdgDbgSdt5ytjXi9gHbAZa9ZXbeebAGuxvh9mA56mpCuaG7ARWGT7ujJn3Q9sBTZhXbkJKuf7ACAA+BHYBewEulM5s4Zj/XnmHcnAKCpnVoCnsP63tQ34Hut/c5X5PStFq8yfkfp8tB99RtrPfvT5aA/6jLwIzlbQuQEfA9cCrYA7bX9WJl9h/UAtaCywFAiz/VlZPmSzgdFYf4bdgMdsjytj3gygL9AOuBTrz7gb8BbwPtAMSALuNytgEZ7E+j/UPJU5K8AVWH+2ecvwVsb3AcAHwO9AC6zvh51Uzqy7sf48LwU6AqnAfCpn1vrASKx/922w/r92MJX/PStnq+yfkV+hz0d70WekfenzsfzpM/JiGIbhTEd3wzAWF/j6edthdq5zjxDDMLYV+Hq3YRh1bY/r2r42O2NRx8+GYfRzgLw+hmFsMAyjq2EY8YZhuBtFvz/MPBoYhrHUMIy+hmEsMgzDpRJnxTCM/YZhBJ1zrjK+D6obhhFj+3lW9qwFj/6GYaysxFnrG4ZxyDCMmob1PbrIMIyrjcr9ntVR+HCEz8gQQ5+P9j70GVm+x35Dn4/2PvQZeYHD2Xro6gOHCnwdaztX2QUDR22P42xfVzYhQHusXceVNa8b1m7541i74fcBJ7H+JhUq1/thCjAGyLV9HUjlzQpgAH9gXbFphO1cZXwfNAFOAP/DOlTnC8CXypm1oMFYh2hA5cx6+P/bu3fWKKIwDuOPV/AGXotIkJjGzsIiTYIEvIAi2ogXFIJgZ2ujjSjYiWBlo4WFF7yh+QCmsBJBv4GXoKAWgoKViBbvDFl2I27hsO+ZPD8I7M42L5sz859z8s4JcAWYJWr7RoyFzGNWvUrMyIznQ7cR8ucjmJFNMR+bZ0b+Q9smdG3wu/rJZDXwiOhd/t71WaZ6fxF/mh8GxoiWgowOEIFa0ta7E8AOolXrDLCz6/Ms42ApUed14gbrB73tGFlqrS0HDgIP5vksS63rgEPEDcFm4iaguzVOalqW86FTKfkIZmRTzMdmmZF9aNuE7iPxoHJtuDqW3WdgqHo9RFzIslhGhNVt4HF1LHO9ECsiM8TDvmuJixjkGQ/jxMXpHXCPeK7hGjlrrdW1fCF62MfIOQ4+VD8vqvcPiQDLWGttH/CKqBFy1robeEus7v4krgXj5B6z6lViRmY8H2ol5iOYkf+b+dgsM7IPbZvQvSQektxKzOiPAdMDrag/08BU9XoKeDrAWjotAm4SD81e7Tiesd5NxIkDsALYQ9Q9Axyujmep9RxxYo8QY/QZcIKctUKsNK3peL2X2MUp4zj4RLSUbave7yJ2octYa+04c60kkLPWWWIDhZXEdaH+XrOOWc2vxIzMeD5AWfkIZmRTzMfmmZF9aOM/Ft9P9F4vIbZAvjzYcnrcBSaBjcQqwwXgCXAf2AK8B44AXwdUX6cJ4DmxHW/dx36eWN3JVu924Bbxe19M1HcJGCVW+NYTPeMnid2+spgEzhItJllrHSVWHSFWmu4Q59UG8o0DiJaiG8QN6xvgFHNjIlutq4ggGCV67iHv93oROEo8D/AaOE08D5BxzOrvMmek+dgcM7IZ5mOzzMg+tXFCJ0mSJEkLQttaLiVJkiRpwXBCJ0mSJEmFckInSZIkSYVyQidJkiRJhXJCJ0mSJEmFckInSZIkSYVyQidJkiRJhXJCJ0mSJEmF+gOxqBhtopqt4QAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 1080x432 with 2 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"plt.figure(figsize=(15, 6))\n",
"plt.subplot(121)\n",
"plt.plot(losses)\n",
"\n",
"plt.subplot(122)\n",
"plt.plot(train_accs, label=\"train\")\n",
"plt.plot(test_accs, label=\"test\")\n",
"plt.legend(loc=4)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"# Weakly-supervised localization"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"INFO:tensorflow:Restoring parameters from fe_challenge/fe_cnn-localizer-400\n"
]
}
],
"source": [
"saver.restore(sess, \"fe_challenge/fe_cnn-localizer-400\")"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[u'WSL_LOGIT/weights/Initializer/random_uniform/shape',\n",
" u'WSL_LOGIT/weights/Initializer/random_uniform/min',\n",
" u'WSL_LOGIT/weights/Initializer/random_uniform/max',\n",
" u'WSL_LOGIT/weights/Initializer/random_uniform/RandomUniform',\n",
" u'WSL_LOGIT/weights/Initializer/random_uniform/sub',\n",
" u'WSL_LOGIT/weights/Initializer/random_uniform/mul',\n",
" u'WSL_LOGIT/weights/Initializer/random_uniform',\n",
" u'WSL_LOGIT/weights',\n",
" u'WSL_LOGIT/weights/Assign',\n",
" u'WSL_LOGIT/weights/read',\n",
" u'WSL_LOGIT/biases/Initializer/zeros',\n",
" u'WSL_LOGIT/biases',\n",
" u'WSL_LOGIT/biases/Assign',\n",
" u'WSL_LOGIT/biases/read',\n",
" u'WSL_LOGIT/MatMul',\n",
" u'WSL_LOGIT/BiasAdd',\n",
" u'WSL_LOGIT/Softmax',\n",
" u'LOSS_WSL/labels_stop_gradient',\n",
" u'LOSS_WSL/xentropy/Rank',\n",
" u'LOSS_WSL/xentropy/Shape',\n",
" u'LOSS_WSL/xentropy/Rank_1',\n",
" u'LOSS_WSL/xentropy/Shape_1',\n",
" u'LOSS_WSL/xentropy/Sub/y',\n",
" u'LOSS_WSL/xentropy/Sub',\n",
" u'LOSS_WSL/xentropy/Slice/begin',\n",
" u'LOSS_WSL/xentropy/Slice/size',\n",
" u'LOSS_WSL/xentropy/Slice',\n",
" u'LOSS_WSL/xentropy/concat/values_0',\n",
" u'LOSS_WSL/xentropy/concat/axis',\n",
" u'LOSS_WSL/xentropy/concat',\n",
" u'LOSS_WSL/xentropy/Reshape',\n",
" u'LOSS_WSL/xentropy/Rank_2',\n",
" u'LOSS_WSL/xentropy/Shape_2',\n",
" u'LOSS_WSL/xentropy/Sub_1/y',\n",
" u'LOSS_WSL/xentropy/Sub_1',\n",
" u'LOSS_WSL/xentropy/Slice_1/begin',\n",
" u'LOSS_WSL/xentropy/Slice_1/size',\n",
" u'LOSS_WSL/xentropy/Slice_1',\n",
" u'LOSS_WSL/xentropy/concat_1/values_0',\n",
" u'LOSS_WSL/xentropy/concat_1/axis',\n",
" u'LOSS_WSL/xentropy/concat_1',\n",
" u'LOSS_WSL/xentropy/Reshape_1',\n",
" u'LOSS_WSL/xentropy',\n",
" u'LOSS_WSL/xentropy/Sub_2/y',\n",
" u'LOSS_WSL/xentropy/Sub_2',\n",
" u'LOSS_WSL/xentropy/Slice_2/begin',\n",
" u'LOSS_WSL/xentropy/Slice_2/size',\n",
" u'LOSS_WSL/xentropy/Slice_2',\n",
" u'LOSS_WSL/xentropy/Reshape_2',\n",
" u'LOSS_WSL/assert_broadcastable/weights/shape',\n",
" u'LOSS_WSL/assert_broadcastable/weights/rank',\n",
" u'LOSS_WSL/assert_broadcastable/values/shape',\n",
" u'LOSS_WSL/assert_broadcastable/values/rank',\n",
" u'LOSS_WSL/assert_broadcastable/is_scalar/x',\n",
" u'LOSS_WSL/assert_broadcastable/is_scalar',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/Switch',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/switch_t',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/switch_f',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/pred_id',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/Switch_1',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/is_same_rank',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/is_same_rank/Switch',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/is_same_rank/Switch_1',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/Switch',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/switch_t',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/switch_f',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/pred_id',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims/dim',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims/Switch',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims/Switch_1',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ones_like/Shape',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ones_like/Const',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ones_like',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/concat/axis',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/concat',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims_1/dim',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims_1',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims_1/Switch',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims_1/Switch_1',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/DenseToDenseSetOperation',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/num_invalid_dims',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/x',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/Switch_1',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/Merge',\n",
" u'LOSS_WSL/assert_broadcastable/is_valid_shape/Merge',\n",
" u'LOSS_WSL/assert_broadcastable/Const',\n",
" u'LOSS_WSL/assert_broadcastable/Const_1',\n",
" u'LOSS_WSL/assert_broadcastable/Const_2',\n",
" u'LOSS_WSL/assert_broadcastable/Const_3',\n",
" u'LOSS_WSL/assert_broadcastable/Const_4',\n",
" u'LOSS_WSL/assert_broadcastable/Const_5',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/Switch',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/switch_t',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/switch_f',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/pred_id',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/NoOp',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/control_dependency',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/Assert/data_0',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/Assert/data_1',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/Assert/data_2',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/Assert/data_4',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/Assert/data_5',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/Assert/data_7',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/Assert',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/Assert/Switch',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/Assert/Switch_1',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/Assert/Switch_2',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/Assert/Switch_3',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/control_dependency_1',\n",
" u'LOSS_WSL/assert_broadcastable/AssertGuard/Merge',\n",
" u'LOSS_WSL/Mul',\n",
" u'LOSS_WSL/Const',\n",
" u'LOSS_WSL/Sum',\n",
" u'LOSS_WSL/num_present/Equal/y',\n",
" u'LOSS_WSL/num_present/Equal',\n",
" u'LOSS_WSL/num_present/zeros_like',\n",
" u'LOSS_WSL/num_present/ones_like/Shape',\n",
" u'LOSS_WSL/num_present/ones_like/Const',\n",
" u'LOSS_WSL/num_present/ones_like',\n",
" u'LOSS_WSL/num_present/Select',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/weights/shape',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/weights/rank',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/values/shape',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/values/rank',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_scalar/x',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_scalar',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/Switch',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/switch_t',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/switch_f',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/pred_id',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/Switch_1',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/is_same_rank',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/is_same_rank/Switch',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/is_same_rank/Switch_1',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/Switch',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/switch_t',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/switch_f',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/pred_id',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims/dim',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims/Switch',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims/Switch_1',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ones_like/Shape',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ones_like/Const',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ones_like',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/concat/axis',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/concat',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims_1/dim',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims_1',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims_1/Switch',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/ExpandDims_1/Switch_1',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/DenseToDenseSetOperation',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/num_invalid_dims',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims/x',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/has_invalid_dims',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/Switch_1',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/has_valid_nonscalar_shape/Merge',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/is_valid_shape/Merge',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/Const',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/Const_1',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/Const_2',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/Const_3',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/Const_4',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/Const_5',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/Switch',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/switch_t',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/switch_f',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/pred_id',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/NoOp',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/control_dependency',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/Assert/data_0',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/Assert/data_1',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/Assert/data_2',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/Assert/data_4',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/Assert/data_5',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/Assert/data_7',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/Assert',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/Assert/Switch',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/Assert/Switch_1',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/Assert/Switch_2',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/Assert/Switch_3',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/control_dependency_1',\n",
" u'LOSS_WSL/num_present/broadcast_weights/assert_broadcastable/AssertGuard/Merge',\n",
" u'LOSS_WSL/num_present/broadcast_weights/ones_like/Shape',\n",
" u'LOSS_WSL/num_present/broadcast_weights/ones_like/Const',\n",
" u'LOSS_WSL/num_present/broadcast_weights/ones_like',\n",
" u'LOSS_WSL/num_present/broadcast_weights',\n",
" u'LOSS_WSL/num_present/Const',\n",
" u'LOSS_WSL/num_present',\n",
" u'LOSS_WSL/Const_1',\n",
" u'LOSS_WSL/Sum_1',\n",
" u'LOSS_WSL/Greater/y',\n",
" u'LOSS_WSL/Greater',\n",
" u'LOSS_WSL/Equal/y',\n",
" u'LOSS_WSL/Equal',\n",
" u'LOSS_WSL/ones_like/Shape',\n",
" u'LOSS_WSL/ones_like/Const',\n",
" u'LOSS_WSL/ones_like',\n",
" u'LOSS_WSL/Select',\n",
" u'LOSS_WSL/div',\n",
" u'LOSS_WSL/zeros_like',\n",
" u'LOSS_WSL/value',\n",
" u'gradients/LOSS_WSL/value_grad/zeros_like',\n",
" u'gradients/LOSS_WSL/value_grad/Select',\n",
" u'gradients/LOSS_WSL/value_grad/Select_1',\n",
" u'gradients/LOSS_WSL/value_grad/tuple/group_deps',\n",
" u'gradients/LOSS_WSL/value_grad/tuple/control_dependency',\n",
" u'gradients/LOSS_WSL/value_grad/tuple/control_dependency_1',\n",
" u'gradients/LOSS_WSL/div_grad/Shape',\n",
" u'gradients/LOSS_WSL/div_grad/Shape_1',\n",
" u'gradients/LOSS_WSL/div_grad/BroadcastGradientArgs',\n",
" u'gradients/LOSS_WSL/div_grad/RealDiv',\n",
" u'gradients/LOSS_WSL/div_grad/Sum',\n",
" u'gradients/LOSS_WSL/div_grad/Reshape',\n",
" u'gradients/LOSS_WSL/div_grad/Neg',\n",
" u'gradients/LOSS_WSL/div_grad/RealDiv_1',\n",
" u'gradients/LOSS_WSL/div_grad/RealDiv_2',\n",
" u'gradients/LOSS_WSL/div_grad/mul',\n",
" u'gradients/LOSS_WSL/div_grad/Sum_1',\n",
" u'gradients/LOSS_WSL/div_grad/Reshape_1',\n",
" u'gradients/LOSS_WSL/div_grad/tuple/group_deps',\n",
" u'gradients/LOSS_WSL/div_grad/tuple/control_dependency',\n",
" u'gradients/LOSS_WSL/div_grad/tuple/control_dependency_1',\n",
" u'gradients/LOSS_WSL/Sum_1_grad/Reshape/shape',\n",
" u'gradients/LOSS_WSL/Sum_1_grad/Reshape',\n",
" u'gradients/LOSS_WSL/Sum_1_grad/Const',\n",
" u'gradients/LOSS_WSL/Sum_1_grad/Tile',\n",
" u'gradients/LOSS_WSL/Select_grad/zeros_like',\n",
" u'gradients/LOSS_WSL/Select_grad/Select',\n",
" u'gradients/LOSS_WSL/Select_grad/Select_1',\n",
" u'gradients/LOSS_WSL/Select_grad/tuple/group_deps',\n",
" u'gradients/LOSS_WSL/Select_grad/tuple/control_dependency',\n",
" u'gradients/LOSS_WSL/Select_grad/tuple/control_dependency_1',\n",
" u'gradients/LOSS_WSL/Sum_grad/Reshape/shape',\n",
" u'gradients/LOSS_WSL/Sum_grad/Reshape',\n",
" u'gradients/LOSS_WSL/Sum_grad/Shape',\n",
" u'gradients/LOSS_WSL/Sum_grad/Tile',\n",
" u'gradients/LOSS_WSL/Mul_grad/Shape',\n",
" u'gradients/LOSS_WSL/Mul_grad/Shape_1',\n",
" u'gradients/LOSS_WSL/Mul_grad/BroadcastGradientArgs',\n",
" u'gradients/LOSS_WSL/Mul_grad/Mul',\n",
" u'gradients/LOSS_WSL/Mul_grad/Sum',\n",
" u'gradients/LOSS_WSL/Mul_grad/Reshape',\n",
" u'gradients/LOSS_WSL/Mul_grad/Mul_1',\n",
" u'gradients/LOSS_WSL/Mul_grad/Sum_1',\n",
" u'gradients/LOSS_WSL/Mul_grad/Reshape_1',\n",
" u'gradients/LOSS_WSL/Mul_grad/tuple/group_deps',\n",
" u'gradients/LOSS_WSL/Mul_grad/tuple/control_dependency',\n",
" u'gradients/LOSS_WSL/Mul_grad/tuple/control_dependency_1',\n",
" u'gradients/LOSS_WSL/num_present_grad/Reshape/shape',\n",
" u'gradients/LOSS_WSL/num_present_grad/Reshape',\n",
" u'gradients/LOSS_WSL/num_present_grad/Shape',\n",
" u'gradients/LOSS_WSL/num_present_grad/Tile',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights_grad/Shape',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights_grad/Shape_1',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights_grad/BroadcastGradientArgs',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights_grad/Mul',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights_grad/Sum',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights_grad/Reshape',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights_grad/Mul_1',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights_grad/Sum_1',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights_grad/Reshape_1',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights_grad/tuple/group_deps',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights_grad/tuple/control_dependency',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights_grad/tuple/control_dependency_1',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights/ones_like_grad/Const',\n",
" u'gradients/LOSS_WSL/num_present/broadcast_weights/ones_like_grad/Sum',\n",
" u'gradients/LOSS_WSL/xentropy/Reshape_2_grad/Shape',\n",
" u'gradients/LOSS_WSL/xentropy/Reshape_2_grad/Reshape',\n",
" u'gradients/LOSS_WSL/xentropy_grad/ExpandDims/dim',\n",
" u'gradients/LOSS_WSL/xentropy_grad/ExpandDims',\n",
" u'gradients/LOSS_WSL/xentropy_grad/mul',\n",
" u'gradients/LOSS_WSL/xentropy_grad/LogSoftmax',\n",
" u'gradients/LOSS_WSL/xentropy_grad/Neg',\n",
" u'gradients/LOSS_WSL/xentropy_grad/ExpandDims_1/dim',\n",
" u'gradients/LOSS_WSL/xentropy_grad/ExpandDims_1',\n",
" u'gradients/LOSS_WSL/xentropy_grad/mul_1',\n",
" u'gradients/LOSS_WSL/xentropy_grad/tuple/group_deps',\n",
" u'gradients/LOSS_WSL/xentropy_grad/tuple/control_dependency',\n",
" u'gradients/LOSS_WSL/xentropy_grad/tuple/control_dependency_1',\n",
" u'gradients/LOSS_WSL/xentropy/Reshape_grad/Shape',\n",
" u'gradients/LOSS_WSL/xentropy/Reshape_grad/Reshape',\n",
" u'gradients/WSL_LOGIT/Softmax_grad/mul',\n",
" u'gradients/WSL_LOGIT/Softmax_grad/Sum/reduction_indices',\n",
" u'gradients/WSL_LOGIT/Softmax_grad/Sum',\n",
" u'gradients/WSL_LOGIT/Softmax_grad/Reshape/shape',\n",
" u'gradients/WSL_LOGIT/Softmax_grad/Reshape',\n",
" u'gradients/WSL_LOGIT/Softmax_grad/sub',\n",
" u'gradients/WSL_LOGIT/Softmax_grad/mul_1',\n",
" u'gradients/WSL_LOGIT/BiasAdd_grad/BiasAddGrad',\n",
" u'gradients/WSL_LOGIT/BiasAdd_grad/tuple/group_deps',\n",
" u'gradients/WSL_LOGIT/BiasAdd_grad/tuple/control_dependency',\n",
" u'gradients/WSL_LOGIT/BiasAdd_grad/tuple/control_dependency_1',\n",
" u'gradients/WSL_LOGIT/MatMul_grad/MatMul',\n",
" u'gradients/WSL_LOGIT/MatMul_grad/MatMul_1',\n",
" u'gradients/WSL_LOGIT/MatMul_grad/tuple/group_deps',\n",
" u'gradients/WSL_LOGIT/MatMul_grad/tuple/control_dependency',\n",
" u'gradients/WSL_LOGIT/MatMul_grad/tuple/control_dependency_1',\n",
" u'WSL_LOGIT/weights/Adam_wsl/Initializer/zeros/shape_as_tensor',\n",
" u'WSL_LOGIT/weights/Adam_wsl/Initializer/zeros/Const',\n",
" u'WSL_LOGIT/weights/Adam_wsl/Initializer/zeros',\n",
" u'WSL_LOGIT/weights/Adam_wsl',\n",
" u'WSL_LOGIT/weights/Adam_wsl/Assign',\n",
" u'WSL_LOGIT/weights/Adam_wsl/read',\n",
" u'WSL_LOGIT/weights/Adam_wsl_1/Initializer/zeros/shape_as_tensor',\n",
" u'WSL_LOGIT/weights/Adam_wsl_1/Initializer/zeros/Const',\n",
" u'WSL_LOGIT/weights/Adam_wsl_1/Initializer/zeros',\n",
" u'WSL_LOGIT/weights/Adam_wsl_1',\n",
" u'WSL_LOGIT/weights/Adam_wsl_1/Assign',\n",
" u'WSL_LOGIT/weights/Adam_wsl_1/read',\n",
" u'WSL_LOGIT/biases/Adam_wsl/Initializer/zeros',\n",
" u'WSL_LOGIT/biases/Adam_wsl',\n",
" u'WSL_LOGIT/biases/Adam_wsl/Assign',\n",
" u'WSL_LOGIT/biases/Adam_wsl/read',\n",
" u'WSL_LOGIT/biases/Adam_wsl_1/Initializer/zeros',\n",
" u'WSL_LOGIT/biases/Adam_wsl_1',\n",
" u'WSL_LOGIT/biases/Adam_wsl_1/Assign',\n",
" u'WSL_LOGIT/biases/Adam_wsl_1/read',\n",
" u'Adam_wsl/update_WSL_LOGIT/weights/ApplyAdam',\n",
" u'Adam_wsl/update_WSL_LOGIT/biases/ApplyAdam']"
]
},
"execution_count": 42,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"[n.name for n in tf.get_default_graph().as_graph_def().node if \"WSL\" in n.name]"
]
},
{
"cell_type": "code",
"execution_count": 43,
"metadata": {},
"outputs": [],
"source": [
"cam_weights = graph.get_tensor_by_name(\"WSL_LOGIT/weights:0\")\n",
"cam_bias = graph.get_tensor_by_name(\"WSL_LOGIT/biases:0\")\n",
"# conv_last : Class Activation Maps (CAM; total 024 maps)\n",
"# cam_weights : weights for each CAM"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {},
"outputs": [],
"source": [
"from skimage.transform import resize\n",
"\n",
"def cam_upscaler(cam, cam_w, pred_):\n",
" cam = cam.reshape(-1, 1024).dot(cam_w)[:, pred_].reshape(24, 24)\n",
" cam = (cam - cam.min()) / (cam.max() - cam.min())\n",
" return resize(cam, (48, 48))\n",
" \n",
"def cam_localizer(chosen_sample, true_label):\n",
" # Class Activation Maps (CAM)\n",
" cam, cam_w, scores = sess.run([conv_last, cam_weights, logit], \n",
" feed_dict={X: whiten(chosen_sample.reshape(-1, 2304)), \n",
" is_training: False})\n",
" pred2, pred1 = np.argsort(scores[0])[-2:]\n",
" #print scores\n",
" cam1 = cam_upscaler(cam, cam_w, pred1)\n",
" cam2 = cam_upscaler(cam, cam_w, pred2)\n",
" \n",
" plt.figure(figsize=(12, 4))\n",
" plt.subplot(131)\n",
" plt.imshow(chosen_sample.reshape(48, 48), cmap=mpl.cm.gray)\n",
" plt.title(\"TRUE: {}\".format(true_label.split(\"_\")[-1].upper()))\n",
" plt.subplot(132)\n",
" plt.imshow(chosen_sample.reshape(48, 48), cmap=mpl.cm.gray, alpha=.4)\n",
" plt.imshow(cam1, alpha=.6)\n",
" plt.title(\"{}: {}\".format(fer[\"categories\"][pred1].split(\"_\")[-1].upper(), round(scores[0][pred1], 2)))\n",
" plt.subplot(133)\n",
" plt.imshow(chosen_sample.reshape(48, 48), cmap=mpl.cm.gray, alpha=.4)\n",
" plt.imshow(cam2, alpha=.6)\n",
" plt.title(\"{}: {}\".format(fer[\"categories\"][pred2].split(\"_\")[-1].upper(), round(scores[0][pred2], 2)))\n",
" return"
]
},
{
"cell_type": "code",
"execution_count": 45,
"metadata": {
"collapsed": true
},
"outputs": [
{
"data": {
"text/plain": [
"array([0, 3, 0, 4, 5, 2, 5, 2, 4, 5, 0, 5, 2, 2, 0, 2, 4, 3, 3, 2, 4, 3,\n",
" 2, 4, 2, 0, 2, 4, 2, 5, 0, 2, 2, 3, 4, 4, 4, 3, 2, 4, 0, 0, 2, 2,\n",
" 2, 5, 2, 0, 2, 0, 0, 2, 3, 3, 3, 5, 5, 5, 5, 3, 3, 0, 2, 3, 3, 5,\n",
" 5, 3, 4, 2, 3, 2, 2, 3, 2, 4, 3, 5, 2, 5, 2, 5, 4, 4, 2, 2, 5, 0,\n",
" 0, 5, 2, 0, 5, 2, 5, 0, 4, 3, 5, 3, 5, 4, 0, 5, 2, 0, 2, 3, 4, 3,\n",
" 0, 5, 2, 0, 4, 2, 2, 2, 2, 3, 4, 4, 2, 3, 0, 4, 4, 0, 4, 2, 0, 2,\n",
" 5, 3, 2, 4, 0, 2, 0, 0, 0, 0, 0, 0, 4, 2, 4, 2, 4, 5, 3, 2, 2, 5,\n",
" 3, 5, 4, 2, 2, 0, 3, 2, 3, 4, 3, 3, 0, 3, 0, 4, 2, 3, 0, 4, 2, 3,\n",
" 4, 5, 0, 0, 0, 4, 5, 0, 0, 2, 2, 4, 3, 4, 4, 5, 0, 0, 0, 0, 2, 2,\n",
" 0, 5, 5, 4, 5, 5, 4, 0, 0, 3, 0, 2, 0, 3, 2, 0, 3, 5, 5, 2, 0, 0,\n",
" 5, 0, 5, 2, 2, 5, 3, 5, 3, 0, 3, 4, 4, 3, 3, 4, 4, 2, 3, 4, 0, 0,\n",
" 4, 0, 2, 5, 3, 4, 5, 5, 3, 0, 3, 4, 5, 2, 0, 2, 0, 3, 3, 0, 0, 3,\n",
" 4, 0, 2, 0, 4, 2, 2, 4, 0, 2, 5, 5, 5, 4, 3, 2, 4, 4, 2, 2, 4, 2,\n",
" 3, 4, 2, 5, 3, 5, 2, 0, 4, 4, 0, 3, 2, 5, 5, 4, 4, 3, 2, 0, 5, 0,\n",
" 4, 3, 3, 3, 0, 0, 0, 5, 2, 4, 4, 0, 4, 5, 2, 5, 3, 3, 5, 0, 2, 3,\n",
" 0, 2, 5, 0, 4, 0, 3, 0, 2, 3, 0, 0, 0, 2, 5, 4, 5, 3, 2, 0, 3, 4,\n",
" 5, 4, 5, 3, 5, 2, 3, 3, 0, 4, 5, 5, 2, 0, 5, 4, 0, 5, 5, 2, 5, 2,\n",
" 5, 4, 0, 0, 0, 3, 0, 5, 4, 2, 3, 5, 2, 2, 4, 0, 3, 0, 5, 0, 2, 4,\n",
" 0, 5, 5, 2, 4, 0, 0, 3, 0, 2, 5, 4, 0, 4, 5, 5, 0, 2, 0, 2, 0, 3,\n",
" 5, 0, 5, 4, 4, 2, 2, 0, 0, 5, 0, 2, 4, 0, 3, 3, 2, 5, 2, 0, 4, 4,\n",
" 0, 4, 2, 3, 4, 2, 2, 3, 5, 5, 5, 3, 2, 3, 5, 5, 5, 0, 0, 4, 4, 0,\n",
" 4, 3, 4, 5, 5, 5, 0, 4, 0, 3, 3, 2, 2, 0, 4, 4, 0, 5, 3, 0, 3, 4,\n",
" 5, 0, 4, 5, 2, 2, 3, 5, 5, 2, 5, 4, 3, 4, 2, 4, 4, 5, 0, 4, 3, 3,\n",
" 5, 5, 5, 0, 5, 2])"
]
},
"execution_count": 45,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess.run(pred, feed_dict={X: whiten(xtrain[:512]), y: ytrain[:512], is_training: False})"
]
},
{
"cell_type": "code",
"execution_count": 51,
"metadata": {