yart/project/2021_ICRA_UMR/script/demo_05_contrastive_loss_negative_fix.ipynb

demo_05_contrastive_loss_negative_fix.ipynb

{
  "cells": [
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "### NCE loss with fixing negative data"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "import numpy as np\nimport tensorflow as tf\nfrom util import print_tf_tensor,gpu_sess\nnp.set_printoptions(precision=3)\nprint (\"Done.\")",
      "execution_count": 1,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "Done.\n"
        }
      ]
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "### Data"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "n_batch,dim,n_neg = 3,2,5\nbias = np.random.randn(n_batch,dim)\neps = 0.1\nha = tf.cast(tf.Variable(bias+eps*np.random.randn(n_batch,dim)),tf.float32,name='ha')\nhb = tf.cast(tf.Variable(bias+eps*np.random.randn(n_batch,dim)),tf.float32,name='hb')\nneg = tf.cast(tf.Variable(np.random.randn(n_neg,dim)),tf.float32,name='neg')",
      "execution_count": 7,
      "outputs": []
    },
    {
      "metadata": {
        "scrolled": true,
        "trusted": true
      },
      "cell_type": "code",
      "source": "sess = gpu_sess()\nsess.run(tf.global_variables_initializer())\nprint_tf_tensor(sess,ha)\nprint_tf_tensor(sess,hb)\nprint_tf_tensor(sess,neg)",
      "execution_count": 8,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "name:[ha:0] shape:(3, 2)\n[[-9.833e-01  8.341e-01]\n [ 8.658e-01 -4.415e-05]\n [-2.182e+00 -6.825e-01]]\nname:[hb:0] shape:(3, 2)\n[[-0.68   0.688]\n [ 0.79  -0.135]\n [-2.138 -0.649]]\nname:[neg_1:0] shape:(5, 2)\n[[-0.354  0.031]\n [ 0.304 -0.457]\n [-0.677  1.099]\n [ 1.348 -1.866]\n [ 0.689  0.571]]\n"
        }
      ]
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "### Loss"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "ha_nzd = tf.math.l2_normalize(ha,axis=-1)\nhb_nzd = tf.math.l2_normalize(hb,axis=-1)\nneg_nzd = tf.math.l2_normalize(neg,axis=-1)\nprint_tf_tensor(sess,ha_nzd)\nprint_tf_tensor(sess,hb_nzd)\nprint_tf_tensor(sess,neg_nzd)",
      "execution_count": 9,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "name:[l2_normalize_3:0] shape:(3, 2)\n[[-7.626e-01  6.469e-01]\n [ 1.000e+00 -5.099e-05]\n [-9.544e-01 -2.986e-01]]\nname:[l2_normalize_4:0] shape:(3, 2)\n[[-0.703  0.711]\n [ 0.986 -0.169]\n [-0.957 -0.291]]\nname:[l2_normalize_5:0] shape:(5, 2)\n[[-0.996  0.087]\n [ 0.553 -0.833]\n [-0.525  0.851]\n [ 0.586 -0.811]\n [ 0.77   0.638]]\n"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "temperature = 1.0\nlogits_ab = tf.matmul(ha_nzd/temperature,hb_nzd,transpose_b=True,name='logits_ab')\nlogits_ba = tf.matmul(hb_nzd/temperature,ha_nzd,transpose_b=True,name='logits_ba')\nlogits_an = tf.matmul(ha_nzd/temperature,neg_nzd,transpose_b=True,name='logits_an') \nlogits_bn = tf.matmul(hb_nzd/temperature,neg_nzd,transpose_b=True,name='logits_bn') \nprint_tf_tensor(sess,logits_ab)\nprint_tf_tensor(sess,logits_ba)\nprint_tf_tensor(sess,logits_an)\nprint_tf_tensor(sess,logits_bn)",
      "execution_count": 10,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "name:[logits_ab_1:0] shape:(3, 3)\n[[ 0.996 -0.861  0.542]\n [-0.703  0.986 -0.957]\n [ 0.458 -0.89   1.   ]]\nname:[logits_ba_1:0] shape:(3, 3)\n[[ 0.996 -0.703  0.458]\n [-0.861  0.986 -0.89 ]\n [ 0.542 -0.957  1.   ]]\nname:[logits_an_1:0] shape:(3, 5)\n[[ 0.816 -0.961  0.951 -0.971 -0.174]\n [-0.996  0.553 -0.525  0.586  0.77 ]\n [ 0.925 -0.279  0.247 -0.317 -0.925]]\nname:[logits_bn_1:0] shape:(3, 5)\n[[ 0.762 -0.981  0.974 -0.988 -0.087]\n [-0.997  0.686 -0.661  0.714  0.651]\n [ 0.928 -0.287  0.255 -0.325 -0.922]]\n"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "labels = tf.one_hot(tf.range(n_batch),n_batch+n_neg,name='labels')\nprint_tf_tensor(sess,labels)",
      "execution_count": 11,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "name:[labels_1:0] shape:(3, 8)\n[[1. 0. 0. 0. 0. 0. 0. 0.]\n [0. 1. 0. 0. 0. 0. 0. 0.]\n [0. 0. 1. 0. 0. 0. 0. 0.]]\n"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "weights = 1.0\nloss_a = tf.losses.softmax_cross_entropy(\n  labels, tf.concat([logits_ab, logits_an], 1), weights=weights)\nloss_b = tf.losses.softmax_cross_entropy(\n  labels, tf.concat([logits_ba, logits_bn], 1), weights=weights)\nnce_loss = loss_a + loss_b\nprint (nce_loss)",
      "execution_count": 18,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "Tensor(\"add_4:0\", shape=(), dtype=float32)\n"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "### Wrap it up with a function"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "def get_nce_negfix_loss(ha,hb,neg,temperature=1.0,weights=1.0):\n    ha_nzd = tf.math.l2_normalize(ha,axis=-1)\n    hb_nzd = tf.math.l2_normalize(hb,axis=-1)\n    neg_nzd = tf.math.l2_normalize(neg,axis=-1)\n    logits_ab = tf.matmul(ha_nzd/temperature,hb_nzd,transpose_b=True,name='logits_ab')\n    logits_ba = tf.matmul(hb_nzd/temperature,ha_nzd,transpose_b=True,name='logits_ba')\n    logits_an = tf.matmul(ha_nzd/temperature,neg_nzd,transpose_b=True,name='logits_an') \n    logits_bn = tf.matmul(hb_nzd/temperature,neg_nzd,transpose_b=True,name='logits_bn') \n    n_batch = tf.shape(ha)[0]\n    labels = tf.one_hot(tf.range(n_batch),n_batch+n_neg,name='labels')\n    loss_a = tf.losses.softmax_cross_entropy(\n      labels, tf.concat([logits_ab, logits_an], 1), weights=weights)\n    loss_b = tf.losses.softmax_cross_entropy(\n      labels, tf.concat([logits_ba, logits_bn], 1), weights=weights)\n    nce_loss = loss_a + loss_b\n    return nce_loss\nprint (\"Done.\")",
      "execution_count": 3,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "Done.\n"
        }
      ]
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "### Usage 1 (random feature maps)"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "tf.reset_default_graph()\nsess = gpu_sess()\nn_batch,dim,n_neg = 5,2,10\nha = tf.cast(tf.Variable(np.random.rand(n_batch,dim)),tf.float32,name='h1')\nhb = tf.cast(tf.Variable(np.random.rand(n_batch,dim)),tf.float32,name='h2')\nneg = tf.cast(tf.Variable(np.random.randn(n_neg,dim)),tf.float32,name='neg')\nnce_loss = get_nce_negfix_loss(ha,hb,neg)\nsess.run(tf.global_variables_initializer())\nprint_tf_tensor(sess,ha)\nprint_tf_tensor(sess,hb)\nprint (\"Loss is [%.4f].\"%(sess.run(nce_loss)))",
      "execution_count": 4,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "name:[h1:0] shape:(5, 2)\n[[0.156 0.677]\n [0.059 0.864]\n [0.216 0.446]\n [0.809 0.325]\n [0.084 0.933]]\nname:[h2:0] shape:(5, 2)\n[[0.024 0.525]\n [0.523 0.541]\n [0.93  0.811]\n [0.175 0.575]\n [0.817 0.023]]\nLoss is [4.7367].\n"
        }
      ]
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "### Usage 2 (similar feature maps)"
    },
    {
      "metadata": {
        "scrolled": true,
        "trusted": true
      },
      "cell_type": "code",
      "source": "tf.reset_default_graph()\nsess = gpu_sess()\nn_batch,dim,n_neg = 5,2,5\nbias = np.random.randn(n_batch,dim)\nha = tf.cast(tf.Variable(bias+0.1*np.random.randn(n_batch,dim)),tf.float32,name='h1')\nhb = tf.cast(tf.Variable(bias+0.1*np.random.randn(n_batch,dim)),tf.float32,name='h2')\nneg = tf.cast(tf.Variable(np.random.randn(n_neg,dim)),tf.float32,name='neg')\nnce_loss = get_nce_negfix_loss(ha,hb,neg)\nsess.run(tf.global_variables_initializer())\nprint_tf_tensor(sess,ha)\nprint_tf_tensor(sess,hb) \nprint (\"Loss is [%.4f].\"%(sess.run(nce_loss)))",
      "execution_count": 20,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "name:[h1:0] shape:(5, 2)\n[[ 0.695 -0.506]\n [ 0.471 -1.665]\n [ 0.116  0.682]\n [ 0.663 -1.569]\n [-0.777 -1.331]]\nname:[h2:0] shape:(5, 2)\n[[ 0.647 -0.36 ]\n [ 0.343 -1.619]\n [ 0.048  0.702]\n [ 0.669 -1.523]\n [-0.927 -1.551]]\nLoss is [3.1887].\n"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "version": "3.6.7",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "gist": {
      "id": "",
      "data": {
        "description": "yart/project/2021_ICRA_UMR/script/demo_05_contrastive_loss_negative_fix.ipynb",
        "public": true
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}