navent-imaginate-challenge-2018.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iWILpQKsTLsp"
   },
   "source": [
    "# Notebook armada para el Challenge FIUBA 2018 - Navent\n",
    "## Basado en: \n",
    "### https://github.com/anujdutt9/Artistic-Style-Transfer-using-Keras-Tensorflow/blob/master/ArtisticStyleTransfer.py\n",
    "\n",
    "\n",
    "### Este es un kit de ayuda que lo podes modificar a tu antojo, podes hacer cualquier cosa y contarnos en el video que modificaciones le hiciste asi ganas el premio tecnico!!. Cualquier modificación cuenta.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "z76Ph9IFVSeD"
   },
   "source": [
    "# 0) Codigo de Kit de ayuda, no hace falta tocar nada -> ejecutar e ir al siguiente paso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "5F-OFUaedZit",
    "outputId": "279b5f3d-dc67-4902-f89a-133a1041d5b8"
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import time\n",
    "import numpy as np\n",
    "import keras.backend as K\n",
    "from PIL import Image\n",
    "from keras.models import Model\n",
    "from keras.applications.vgg16 import VGG16\n",
    "from scipy.optimize import fmin_l_bfgs_b\n",
    "\n",
    "\n",
    "def content_loss(content, combination):\n",
    "    return K.sum(K.square(combination - content))\n",
    "\n",
    "def gram_matrix(x):\n",
    "    features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))\n",
    "    gram = K.dot(features, K.transpose(features))\n",
    "    return gram\n",
    "\n",
    "def style_loss(style, combination):\n",
    "    S = gram_matrix(style)\n",
    "    C = gram_matrix(combination)\n",
    "    channels = 3\n",
    "    size = ht * wd\n",
    "    return K.sum(K.square(S - C))/(4. *(channels ** 2)*(size ** 2))\n",
    "\n",
    "def total_variation_loss(x):\n",
    "    a = K.square(x[:, :ht-1, :wd-1, :] - x[:, 1:, :wd-1, :])\n",
    "    b = K.square(x[:, :ht-1, :wd-1, :] - x[:, :ht-1, 1:, :])\n",
    "    return K.sum(K.pow(a + b, 1.25))\n",
    "\n",
    "def eval_loss_and_grads(x):\n",
    "    x = x.reshape((1, ht, wd, 3))\n",
    "    outs = f_outputs([x])\n",
    "    loss_value = outs[0]\n",
    "    if len(outs[1:]) == 1:\n",
    "        grad_values = outs[1].flatten().astype('float64')\n",
    "    else:\n",
    "        grad_values = np.array(outs[1:]).flatten().astype('float64')\n",
    "    return loss_value, grad_values\n",
    "\n",
    "class Evaluator(object):\n",
    "    def __init__(self):\n",
    "        self.loss_value = None\n",
    "        self.grads_values = None\n",
    "\n",
    "    def loss(self, x):\n",
    "        assert self.loss_value is None\n",
    "        loss_value, grad_values = eval_loss_and_grads(x)\n",
    "        self.loss_value = loss_value\n",
    "        self.grad_values = grad_values\n",
    "        return self.loss_value\n",
    "\n",
    "    def grads(self, x):\n",
    "        assert self.loss_value is not None\n",
    "        grad_values = np.copy(self.grad_values)\n",
    "        self.loss_value = None\n",
    "        self.grad_values = None\n",
    "        return grad_values\n",
    "\n",
    "def convert_to_img(y):\n",
    "  x = y.copy()\n",
    "  x = x.reshape((ht, wd, 3))\n",
    "  x = x[:, :, ::-1]\n",
    "  x[:, :, 0] += 103.939\n",
    "  x[:, :, 1] += 116.779\n",
    "  x[:, :, 2] += 123.68\n",
    "  x = np.clip(x, 0, 255).astype('uint8')\n",
    "\n",
    "  return Image.fromarray(x)\n",
    "\n",
    "def print_img(x):\n",
    "  display(convert_to_img(x))\n",
    "\n",
    "def build_model(content_image, style_image, content_weight, style_weight, total_variation_weight):\n",
    "  # Add a new Dimension to both Images in order to\n",
    "  # Concatinate the two Images at a later stage\n",
    "\n",
    "  content_array = np.asarray(content_image, dtype='float32')\n",
    "  content_array = np.expand_dims(content_array, axis=0)\n",
    "\n",
    "  style_array = np.asarray(style_image, dtype='float32')\n",
    "  style_array = np.expand_dims(style_array, axis=0)\n",
    "\n",
    "  content_array[:, :, :, 0] -= 103.939\n",
    "  content_array[:, :, :, 1] -= 116.779\n",
    "  content_array[:, :, :, 2] -= 123.68\n",
    "  style_array[:, :, :, 0] -= 103.939\n",
    "  style_array[:, :, :, 1] -= 116.779\n",
    "  style_array[:, :, :, 2] -= 123.68\n",
    "  content_array = content_array[:, :, :, ::-1]\n",
    "  style_array = style_array[:, :, :, ::-1]\n",
    "\n",
    "  content_image = K.variable(content_array)\n",
    "  style_image = K.variable(style_array)\n",
    "\n",
    "  combination_image = K.placeholder((1,ht, wd, 3))\n",
    "\n",
    "  input_tensor = K.concatenate([content_image, style_image, combination_image], axis=0)\n",
    "\n",
    "  model = VGG16(input_tensor=input_tensor, weights='imagenet', include_top=False)\n",
    "\n",
    "  layers = dict([(layer.name, layer.output) for layer in model.layers])\n",
    "\n",
    "  loss = K.variable(0.)\n",
    "\n",
    "  layer_features = layers['block2_conv2']\n",
    "\n",
    "  content_image_features = layer_features[0, :, :, :]\n",
    "  combination_features = layer_features[2, :, :, :]\n",
    "  loss += content_weight * content_loss(content_image_features, combination_features)\n",
    "\n",
    "  # Feature Layers of VGG16 Trained Neural Network\n",
    "  feature_layers = ['block1_conv2', 'block2_conv2',\n",
    "                    'block3_conv3', 'block4_conv3',\n",
    "                    'block5_conv3']\n",
    "\n",
    "  for layer_name in feature_layers:\n",
    "      layer_features = layers[layer_name]\n",
    "      style_features = layer_features[1, :, :, :]\n",
    "      combination_features = layer_features[2, :, :, :]\n",
    "      sl = style_loss(style_features, combination_features)\n",
    "      loss += (style_weight / len(feature_layers)) * sl\n",
    "\n",
    "\n",
    "  loss += total_variation_weight * total_variation_loss(combination_image)\n",
    "  grads = K.gradients(loss, combination_image)\n",
    "\n",
    "  outputs = [loss]\n",
    "  if type(grads) in {list, tuple}:\n",
    "      outputs += grads\n",
    "  else:\n",
    "      outputs.append(grads)\n",
    "\n",
    "  f_outputs = K.function([combination_image], outputs)\n",
    "\n",
    "\n",
    "  evaluator = Evaluator()\n",
    "  x = np.random.uniform(0, 255, (1, ht, wd, 3)) - 128.\n",
    "  return f_outputs, x, evaluator\n",
    "\n",
    "def run_model(f_outputs, x, evaluator, iterations=100):\n",
    "  print(\"Imagen final - antes de iterar\")\n",
    "  print_img(x)\n",
    "\n",
    "  for i in range(iterations):\n",
    "      start_time = time.time()\n",
    "      x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x.flatten(), fprime=evaluator.grads, maxfun=20)\n",
    "#       print(f\"Iteration: {i}, Current loss value:{min_val}, iteration time: {time.time() - start_time :.0f} secs\")\n",
    "\n",
    "      if i %2 == 0:\n",
    "        print_img(x)\n",
    "\n",
    "\n",
    "  print_img(x)\n",
    "  return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Fq1pFsBvXlOf"
   },
   "source": [
    "## 1) En este paso, ejecutar la celda y elegir las imagens a subir (con Ctr se elije más de una)\n",
    "### Se tienen que subir las fotos de los logos de Navent y todas las imagenes que hayan considerado como fuente de estilo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mr55ZNImXuSd"
   },
   "source": [
    "## 2) Ahora \"cargamos\" las imagenes en python "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XcgtED7rYCcA"
   },
   "outputs": [],
   "source": [
    "nombre_de_imagen_base_subida = \"navent_comedor.jpg\" # poner aqui el nombre de la imagen base recien subida \n",
    "nombre_de_imagen_estilo_subida = \"city.jpg\" # poner aqui el nombre de la imagen estilo recien subida\n",
    "\n",
    "ht = 512 # alto de imagen final (no poner un numero grande porque el modelo se queda sin memoria y explota)\n",
    "wd = 512 # ancho de imagen final (no poner un numero grande porque el modelo se queda sin memoria y explota)\n",
    "\n",
    "imagen_base = Image.open(nombre_de_imagen_base_subida).resize((ht, wd), Image.ANTIALIAS)\n",
    "imagen_estilo = Image.open(nombre_de_imagen_estilo_subida).resize((ht, wd), Image.ANTIALIAS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uNClcaWrqqlF"
   },
   "source": [
    "## 2.1) Observamos la imagen base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 529
    },
    "colab_type": "code",
    "id": "poIe7Mq-qvc-",
    "outputId": "aea45806-a93b-4c52-e7e8-f796ce3157d7"
   },
   "outputs": [],
   "source": [
    "imagen_base"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FUmnQbnvqxlf"
   },
   "source": [
    "## 2.2) Observamos la imagen estilo "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 529
    },
    "colab_type": "code",
    "id": "PIvReoyTq3vY",
    "outputId": "01ece451-19de-40df-f69a-185132f69c5f"
   },
   "outputs": [],
   "source": [
    "imagen_estilo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yEL5esmFK-jg"
   },
   "source": [
    "## 3) Variables a modificar\n",
    "### Estas variables tienen que ser ajustadas por ustedes para obtener una imagen que ustedes consideren \"linda\", los parametros dependen de la imagen base y la imagen estilo. Asi que para cada par imagen_base, imagen_estilo hay una configuración de parametros distinta."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "--8Yt96wLEv2"
   },
   "outputs": [],
   "source": [
    "# Cuanta importancia le da el modelo a la imagen base, si es un valor alto, va a haber un predominio importante en la imagen final, si es muy bajo, ocurrirá lo contrario\n",
    "importancia_de_imagen_base = 40.25 \n",
    "\n",
    "# Cuanta importancia le da el modelo a la imagen base, si es un valor alto, va a haber un predominio importante en la imagen final, si es muy bajo, ocurrirá lo contrario\n",
    "importancia_de_imagen_estilo = 10000.0\n",
    "\n",
    "# Le da un poco de importancia a la suavidad de la imagen final, a mayor valor, mayor suavidad.\n",
    "importancia_suavidez_imagen_final = 60.1 \n",
    "\n",
    "# El metodo de transfer learning (transferencia de estilo), es una metodo iterativo, a más iteraciones, mayor va a ser la variación de la imagen final.\n",
    "# Un tema a tener en cuenta es, que despues de una cierta cantidad de iteraciones el \"Current loss value\", el valor que se muestra en cada iteracion\n",
    "# en la ejecución de la siguiente celda va a cambiar cada vez menos y por ende llegaria a un \"equilibrio\" en donde más iteraciones no marcarán una\n",
    "# diferencia significativa en la imagen final\n",
    "cantidad_de_iteraciones = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "b8VbB32Pq8Dm"
   },
   "source": [
    "## 4) Corremos modelo y vemos la magia de como se va transfiriendo el estilo :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Uqke-cF0j5Th"
   },
   "outputs": [],
   "source": [
    "f_outputs, x, evaluator = build_model(imagen_base, imagen_estilo, importancia_de_imagen_base, importancia_de_imagen_estilo, importancia_suavidez_imagen_final)\n",
    "run_model(f_outputs, x, evaluator, iterations=cantidad_de_iteraciones)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "z8T0DYEvtUSb"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "Copy of Challenge FIUBA - 2018 - kit de ayuda.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}