Scale and Render an Image for Visual Search

readme.md

Render an Image for Visual Search

On the way to building Visual Search for the Pinterest browser extension, we had to solve the surprisingly complex problem of loading and resizing an image to fit vertically within the available space but only take up half the available width, and then render a visible selector box.

This is one of the testing prototypes we made along the way. Load it up, drag in an image, and watch your console for fake API queries.

behavior.js

(function (w, d, a) {
  var $ = w[a.k] = {
    "a": a, "w": w, "d": d, 
    "s": {}, 
    "v": {},
    "f": (function () {
      return {
        // get a DOM property or text attribute
        get: function (el, att) {
          var v = null;
          if (typeof el[att] !== 'undefined') {
            v = el[att];
          } else {
            v = el.getAttribute(att);
          }
          return v;
        },
        // set a DOM property or text attribute
        set: function (el, att, string) {
          if (typeof el[att] === 'string') {
            el[att] = string;
          } else {
            el.setAttribute(att, string);
          }
        },
        // create a DOM element
        make: function (obj) {
          var el = false, tag, att, key;
          for (tag in obj) {
            if (obj[tag].hasOwnProperty) {
              el = $.d.createElement(tag);
              for (att in obj[tag]) {
                if (obj[tag][att] && obj[tag][att].hasOwnProperty) {
                  if (typeof obj[tag][att] === 'string') {
                    $.f.set(el, att, obj[tag][att]);
                  } else {
                    if (att === 'style') {
                      for (key in obj[tag][att]) {
                        if (el.style.setProperty) {
                          // modern browsers
                          el.style.setProperty(key, obj[tag][att][key], 'important');
                        } else {
                          // be nice to IE8
                          el.style[key] = obj[tag][att][key];
                        }
                      }
                    }
                  }
                }
              }
              break;
            }
          }
          return el;
        },
        listen : function (el, ev, fn, detach) {
          if (!detach) {
            // add listener
            if (typeof $.w.addEventListener !== 'undefined') {
              el.addEventListener(ev, fn, false);
            } else if (typeof $.w.attachEvent !== 'undefined') {
              el.attachEvent('on' + ev, fn);
            }
          } else {
            // remove listener
            if (typeof el.removeEventListener !== 'undefined') {
              el.removeEventListener(ev, fn, false);
            } else if (typeof el.detachEvent !== 'undefined') {
              el.detachEvent('on' + ev, fn);
            }
          }
        },
        // scale one rectangle to always fit inside another
        scale: function (o) {
          
          // expects: { 'expand': BOOL|UNDEFINED, 'a': { 'h': NUMBER, 'w': NUMBER }, 'b': { 'h': NUMBER, 'w': NUMBER } }
          // returns: { 'n': : { 'h': NUMBER, 'w': NUMBER }, expand': BOOL|UNDEFINED, 'a': { 'h': NUMBER, 'w': NUMBER }, 'b': { 'h': NUMBER, 'w': NUMBER } }
          
          // default: return originals
          o.n = {
            'h': o.a.h,
            'w': o.a.w
          };

          // get aspect ratios for image and container          
          var ratio = {
            'a': o.n.h / o.n.w,
            'b': o.b.h / o.b.w
          }

          // fit width; scale height
          var fitWidth = function () {
            o.n.w = o.b.w;
            o.n.h = o.n.w * ratio.a;
          };

          // fit height; scale width
          var fitHeight = function () {
            o.n.h = o.b.h;
            o.n.w = o.n.h / ratio.a;
          };

          // decide if we need to fit to width or height
          var getFit = function () {
            if (ratio.a < ratio.b) {
              // image is proportionally wider than container
              fitWidth();
            } else {        
              // image is proportionally taller than container
              fitHeight();
            }
          }

          // trivial condition: we are not expanding, and our image fits inside the container
          if (!o.expand) {
            if (o.n.h <= o.b.h && o.n.w <= o.b.w) {
              return o;
            }
          }

          // image size is changing
          if (ratio.a === 1) {
            // scale the square image to fit the smaller side of the rectangular container
            o.n.w = o.n.h = Math.min(o.b.h, o.b.w);
          } else {
            // look at the shape of our container
            if (ratio.b === 1) {
              // square container
              if (ratio.a > 1) {
                // portrait image in square container; fit to height
                fitHeight();
              } else {
                // landscape image in square container; fit to width
                fitWidth();
              }
            } else {
              // rectangular container
              if (ratio.a > 1) {
                // portrait image
                if (ratio.b < 1) {
                  // portrait image in lansdcape container; fit to height
                  fitHeight();
                } else {
                  // portrait image in portrait container; decide if we need to fit to height or width
                  getFit();
                }
              } else {
                // landscape image
                if (ratio.b > 1) {
                  // landscape image in portrait container; fit to width
                  fitWidth();
                } else {
                  // landscape image in landscape container; decide if we need to fit to height or width
                  getFit();
                }
              }
            }
          }
          return o;
        },
        
        edit: function (scaledImg) {
          // send the query off to the API
          var query = function (o) {
            console.log('Querying:');
            console.log(o);
          };

          // where are we over the canvas?
          var getPos = function (e) {
            var rect = $.s.canvas.getBoundingClientRect();
            return {
              x: e.clientX - rect.left,
              y: e.clientY - rect.top
            }
          };  

          var select = function (hazCorners) {
            
            // corner width, length, and style
            var lw = 6;
            var ll = 25;
            var ls = '#ffe';
            
            // redraw the selector
            ctx.clearRect(0, 0, scaledImg.n.w, scaledImg.n.h);
            
            // outer shape: always the entire canvas
            ctx.beginPath();
            ctx.moveTo(0,0);
            ctx.lineTo(scaledImg.n.w, 0);
            ctx.lineTo(scaledImg.n.w, scaledImg.n.h);
            ctx.lineTo(0, scaledImg.n.h);
            ctx.closePath();
            
            var minX = ~~Math.min(box.x1, box.x2);
            var minY = ~~Math.min(box.y1, box.y2);

            var maxX = ~~Math.max(box.x1, box.x2);
            var maxY = ~~Math.max(box.y1, box.y2);
            
            // pass to query
            var selectX = minX;
            var selectY = minY;
            var selectH = maxX - minX;
            var selectW = maxY - minY;
            
            // inner shape: only the selected area
            ctx.moveTo(minX, minY);
            ctx.lineTo(maxX, minY);
            ctx.lineTo(maxX, maxY);
            ctx.lineTo(minX, maxY);
            ctx.closePath();

            // fill outside the selected area with 50% black
            ctx.fillStyle = "rgba(0,0,0,.50)";
            ctx.mozFillRule = 'evenodd'; // elderly Firefox
            ctx.fill('evenodd'); // modern browsers
            
            // draw the selector corners
            if (hazCorners) {
              ctx.strokeStyle = ls;
              ctx.lineWidth = lw;
  
              ctx.beginPath();
              ctx.moveTo(minX + lw / 2, minY + ll);
              ctx.lineTo(minX + lw / 2, minY + lw / 2);
              ctx.lineTo(minX + ll, minY + lw / 2);
              ctx.stroke();
  
              ctx.beginPath();
              ctx.moveTo(maxX - lw / 2, minY + ll);
              ctx.lineTo(maxX - lw / 2, minY + lw / 2);
              ctx.lineTo(maxX - ll, minY + lw / 2);
              ctx.stroke();
  
              ctx.beginPath();
              ctx.moveTo(maxX - lw / 2, maxY - ll);
              ctx.lineTo(maxX - lw / 2, maxY - lw / 2);
              ctx.lineTo(maxX - ll, maxY - lw / 2);
              ctx.stroke();
  
              ctx.beginPath();
              ctx.moveTo(minX + lw / 2, maxY - ll);
              ctx.lineTo(minX  + lw / 2, maxY - lw / 2);
              ctx.lineTo(minX + ll, maxY - lw / 2);
              ctx.stroke();
            }
          };

          var mouseDown = function (e) {
            var p = getPos(e);
            box.x1 = box.x2 = p.x;
            box.y1 = box.y2 = p.y;
            selecting = true;
          };
            
          var mouseMove = function (e) {   
            if (selecting) {
              var p = getPos(e);
              box.x2 = p.x;
              box.y2 = p.y;
              select();
            }
          };

          var mouseUp = function () {
            if (selecting) {
              select(true);
              query(box);
              selecting = false;
            }
          };

          var mouseOut = function (e) {
            if (selecting) {
              var p = getPos(e);
              // keep selector inside canvas
              if (p.x < 0) {
                box.x2 = 0;
              }
              if (p.x > scaledImg.n.w) {
                box.x2 = scaledImg.n.w;
              }
              if (p.y < 0) {
                box.y2 = 0;
              }
              if (p.y > scaledImg.n.h) {
                box.y2 = scaledImg.n.h;
              }
              
              select(true);
              query(box);
              selecting = false;
            }
          };
          
          // STUFF HAPPENS BELOW HERE

          var ctx = $.s.canvas.getContext('2d');
          var selecting = false;   

          // when the search preview first comes up, let's animate the selector box just a little bit
          var currentAccio = 1;
          var maxAccio = Math.min(scaledImg.n.h, scaledImg.n.w) / 20;
          var accioDelay = maxAccio + 10;

          // selector box; start at 100%
          var box = {};

          // "attract mode" animates a selection box on load and runs search, so we know what's going on 
          var accio = function () {
            // select the middle of the image
            box = {
              'x1': currentAccio,
              'y1': currentAccio,
              'x2': scaledImg.n.w - currentAccio,
              'y2': scaledImg.n.h - currentAccio
            }
            select(true);
            currentAccio = currentAccio + 1;
            if (currentAccio < maxAccio) {
              accioDelay = accioDelay - 2;
              if (!accioDelay) {
                accioDelay = 1;
              }
              $.w.setTimeout(accio, accioDelay);
            } else {
              // done animating; let's run that query
              query(box);
            }
          }
          
          $.w.setTimeout(function () {
            accio();
          }, 10);

          $.f.listen($.s.canvas, 'mousedown', mouseDown);
          $.f.listen($.s.canvas, 'mousemove', mouseMove);
          $.f.listen($.s.canvas, 'mouseup', mouseUp);
          $.f.listen($.s.canvas, 'mouseout', mouseOut);
        },
        // image has loaded; let's show it
        render: function (img) {
          $.s.output.className = '';
          $.s.output.style.height = $.s.container.offsetHeight + 'px';
          var scaledImg = $.f.scale({
            'expand': $.a.expand,
            'a': {
              'h': img.naturalHeight,
              'w': img.naturalWidth
            }, 
            'b': {
              'h': $.s.container.offsetHeight,
              'w': $.s.container.offsetWidth / 2 - $.a.gutter / 2
            }
          })
          img.height = scaledImg.n.h;
          img.width = scaledImg.n.w;
          $.s.output.style.width = img.width + 'px';
          $.s.output.appendChild(img);
          $.s.canvas = $.f.make({'CANVAS': {
            'height': scaledImg.n.h + '',
            'width': scaledImg.n.w + ''
          }});
          $.s.output.appendChild($.s.canvas);
          // start the edit 
          $.f.edit(scaledImg);
        },
        load : function (file) {
          $.s.progress.className = 'hidden';
          var reader = new FileReader();
          if (file.type.match(/^image\//)) {
            reader.addEventListener("loadend", function (e) {
              var t = new Image();
              t.onload = function () {
                $.f.render(this);
              };
              t.src = e.target.result;              
            }, false);
            reader.readAsDataURL(file);
          }
        },
        drop : function (e) {
          $.s.input.className = 'hidden';
          $.s.progress.className = '';
          var data = e.dataTransfer;
          e.stopPropagation();
          e.preventDefault();
          for (var i = 0, n = data.files.length; i < n; i++) {
            $.f.load(data.files[i]);
            break;
          }
        },
        halt : function (e) {
          e.stopPropagation();
          e.preventDefault();
        },
        init : function () {
          $.d.b = $.d.body;
          for (var i = 0; i < $.a.s.length; i = i + 1) {
            $.s[$.a.s[i]] = $.d.getElementById($.a.s[i]);
          }
          $.v.msg = $.a.str.en;
          if (typeof FileReader !== "function") {
            $.s.input.innerHTML = $.v.msg.compat;
          } else {
            $.f.listen($.s.input, 'dragenter', $.f.halt);
            $.f.listen($.s.input, 'dragover', $.f.halt);
            $.f.listen($.s.input, 'drop', $.f.drop);
            $.s.input.innerHTML = $.v.msg.ready;
          }
        }
      };
    }())
  };
  $.w.addEventListener("load", $.f.init, false);
}(window, document, {
  'k': 'I',
  'expand': false,
  'gutter': 40,
  'str': {
    'en': {
      'compat': 'NO FILEREADER PRESENT',
      'ready': 'READY FOR FILE',
      'error': 'TROUBLE READING FILE',
      'progress': 'READING FILE'
    }
  },
  's': ['input', 'output', 'progress', 'container']
}));

index.html

<!doctype html>
<html>
  <head>
    <title></title>
    <meta charset="utf-8">
    <link rel="stylesheet" href="presentation.css">
  </head>
  <body>
    <div id="container">
      <div id="input"></div>
      <div id="progress" class="hidden"></div>
      <div id="output" class="hidden"></div>
    </div>
    <script src="behavior.js"></script>
  </body>
</html>

presentation.css

html, body {
  height: 100%;
}

body {
  margin: 0;
  padding: 0;
  text-align: center;
}

#container {
  width: 90%;
  height: 90%;
  margin: 0 auto;
}

#input {
  height: 100%;
  width: 100%;
  background: #efe;
  text-align: center;
  border: 2px solid #000;
}
#output {
  background: #eee;
  text-align: left;
  position: relative;
}

#output canvas {
  position: absolute;
  top: 0;
  left: 0;
  cursor: crosshair;
}

.hidden {
  display: none!important;
}