prez d3.js + elasticsearch on bordeaux open data @BordeauxJS

bulk-import-sample.json

{ "index" : { "_index" : "musees", "_type" : "capc"} }
{"Auteur" : "AA BRONSON (TIMS Mickael, dit)", "Titre" : "Jorge, February 3, 1994", "Date" : 2000, "Domaine" : "Photographie", "Deno" : null, "MatSuppTech" : "Impression sépia sur mylar", "Description" : null, "Dimensions" : "3 x (180,5 x 91,5) cm", "Column 10" : null, "Column 11" : null, "Column 12" : null, "Column 13" : null, "Column 14" : null, "Column 15" : null, "Column 16" : null, "Column 17" : null, "Column 18" : null, "Column 19" : null, "Column 20" : null, "Column 21" : null }
{ "index" : { "_index" : "musees", "_type" : "capc"} }
{"Auteur" : "ABSALON (MEIR Eshel, dit)", "Titre" : "Cellule (3)", "Date" : 1991, "Domaine" : "Sculpture", "Deno" : null, "MatSuppTech" : "Bois, carton, Plexiglas, tubes fluorescents, peinture glycerophtalique", "Description" : null, "Dimensions" : "133x240x161 cmPorte : 34 x 179 x 25 cm", "Column 10" : null, "Column 11" : null, "Column 12" : null, "Column 13" : null, "Column 14" : null, "Column 15" : null, "Column 16" : null, "Column 17" : null, "Column 18" : null, "Column 19" : null, "Column 20" : null, "Column 21" : null }
{ "index" : { "_index" : "musees", "_type" : "capc"} }
{"Auteur" : "ABSALON (MEIR Eshel, dit)", "Titre" : "Cellule (4)", "Date" : 1991, "Domaine" : "Sculpture", "Deno" : null, "MatSuppTech" : "Bois, carton, Plexiglas, tubes fluorescents et peinture glycérophtalique.", "Description" : null, "Dimensions" : "147x180x247 cmPorte: 78,5 x 74 x 30 cm", "Column 10" : null, "Column 11" : null, "Column 12" : null, "Column 13" : null, "Column 14" : null, "Column 15" : null, "Column 16" : null, "Column 17" : null, "Column 18" : null, "Column 19" : null, "Column 20" : null, "Column 21" : null }

capc.js

'use strict';

prezApp.controller('CapcCtrl', ['$scope',function($scope) {

 // constants for svg viz
 var width = 940,
 height = 352;// 940 * 9 / 16

 var barWidth = 13;

 var transitionDuration = 500;
 var padding = 18;

 // years 
 var hYears = height / 2 - padding,
  wYears = width - (padding * 2),
  yYears = 0;

 var hDomaines = height / 2,
  wDomaines = 280 - (padding * 2),
  yDomaines = height / 2 + padding * 2,
  textPaddingLeft = 10,
  textPaddingTop = (barWidth / 2) + 4;

 var hArtists = height / 2,
  wArtists = 240 - (padding * 2),
  yArtists = height / 2 + padding * 2,
  xArtists = 420;

 var maxYears,
  maxDomaines,
  maxArtists;

 
 var yScaleYears;

 // GLOBALE SCALES
 var globalMaxDomaines;
 var globalMaxArtists;

 var formatAxis = d3.format("d");

 var server = es.server('http://127.0.0.1:9200/musees/capc/_search');

 function initApp() {

  // INITIAL VIZ
  d3.select('#capc')
  .append("svg")
  .attr("class", "chart")
  .attr("width", width)
  .attr("height", height);


  d3.select('#capc svg')
  .append("g")
  .attr("class", "years")
  .attr("width", wYears)
  .attr("height", hYears);

  d3.select('#capc svg')
  .append("g")
  .attr("class", "domaines")
  .attr("width", wDomaines)
  .attr("height", hDomaines);

  d3.select('#capc svg')
  .append("g")
  .attr("class", "artists")
  .attr("width", wArtists)
  .attr("height", hArtists);

   // INITIAL SEARCH
   doSearch('*');
 }

 // display repartition of works per years
 function displayYears(data) {
   var vis = d3.select('#capc svg g.years');

   var dataSize = data.length;

   // SCALES
   maxYears = d3.max(data, function(d) { return d['count']; });
  
   yScaleYears = d3.scale.linear()
   .domain([0, maxYears])
   .range([0, hYears]);
   
   var yOrdinal = d3.scale.linear()
   .domain([0, maxYears])
   .range([hYears + yYears, yYears + padding]);

   var x = d3.scale.linear()
   .domain([1950, 2011])
   .range([padding, wYears]);

   // WITH DATA (NEW)
   vis.selectAll("rect.all")
   .data(data)
   .enter().append("rect")
   .attr("class", "all" )
   .attr("x", function(d, i) { return x(d['term']) ; })
   .attr("y", function(d, i) { return hYears + yYears - yScaleYears(d['count']) - .5; })
   .attr("width", ( barWidth - 2) / 3 )
   .attr("height", function(d) { return yScaleYears(d['count']); });

   // UPDATE !!
   vis.selectAll("rect.all")
   .data(data)
   .transition()
   .duration(transitionDuration)
   .attr("y", function(d, i) { return yYears + hYears - yScaleYears(d['count']) - .5; })
   .attr("height", function(d) { return yScaleYears(d['count']); });

   // EXIT !!
   vis.selectAll("rect.all")
   .data(data)
   .exit().remove();


   // AXIS
   var xAxis = d3.svg.axis()
     .scale(x)
     .tickFormat(formatAxis)
     .orient("bottom");
   
   vis.selectAll("g.axis")
   .remove();

   vis.append("g")
    .attr("class", "axis")
    .attr("transform", "translate(0," + (yYears + hYears) + ")")
    .call(xAxis);

    //Define Y axis
    var yAxis = d3.svg.axis()
    .scale(yOrdinal)
    .orient("right")
    .ticks(4);

    vis.append("g")
    .attr("class", "axis")
    .attr("opacity", 0)
    .attr("transform", "translate(" + (wYears + padding / 2) + ",0)")
    .transition()
    .duration(transitionDuration)
    .attr("opacity", 1)
    .call(yAxis);
 }

 // display repartition of works per years for the most represented domain
 function displayYearsTopDomain(data) {
   var vis = d3.select('#capc svg g.years');

   var dataSize = data.length;

   // SCALES
   maxYears = d3.max(data, function(d) { return d['count']; });
  

   var x = d3.scale.linear()
   .domain([1950, 2011])
   .range([padding + ( barWidth - 2) / 3 + 1, wYears + ( barWidth - 2) / 3 + 1]);

   // WITH DATA (NEW)
   vis.selectAll("rect.domain")
   .data(data)
   .enter().append("rect")
   .attr("class", "domain" )
   .attr("x", function(d, i) { return x(d['term']) ; })
   .attr("y", function(d, i) { return hYears + yYears -  yScaleYears(d['count']) - .5; })
   .attr("width", ( barWidth - 2) / 3 )
   .attr("height", function(d) { return yScaleYears(d['count']); });

   // UPDATE !!
   vis.selectAll("rect.domain")
   .data(data)
   .transition()
   .duration(transitionDuration)
   .attr("y", function(d, i) { return yYears + hYears - yScaleYears(d['count']) - .5; })
   .attr("height", function(d) { return yScaleYears(d['count']); });

   // EXIT !!
   vis.selectAll("rect.domain")
   .data(data)
   .exit().remove();
 }

 // display repartition of works per years for the most represented artist
 function displayYearsTopArtist(data) {
   var vis = d3.select('#capc svg g.years');

   var dataSize = data.length;

   // SCALES
   maxYears = d3.max(data, function(d) { return d['count']; });


   var x = d3.scale.linear()
   .domain([1950, 2011])
   .range([padding + ( barWidth - 2) / 3 * 2 + 2, wYears + ( barWidth - 2) / 3 * 2 + 2]);

   // WITH DATA (NEW)
   vis.selectAll("rect.artist")
   .data(data)
   .enter().append("rect")
   .attr("class", "artist" )
   .attr("x", function(d, i) { return x(d['term']) ; })
   .attr("y", function(d, i) { return hYears + yYears - yScaleYears(d['count']) - .5; })
   .attr("width", ( barWidth - 2) / 3 )
   .attr("height", function(d) { return yScaleYears(d['count']); });

   // UPDATE !!
   vis.selectAll("rect.artist")
   .data(data)
   .transition()
   .duration(transitionDuration)
   .attr("y", function(d, i) { return yYears + hYears - yScaleYears(d['count']) - .5; })
   .attr("height", function(d) { return yScaleYears(d['count']); });

   // EXIT !!
   vis.selectAll("rect.artist")
   .data(data)
   .exit().remove();
 }

 function displayDomaines(data) {
   var vis = d3.select('#capc svg g.domaines');

   var dataSize = data.length;

   maxDomaines = d3.max(data, function(d) { return d['count']; });

   var x = d3.scale.linear()
    .domain([0, maxDomaines])
    .range([padding, wDomaines - padding]);

   if (!globalMaxDomaines) {
      globalMaxDomaines = maxDomaines;
   }

    var opacityScale = d3.scale.pow().exponent(0.5)
    .domain([0, globalMaxDomaines])
    .range([0.2, 1]);
   
   var xOrdinal = d3.scale.linear()
    .domain([0, maxDomaines])
    .range([wDomaines, padding]);

   var y = d3.scale.linear()
   .domain([0, dataSize - 1])
   .range([yDomaines , yDomaines + hDomaines]);

   // WITH DATA (NEW)
   vis.selectAll("rect")
   .data(data)
   .enter().append("rect")
   .attr("x", function(d, i) { return wDomaines - x(d['count']);})
   .attr("y", function(d, i) { return i * barWidth + yDomaines; })
   .attr("width", function(d) { return x(d['count']); }) 
   .attr("opacity", function(d) { return opacityScale(d['count']); })
   .attr("height", barWidth - 1);


   // FORCE UPDATE
   vis.selectAll("text")
    .remove();

   vis.selectAll("text")
   .data(data)
   .enter().append("text")
   .attr("x", function(d, i) { return wDomaines + textPaddingLeft;})
   .attr("y", function(d, i) { return i * barWidth + yDomaines + textPaddingTop; })
   .attr("height", barWidth - 1)
   .text(function(d) { return d['term']; });

   // UPDATE !!
   vis.selectAll("rect")
   .data(data)
   .transition()
   .duration(transitionDuration)
   .attr("x", function(d, i) { return wDomaines - x(d['count']);})
   .attr("y", function(d, i) { return i * barWidth + yDomaines; })
   .attr("opacity", function(d) { return opacityScale(d['count']); })
   .attr("width", function(d) { return x(d['count']); });

    // Exit…
    vis.selectAll("rect")
     .data(data)
     .exit()
     .transition()
     .duration(transitionDuration)
     .attr("opacity", 0)
     .remove();

    // AXIS
    var xAxis = d3.svg.axis()
    .scale(xOrdinal)
    .tickFormat(formatAxis)
    .orient("top")
    .ticks(4);

    vis.selectAll("g.axis")
    .remove();
 
    vis.append("g")
    .attr("class", "axis")
    .attr("transform", "translate(0," + (yDomaines) + ")")
    .call(xAxis);
 }

 function displayArtists(data) {
   var vis = d3.select('#capc svg g.artists');

   var dataSize = data.length;
   //console.log(data);
   // SCALES
   var x = d3.scale.linear()
   .domain([0, d3.max(data, function(d) { return d['count']; })])
   .range([0, wArtists]);

   var maxArtists = d3.max(data, function(d) { return d['count']; });

   if (!globalMaxArtists) {
      globalMaxArtists = maxArtists;
   }

    var opacityScale = d3.scale.pow().exponent(0.5)
    .domain([0, globalMaxArtists])
    .range([0.2, 1]);


   var xOrdinal = d3.scale.linear()
   .domain([0, d3.max(data, function(d) { return d['count']; })])
   .range([wArtists + xArtists, xArtists]);
   
   var y = d3.scale.linear()
   .domain([0, dataSize - 1])
   .range([yArtists , yArtists + hArtists]);

   // WITH DATA (NEW)
   vis.selectAll("rect")
   .data(data)
   .enter().append("rect")
   .attr("x", function(d, i) { return wArtists + xArtists - x(d['count']);})
   .attr("y", function(d, i) { return i * barWidth + yArtists; })
   .attr("width", function(d) { return x(d['count']); })
   .attr("height", barWidth - 1)
   .attr("opacity", function(d) { return opacityScale(d['count']); })
   .transition()
   .duration(transitionDuration)
   .attr("opacity", 1);


   // FORCE UPDATE
   vis.selectAll("text")
    .remove();

   vis.selectAll("text")
   .data(data)
   .enter().append("text")
   .attr("x", function(d, i) { return wArtists + xArtists + textPaddingLeft;})
   .attr("y", function(d, i) { return i * barWidth + yArtists + textPaddingTop; })
   .attr("height", barWidth - 1)
   .attr("opacity", 0)
   .text(function(d) { return d['term']; })
   .transition()
   .duration(transitionDuration)
   .attr("opacity", 1);

   // UPDATE !!
   vis.selectAll("rect")
   .data(data)
   .transition()
   .duration(transitionDuration)
   .attr("x", function(d, i) { return wArtists + xArtists - x(d['count']);})
   .attr("y", function(d, i) { return i * barWidth + yArtists; })
   .attr("width", function(d) { return x(d['count']); })
   .attr("opacity", function(d) { return opacityScale(d['count']); });

    // Exit…
    vis.selectAll("rect")
     .data(data)
     .exit()
     .transition()
     .duration(transitionDuration)
     .attr("opacity", 0)
     .remove();


    // AXIS
    var xAxis = d3.svg.axis()
    .scale(xOrdinal)
    .tickFormat(formatAxis)
    .orient("top")
    .ticks(4);

    vis.selectAll("g.axis")
    .remove();
 
    vis.append("g")
    .attr("class", "axis")
    .attr("transform", "translate(0," + (yDomaines) + ")")
    .call(xAxis);
 }


 function doSearchByDomain(searchTerm, domain) {
   var filter = {
    "and" : [
    {
      "query" : { "match" : { "Domaine" : domain}}
    }
    ]
  };

  server.search(getQuery(searchTerm, filter), function(resp) { 
   displayYearsTopDomain(resp.facets.years.terms);
  });
 }

 function doSearchByArtist(searchTerm, artist) {
  var filter = {
    "and" : [
    {
      "query" : { "match" : { "Auteur" : artist}}
    }
    ]
  };

 server.search(getQuery(searchTerm, filter), function(resp) { 
  displayYearsTopArtist(resp.facets.years.terms);
 });

 }

 function getQuery(searchTerm, filter)
 {
   // ELASTICSEARCH QUERY DSL
   var esCall = {
       "query" : {
       "filtered" : {
         "query" : {
           "query_string" : {"query" : "*" + searchTerm + "*"},
         }
        }
      },
     "facets" : {
       "years" : {
         "terms" : {
           "field" : "Date",
           "size" : 1000000,
           "order" : "term"
         }
       },
       "domaines" : {
        "terms" : {
          // handle split on space
          "script_field" : "_source.Domaine",
          "all_terms" : true,
          "size" : 7
        }
      },
      "artists" : {
        "terms" : {
          // handle split on space
          "script_field" : "_source.Auteur",
          "all_terms" : true,
          "size" : 7
        }
      }
    }
  };
 
  if (filter) {
   esCall.query.filtered.filter = filter;
  }

  return esCall;
 }

 function doSearch(searchTerm) {

 server.search(getQuery(searchTerm), function(resp) { 
  displayYears(resp.facets.years.terms); 
  displayDomaines(resp.facets.domaines.terms); 
  displayArtists(resp.facets.artists.terms); 
  
  if (resp.facets.domaines.terms[0])
  {
    var topDomain = resp.facets.domaines.terms[0]['term'];
    doSearchByDomain(searchTerm, topDomain);

    var topArtist = resp.facets.artists.terms[0]['term'];
    doSearchByArtist(searchTerm, topArtist);
  }
  else {
    displayYearsTopDomain([]);
    displayYearsTopArtist([]);
  }

  });
}

  // WATCHING TEXT SEARCH INPUT
  $scope.$watch('search', function(newValue, oldValue) { 
    if (!newValue) {
     return;
   }

   doSearch(newValue);
 });

  // RUN !!!!
  initApp();

}]);

elasticsearch.yml

# always give a cluster name,
# it prevents bad suprises too with multicast discovery
cluster.name: d3demo

# deactivate multicast zen discovery in our case
discovery: 
    zen: 
        ping: 
            multicast: 
                enabled: false 

es.js

(function () {
  'use strict';

  var
    // save reference to global object
    // `window` in browser
    // `exports` on server
    root = this,
    es;

  // create namespace
  // use existing es object if it exists
  if (typeof exports !== 'undefined') {
    es = exports;
  } else {
    if (root.es == null) {
      es = root.es = {};
    } else {
      es = root.es;
    }
  }

  // get our server instance
  // define the es server inplementation
  es.server = function(url) {

    // server instance
    return {

      opt: {
        contentType: 'application/json',
        dataType: 'json',
        processData: false,
        url: url,
        error:  function( req, status, err ) {
          console.log( 'something went wrong on elascticsearch call', status, err );
        }
      },

      search: function(esQuery, callBack) {
        console.log(JSON.stringify(esQuery));
        this.opt.type = 'POST';
        this.opt.data = JSON.stringify(esQuery);

        this.opt.success = function( resp ) {
          console.log('query took ' + resp.took + 'ms');
          callBack(resp);
        };

        jQuery.ajax(this.opt);
      }
    }
  };

}).call(this);

gistfile1.txt

#!/bin/bash
# we use bulk api of elasticsearch
curl -s -XPOST localhost:9200/_bulk --data-binary @bulk-import-sample.json; echo

google-refine-templating

{ "index" : { "_index" : "musees", "_type" : "capc"} } 
{"Auteur" : {{jsonize(cells["Auteur"].value)}}, "Titre" : {{jsonize(cells["Titre"].value)}}, "Date" : {{jsonize(cells["Date"].value)}}, "Domaine" : {{jsonize(cells["Domaine"].value)}}, "Deno" : {{jsonize(cells["Deno"].value)}}, "MatSuppTech" : {{jsonize(cells["MatSuppTech"].value)}}, "Description" : {{jsonize(cells["Description"].value)}}, "Dimensions" : {{jsonize(cells["Dimensions"].value)}}, "Column 10" : {{jsonize(cells["Column 10"].value)}}, "Column 11" : {{jsonize(cells["Column 11"].value)}}, "Column 12" : {{jsonize(cells["Column 12"].value)}}, "Column 13" : {{jsonize(cells["Column 13"].value)}}, "Column 14" : {{jsonize(cells["Column 14"].value)}}, "Column 15" : {{jsonize(cells["Column 15"].value)}}, "Column 16" : {{jsonize(cells["Column 16"].value)}}, "Column 17" : {{jsonize(cells["Column 17"].value)}}, "Column 18" : {{jsonize(cells["Column 18"].value)}}, "Column 19" : {{jsonize(cells["Column 19"].value)}}, "Column 20" : {{jsonize(cells["Column 20"].value)}}, "Column 21" : {{jsonize(cells["Column 21"].value)}} }

start-es.sh

#!/bin/bash

# download es if needed
if [ ! -d elasticsearch-0.20.2 ]; then
    wget http://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.20.2.tar.gz
    tar xvzf elasticsearch-0.20.2.tar.gz
     
    #bigdesk plugin is great
    # check 
    ./elasticsearch-0.20.2/bin/plugin -install lukas-vlcek/bigdesk
fi

# run with our config file
./elasticsearch-0.20.2/bin/elasticsearch -Des.config=elasticsearch.yml -f