Skip to content

Instantly share code, notes, and snippets.

@stormpython
Last active March 21, 2022 13:45
Show Gist options
  • Save stormpython/8796112 to your computer and use it in GitHub Desktop.
Save stormpython/8796112 to your computer and use it in GitHub Desktop.
Data Visualization with Elasticsearch Aggregations and D3 (Tutorial)

Data Visualization with Elasticsearch Aggregations and D3

Link to the Elasticsearch Blog.

<!DOCTYPE html>
<html>
<head>
<title>Elastic Aggregations</title>
<script src="scripts/require.js"></script>
<script>require(["scripts/main"], function () {})</script>
<style>
body {
font: 14px sans-serif;
}
.arc path {
stroke: #fff;
stroke-width: 3px;
}
.node circle {
fill: #fff;
stroke: steelblue;
stroke-width: 1.5px;
}
.node {
font: 10px sans-serif;
}
.link {
fill: none;
stroke: #ccc;
stroke-width: 1.5px;
}
</style>
</head>
<body>
<div id="donut-chart"></div>
<div id="dendrogram"></div>
</body>
</html>
define(['scripts/d3.v3', 'scripts/elasticsearch'], function (d3, elasticsearch) {
"use strict";
var client = new elasticsearch.Client();
client.search({
index: 'nfl',
size: 5,
body: {
// Begin query.
query: {
// Boolean query for matching and excluding items.
bool: {
must: { match: { "description": "TOUCHDOWN" }},
must_not: { match: { "qtr": 5 }}
}
},
// Aggregate on the results
aggs: {
touchdowns: {
terms: {
field: "qtr",
order: { "_term" : "asc" }
}
}
}
// End query.
}
}).then(function (resp) {
console.log(resp);
// D3 code goes here.
var touchdowns = resp.aggregations.touchdowns.buckets;
// d3 donut chart
var width = 600,
height = 300,
radius = Math.min(width, height) / 2;
var color = ['#ff7f0e', '#d62728', '#2ca02c', '#1f77b4'];
var arc = d3.svg.arc()
.outerRadius(radius - 60)
.innerRadius(120);
var pie = d3.layout.pie()
.sort(null)
.value(function (d) { return d.doc_count; });
var svg = d3.select("#donut-chart").append("svg")
.attr("width", width)
.attr("height", height)
.append("g")
.attr("transform", "translate(" + width/1.4 + "," + height/2 + ")");
var g = svg.selectAll(".arc")
.data(pie(touchdowns))
.enter()
.append("g")
.attr("class", "arc");
g.append("path")
.attr("d", arc)
.style("fill", function (d, i) {
return color[i];
});
g.append("text")
.attr("transform", function (d) { return "translate(" + arc.centroid(d) + ")"; })
.attr("dy", ".35em")
.style("text-anchor", "middle")
.style("fill", "white")
.text(function (d) { return d.data.key; });
});
client.search({
index: 'nfl',
size: 5,
body: {
query: {
bool: {
must: { match: { "description": "TOUCHDOWN"}},
must_not: [
{ match: { "description": "intercepted"}},
{ match: { "description": "incomplete"}},
{ match: { "description": "FUMBLES"}},
{ match: { "description": "NULLIFIED"}}
]
}
},
aggs: {
teams: {
terms: {
field: "off",
exclude: "",
size: 5
},
aggs: {
players: {
terms: {
field: "description",
include: "([a-z]?[.][a-z]+)",
size: 200
},
aggs: {
qtrs: {
terms: {
field: "qtr",
order: { "_term": "asc" }
}
}
}
}
}
}
}
}
}).then(function (resp) {
console.log(resp);
// D3 code goes here.
var root = createChildNodes(resp);
// d3 dendrogram
var width = 600,
height = 2000;
var color = ['#ff7f0e', '#d62728', '#2ca02c', '#1f77b4'];
var cluster = d3.layout.cluster()
.size([height, width - 200]);
var diagonal = d3.svg.diagonal()
.projection(function(d) { return [d.y, d.x]; });
var svg = d3.select("#dendrogram").append("svg")
.attr("width", width)
.attr("height", height)
.append("g")
.attr("transform", "translate(120,0)");
var nodes = cluster.nodes(root),
links = cluster.links(nodes);
var link = svg.selectAll(".link")
.data(links)
.enter().append("path")
.attr("class", "link")
.attr("d", diagonal);
var node = svg.selectAll(".node")
.data(nodes)
.enter().append("g")
.attr("class", "node")
.attr("transform", function(d) { return "translate(" + d.y + "," + d.x + ")"; });
node.append("circle")
.attr("r", 4.5)
.style("fill", function (d) {
return d.children ? "#ffffff" : color[d.key - 1];
})
.style("stroke", function (d) {
return d.children ? "#4682B4" : color[d.key - 1];
});
node.append("text")
.attr("dx", function(d) { return d.children ? -8 : 8; })
.attr("dy", 3)
.style("text-anchor", function(d) { return d.children ? "end" : "start"; })
.text(function(d) { return d.children? d.key : d.key + ": " + d.doc_count; });
d3.select(self.frameElement).style("height", height + "px");
function createChildNodes(dataObj) {
var root = {};
root.key = "NFL";
root.children = dataObj.aggregations.teams.buckets;
root.children.forEach(function (d) { d.children = d.players.buckets; });
root.children.forEach(function (d) {
d.children.forEach(function (d) {
d.children = d.qtrs.buckets;
});
});
return root;
}
});
});
@roccia
Copy link

roccia commented Jan 5, 2016

OK I rewrite the mapping.json, it looks like this:

{
"mappings" : {

 "my_type": {

    "properties" : {

        "gameid" : {
            "type" : "string",
            "index" : "not_analyzed",
            "store" : "true"
        },
        "qtr" : {
            "type" : "short",
            "store" : "true"
        },

        "min" : {
            "type" : "short",
            "store" : "true",
             "ignore_malformed": "true"
            },

        "sec" : {
            "type" : "short",
            "store" : "true"

        },
        "off" : {
            "type" : "string"

        },
        "def" : {
            "type" : "string"

        },
        "down" : {
            "type" : "short",
            "store" : "true",
            "ignore_malformed": "true"

        },
        "togo" : {
            "type" : "short",
            "store" : "true",
            "ignore_malformed": "true"

        },
        "ydline" : {
            "type" : "short",
            "store" : "true",
            "ignore_malformed": "true"

        },
        "scorediff" : {
            "type" : "short",
            "store" : "true"
        },
        "series1stdn" : {
            "type" : "short",
            "store" : "true"
        },
        "description" : {
            "type" : "string",
            "store" : "true"
        },
        "scorechange" : {
            "type" : "short",
            "store" : "true"
        },
        "nextscore" : {
            "type" : "short",
            "store" : "true"
        },
        "teamwin" : {
            "type" : "short",
            "store" : "true"
        },
        "offscore" : {
            "type" : "short",
            "store" : "true"
        },
        "defscore" : {
            "type" : "short",
            "store" : "true"
        },
        "season" : {
            "type" : "date",
            "format" : "YYYY",
           "ignore_malformed": "true"


        }
    }
   }
}

}

but I'm still getting "reason" : "Root mapping definition has unsupported parameters: error , any idea why? need your help! thank you

@Veer-Abhimanyu
Copy link

@roccia try this..

@Veer-Abhimanyu
Copy link

{
"2013" : {
"properties" : {
"gameid" : {
"type" : "string",
"index" : "not_analyzed",
"store" : "yes"
},
"qtr" : {
"type" : "short",
"store" : "yes"
},
"min" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"sec" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"off" : {
"type" : "string",
"index" : "not_analyzed"
},
"def" : {
"type" : "string",
"index" : "not_analyzed"
},
"down" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"togo" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"ydline" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"scorediff" : {
"type" : "short",
"store" : "yes"
},
"series1stdn" : {
"type" : "short",
"store" : "yes"
},
"description" : {
"type" : "string",
"store" : "yes"
},
"scorechange" : {
"type" : "short",
"store" : "yes"
},
"nextscore" : {
"type" : "short",
"store" : "yes"
},
"teamwin" : {
"type" : "short",
"store" : "yes"
},
"offscore" : {
"type" : "short",
"store" : "yes"
},
"defscore" : {
"type" : "short",
"store" : "yes"
},
"season" : {
"type" : "date",
"format" : "YYYY",
"ignore_malformed" : "true"
}
}
}
}

@tiwarisahil91
Copy link

tiwarisahil91 commented Nov 21, 2016

Hi,while I am trying to view the chart on the http-server, I created using node js and with the following port no. as http://127.0.0.1:8080, I am getting this kind of error in my console.
XMLHttpRequest cannot load http://localhost:9200/nfl/2013/_search?size=5. No 'Access-Control-Allow-Origin' header is present on the requested resource. Origin 'http://127.0.0.1:8080' is therefore not allowed access.
Anybody knowing how to remove this

@aakritip12
Copy link

@tiwarisahil91 open run and enter chrome.exe --user-data-dir="C:/Chrome dev session" --disable-web-security . A new chrome browser will open run your url in that window

@rolfrennemo
Copy link

Hi,

I have a problem with this example. Please have a look at the screeen shots enclosed.
image

image

Thanks!

@VSJTech
Copy link

VSJTech commented Apr 19, 2018

hi

i am getting the following error while launching the server:

elasticsearch.js:19450 ERROR: 2018-04-19T10:24:55Z
Error: Request error, retrying -- Request failed to complete.
at Log.error (http://localhost:8001/scripts/elasticsearch.js:19151:56)
at checkRespForFailure (http://localhost:8001/scripts/elasticsearch.js:19817:18)
at XMLHttpRequest.xhr.onreadystatechange (http://localhost:8001/scripts/elasticsearch.js:18655:7)

@VSJTech
Copy link

VSJTech commented Apr 19, 2018

the following helped to get the issue cleared...

var elasticsearch = require('elasticsearch');
var AgentKeepAlive = require('agentkeepalive');
var client = new elasticsearch.Client({

host: 'localhost:9200',
requestTimeout: Infinity, // Tested
keepAlive: true ,// Tested

    maxRetries: 10,
    keepAlive: true,
    maxSockets: 10,
    minSockets: 10,
    createNodeAgent: function (connection, config) {
      return new AgentKeepAlive(connection.makeAgentConfig(config));
    }
});

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment