Skip to content

Instantly share code, notes, and snippets.

@stormpython
Last active March 21, 2022 13:45
Show Gist options
  • Save stormpython/8796112 to your computer and use it in GitHub Desktop.
Save stormpython/8796112 to your computer and use it in GitHub Desktop.
Data Visualization with Elasticsearch Aggregations and D3 (Tutorial)

Data Visualization with Elasticsearch Aggregations and D3

Link to the Elasticsearch Blog.

<!DOCTYPE html>
<html>
<head>
<title>Elastic Aggregations</title>
<script src="scripts/require.js"></script>
<script>require(["scripts/main"], function () {})</script>
<style>
body {
font: 14px sans-serif;
}
.arc path {
stroke: #fff;
stroke-width: 3px;
}
.node circle {
fill: #fff;
stroke: steelblue;
stroke-width: 1.5px;
}
.node {
font: 10px sans-serif;
}
.link {
fill: none;
stroke: #ccc;
stroke-width: 1.5px;
}
</style>
</head>
<body>
<div id="donut-chart"></div>
<div id="dendrogram"></div>
</body>
</html>
define(['scripts/d3.v3', 'scripts/elasticsearch'], function (d3, elasticsearch) {
"use strict";
var client = new elasticsearch.Client();
client.search({
index: 'nfl',
size: 5,
body: {
// Begin query.
query: {
// Boolean query for matching and excluding items.
bool: {
must: { match: { "description": "TOUCHDOWN" }},
must_not: { match: { "qtr": 5 }}
}
},
// Aggregate on the results
aggs: {
touchdowns: {
terms: {
field: "qtr",
order: { "_term" : "asc" }
}
}
}
// End query.
}
}).then(function (resp) {
console.log(resp);
// D3 code goes here.
var touchdowns = resp.aggregations.touchdowns.buckets;
// d3 donut chart
var width = 600,
height = 300,
radius = Math.min(width, height) / 2;
var color = ['#ff7f0e', '#d62728', '#2ca02c', '#1f77b4'];
var arc = d3.svg.arc()
.outerRadius(radius - 60)
.innerRadius(120);
var pie = d3.layout.pie()
.sort(null)
.value(function (d) { return d.doc_count; });
var svg = d3.select("#donut-chart").append("svg")
.attr("width", width)
.attr("height", height)
.append("g")
.attr("transform", "translate(" + width/1.4 + "," + height/2 + ")");
var g = svg.selectAll(".arc")
.data(pie(touchdowns))
.enter()
.append("g")
.attr("class", "arc");
g.append("path")
.attr("d", arc)
.style("fill", function (d, i) {
return color[i];
});
g.append("text")
.attr("transform", function (d) { return "translate(" + arc.centroid(d) + ")"; })
.attr("dy", ".35em")
.style("text-anchor", "middle")
.style("fill", "white")
.text(function (d) { return d.data.key; });
});
client.search({
index: 'nfl',
size: 5,
body: {
query: {
bool: {
must: { match: { "description": "TOUCHDOWN"}},
must_not: [
{ match: { "description": "intercepted"}},
{ match: { "description": "incomplete"}},
{ match: { "description": "FUMBLES"}},
{ match: { "description": "NULLIFIED"}}
]
}
},
aggs: {
teams: {
terms: {
field: "off",
exclude: "",
size: 5
},
aggs: {
players: {
terms: {
field: "description",
include: "([a-z]?[.][a-z]+)",
size: 200
},
aggs: {
qtrs: {
terms: {
field: "qtr",
order: { "_term": "asc" }
}
}
}
}
}
}
}
}
}).then(function (resp) {
console.log(resp);
// D3 code goes here.
var root = createChildNodes(resp);
// d3 dendrogram
var width = 600,
height = 2000;
var color = ['#ff7f0e', '#d62728', '#2ca02c', '#1f77b4'];
var cluster = d3.layout.cluster()
.size([height, width - 200]);
var diagonal = d3.svg.diagonal()
.projection(function(d) { return [d.y, d.x]; });
var svg = d3.select("#dendrogram").append("svg")
.attr("width", width)
.attr("height", height)
.append("g")
.attr("transform", "translate(120,0)");
var nodes = cluster.nodes(root),
links = cluster.links(nodes);
var link = svg.selectAll(".link")
.data(links)
.enter().append("path")
.attr("class", "link")
.attr("d", diagonal);
var node = svg.selectAll(".node")
.data(nodes)
.enter().append("g")
.attr("class", "node")
.attr("transform", function(d) { return "translate(" + d.y + "," + d.x + ")"; });
node.append("circle")
.attr("r", 4.5)
.style("fill", function (d) {
return d.children ? "#ffffff" : color[d.key - 1];
})
.style("stroke", function (d) {
return d.children ? "#4682B4" : color[d.key - 1];
});
node.append("text")
.attr("dx", function(d) { return d.children ? -8 : 8; })
.attr("dy", 3)
.style("text-anchor", function(d) { return d.children ? "end" : "start"; })
.text(function(d) { return d.children? d.key : d.key + ": " + d.doc_count; });
d3.select(self.frameElement).style("height", height + "px");
function createChildNodes(dataObj) {
var root = {};
root.key = "NFL";
root.children = dataObj.aggregations.teams.buckets;
root.children.forEach(function (d) { d.children = d.players.buckets; });
root.children.forEach(function (d) {
d.children.forEach(function (d) {
d.children = d.qtrs.buckets;
});
});
return root;
}
});
});
@stormpython
Copy link
Author

@agttga, my apologies for the late reply. I was unaware of the comment you made. I updated the link to the blog post which these files are supplemental material for. How to use these files is outlined there. Just click on the Blog link in the README.

@roccia
Copy link

roccia commented Jan 5, 2016

when I tried to use curl -XPUT localhost:9200/nfl/2013/_mapping?pretty -d @nfl_mapping.json
it keeps getting this error:
curl -XPUT localhost:9200/nfl/2013/_mapping?pretty -d @/Users/roccia/Downloads/nfl_mapping.json
{
"error" : {
"root_cause" : [ {
"type" : "mapper_parsing_exception",
"reason" : "Mapping definition for [_index] has unsupported parameters: [enabled : true]"
} ],
"type" : "mapper_parsing_exception",
"reason" : "Mapping definition for [_index] has unsupported parameters: [enabled : true]"
},
"status" : 400
}

it seems the issue has something related to this : elastic/elasticsearch#12356
in the nfl_mapping.json, the _index filed was set to ["enabled", true] which is no longer allowed to modified now.
not sure if I am right about this, if so would you mind fix the tutorial? Thanks

@roccia
Copy link

roccia commented Jan 5, 2016

OK I rewrite the mapping.json, it looks like this:

{
"mappings" : {

 "my_type": {

    "properties" : {

        "gameid" : {
            "type" : "string",
            "index" : "not_analyzed",
            "store" : "true"
        },
        "qtr" : {
            "type" : "short",
            "store" : "true"
        },

        "min" : {
            "type" : "short",
            "store" : "true",
             "ignore_malformed": "true"
            },

        "sec" : {
            "type" : "short",
            "store" : "true"

        },
        "off" : {
            "type" : "string"

        },
        "def" : {
            "type" : "string"

        },
        "down" : {
            "type" : "short",
            "store" : "true",
            "ignore_malformed": "true"

        },
        "togo" : {
            "type" : "short",
            "store" : "true",
            "ignore_malformed": "true"

        },
        "ydline" : {
            "type" : "short",
            "store" : "true",
            "ignore_malformed": "true"

        },
        "scorediff" : {
            "type" : "short",
            "store" : "true"
        },
        "series1stdn" : {
            "type" : "short",
            "store" : "true"
        },
        "description" : {
            "type" : "string",
            "store" : "true"
        },
        "scorechange" : {
            "type" : "short",
            "store" : "true"
        },
        "nextscore" : {
            "type" : "short",
            "store" : "true"
        },
        "teamwin" : {
            "type" : "short",
            "store" : "true"
        },
        "offscore" : {
            "type" : "short",
            "store" : "true"
        },
        "defscore" : {
            "type" : "short",
            "store" : "true"
        },
        "season" : {
            "type" : "date",
            "format" : "YYYY",
           "ignore_malformed": "true"


        }
    }
   }
}

}

but I'm still getting "reason" : "Root mapping definition has unsupported parameters: error , any idea why? need your help! thank you

@Veer-Abhimanyu
Copy link

@roccia try this..

@Veer-Abhimanyu
Copy link

{
"2013" : {
"properties" : {
"gameid" : {
"type" : "string",
"index" : "not_analyzed",
"store" : "yes"
},
"qtr" : {
"type" : "short",
"store" : "yes"
},
"min" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"sec" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"off" : {
"type" : "string",
"index" : "not_analyzed"
},
"def" : {
"type" : "string",
"index" : "not_analyzed"
},
"down" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"togo" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"ydline" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"scorediff" : {
"type" : "short",
"store" : "yes"
},
"series1stdn" : {
"type" : "short",
"store" : "yes"
},
"description" : {
"type" : "string",
"store" : "yes"
},
"scorechange" : {
"type" : "short",
"store" : "yes"
},
"nextscore" : {
"type" : "short",
"store" : "yes"
},
"teamwin" : {
"type" : "short",
"store" : "yes"
},
"offscore" : {
"type" : "short",
"store" : "yes"
},
"defscore" : {
"type" : "short",
"store" : "yes"
},
"season" : {
"type" : "date",
"format" : "YYYY",
"ignore_malformed" : "true"
}
}
}
}

@tiwarisahil91
Copy link

tiwarisahil91 commented Nov 21, 2016

Hi,while I am trying to view the chart on the http-server, I created using node js and with the following port no. as http://127.0.0.1:8080, I am getting this kind of error in my console.
XMLHttpRequest cannot load http://localhost:9200/nfl/2013/_search?size=5. No 'Access-Control-Allow-Origin' header is present on the requested resource. Origin 'http://127.0.0.1:8080' is therefore not allowed access.
Anybody knowing how to remove this

@aakritip12
Copy link

@tiwarisahil91 open run and enter chrome.exe --user-data-dir="C:/Chrome dev session" --disable-web-security . A new chrome browser will open run your url in that window

@rolfrennemo
Copy link

Hi,

I have a problem with this example. Please have a look at the screeen shots enclosed.
image

image

Thanks!

@VSJTech
Copy link

VSJTech commented Apr 19, 2018

hi

i am getting the following error while launching the server:

elasticsearch.js:19450 ERROR: 2018-04-19T10:24:55Z
Error: Request error, retrying -- Request failed to complete.
at Log.error (http://localhost:8001/scripts/elasticsearch.js:19151:56)
at checkRespForFailure (http://localhost:8001/scripts/elasticsearch.js:19817:18)
at XMLHttpRequest.xhr.onreadystatechange (http://localhost:8001/scripts/elasticsearch.js:18655:7)

@VSJTech
Copy link

VSJTech commented Apr 19, 2018

the following helped to get the issue cleared...

var elasticsearch = require('elasticsearch');
var AgentKeepAlive = require('agentkeepalive');
var client = new elasticsearch.Client({

host: 'localhost:9200',
requestTimeout: Infinity, // Tested
keepAlive: true ,// Tested

    maxRetries: 10,
    keepAlive: true,
    maxSockets: 10,
    minSockets: 10,
    createNodeAgent: function (connection, config) {
      return new AgentKeepAlive(connection.makeAgentConfig(config));
    }
});

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment