Create a gist now

Instantly share code, notes, and snippets.

@mansweet /README.md
Created Apr 15, 2016

Download ZIP
The Data Scientist's Guide to Apache Spark: Interactive Visualization of Results
Raw
README.md

Join the chat at https://gitter.im/Jay-Oh-eN/data-scientists-guide-apache-spark Binder

This gist contains a visualization of results from analysing DonorsChoose.org project data with Spark. The Spark code and notebooks can be found in the following repo: https://github.com/Jay-Oh-eN/data-scientists-guide-apache-spark

Materials

For the workshop (and after) we will use a Gitter chatroom to keep the conversation going: https://gitter.im/Jay-Oh-eN/data-scientists-guide-apache-spark.

And/or please do not hesitate to reach out to me directly via email at jondinu@gmail.com or over twitter @clearspandex

The presentation can be found on Slideshare here.

Data: http://data.donorschoose.org/open-data/project-data/

forked from Jay-Oh-eN's block: The Data Scientist's Guide to Apache Spark: Interactive Visualization of Results

Raw
data.json
[{"index":0.0,"count":391},{"index":49.65,"count":457},{"index":99.3,"count":940},{"index":148.95,"count":996},{"index":198.6,"count":964},{"index":248.25,"count":789},{"index":297.9,"count":777},{"index":347.55,"count":666},{"index":397.2,"count":632},{"index":446.85,"count":525},{"index":496.5,"count":415},{"index":546.15,"count":335},{"index":595.8,"count":235},{"index":645.45,"count":180},{"index":695.1,"count":162},{"index":744.75,"count":118},{"index":794.4,"count":131},{"index":844.05,"count":111},{"index":893.7,"count":83},{"index":943.35,"count":84},{"index":993.0,"count":110},{"index":1042.65,"count":66},{"index":1092.3,"count":80},{"index":1141.95,"count":51},{"index":1191.6,"count":47},{"index":1241.25,"count":31},{"index":1290.9,"count":41},{"index":1340.55,"count":43},{"index":1390.2,"count":33},{"index":1439.85,"count":22},{"index":1489.5,"count":25},{"index":1539.15,"count":26},{"index":1588.8,"count":28},{"index":1638.45,"count":16},{"index":1688.1,"count":30},{"index":1737.75,"count":15},{"index":1787.4,"count":19},{"index":1837.05,"count":10},{"index":1886.7,"count":11},{"index":1936.35,"count":14},{"index":1986.0,"count":18},{"index":2035.65,"count":23},{"index":2085.3,"count":10},{"index":2134.95,"count":10},{"index":2184.6,"count":7},{"index":2234.25,"count":24},{"index":2283.9,"count":7},{"index":2333.55,"count":11},{"index":2383.2,"count":5},{"index":2432.85,"count":8},{"index":2482.5,"count":3},{"index":2532.15,"count":9},{"index":2581.8,"count":3},{"index":2631.45,"count":7},{"index":2681.1,"count":4},{"index":2730.75,"count":6},{"index":2780.4,"count":2},{"index":2830.05,"count":6},{"index":2879.7,"count":2},{"index":2929.35,"count":2},{"index":2979.0,"count":9},{"index":3028.65,"count":3},{"index":3078.3,"count":5},{"index":3127.95,"count":3},{"index":3177.6,"count":1},{"index":3227.25,"count":2},{"index":3276.9,"count":7},{"index":3326.55,"count":1},{"index":3376.2,"count":2},{"index":3425.85,"count":1},{"index":3475.5,"count":2},{"index":3525.15,"count":1},{"index":3574.8,"count":0},{"index":3624.45,"count":1},{"index":3674.1,"count":0},{"index":3723.75,"count":3},{"index":3773.4,"count":0},{"index":3823.05,"count":3},{"index":3872.7,"count":0},{"index":3922.35,"count":2},{"index":3972.0,"count":2},{"index":4021.65,"count":0},{"index":4071.3,"count":4},{"index":4120.95,"count":0},{"index":4170.6,"count":2},{"index":4220.25,"count":1},{"index":4269.9,"count":1},{"index":4319.55,"count":2},{"index":4369.2,"count":1},{"index":4418.85,"count":1},{"index":4468.5,"count":1},{"index":4518.15,"count":2},{"index":4567.8,"count":0},{"index":4617.45,"count":2},{"index":4667.1,"count":3},{"index":4716.75,"count":2},{"index":4766.4,"count":1},{"index":4816.05,"count":1},{"index":4865.7,"count":2},{"index":4915.35,"count":2}]
Raw
index.html
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<script src="http://d3js.org/d3.v3.min.js"></script>
<style>
body {
font-family: futura;
}
.axis {
font-family: arial;
font-size: 0.7em;
}
.axis text {
font-size: 0.75em;
stroke: none;
font-family: 'Gill Sans', 'Gill Sans MT', Calibri, sans-serif;
}
text {
fill: black;
}
path {
fill: none;
stroke: black;
stroke-width: 1px;
}
.tick {
fill: none;
stroke: black;
}
rect {
fill: #4eb0bb;
}
#tooltip {
position: absolute;
width: 90px;
height: auto;
padding: 5px;
margin-bottom: 10px;
background-color: white;
-webkit-box-shadow: 4px 4px 10px rgba(0, 0, 0, 0.4);
-moz-box-shadow: 4px 4px 10px rgba(0, 0, 0, 0.4);
box-shadow: 4px 4px 10px rgba(0, 0, 0, 0.4);
pointer-events: none;
}
#tooltip.hidden {
display: none;
}
#tooltip p {
margin: 0;
font-family: sans-serif;
font-size: 16px;
line-height: 20px;
}
</style>
<script type="text/javascript">
// https://github.com/mbostock/d3/wiki/Time-Formatting
format = d3.time.format("%Y-%m");
function draw(data) {
"use strict";
// add a null data element so the axis extends past the last
// bar of the histogram.
data.push({ count: 0, index: 5000});
// set margins according to Mike Bostock's margin conventions
// http://bl.ocks.org/mbostock/3019563
var margin = {top: 25, right: 40, bottom: 50, left: 75};
// set height and width of chart
var width = 960 - margin.left - margin.right,
height = 500 - margin.top - margin.bottom;
// specify column we want to plot
var field = 'count';
// append the SVG tag with height and width to accommodate for margins
var svg = d3.select("body")
.append("svg")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append('g')
.attr('class','chart')
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
// Append the title for the graph
svg.append('text')
.attr('x', width / 2)
.attr('y', 0)
.style('text-anchor', 'middle')
.text("DonorsChoose.org Project Price")
// bind our data to svg circles for the scatter plot
svg.selectAll("rect")
.data(data)
.enter()
.append("rect")
// maximum count
var max_y = d3.max(data, function(d) {
return +d[field];
});
// get min/max price
var price_extent = d3.extent(data, function(d){
return +d['index'];
});
// Create x-axis scale mapping dates -> pixels
var price_scale = d3.scale.linear()
.range([0, width])
.domain(price_extent);
// Create y-axis scale mapping price -> pixels
var measure_scale = d3.scale.linear()
.range([height, 0])
.domain([0, max_y]);
// Create D3 axis object from time_scale for the x-axis
var price_axis = d3.svg.axis()
.scale(price_scale)
.tickFormat(d3.format('$'));
// Create D3 axis object from measure_scale for the y-axis
var measure_axis = d3.svg.axis()
.scale(measure_scale)
.orient("left");
// Append SVG to page corresponding to the D3 x-axis
var xaxis = svg.append('g')
.attr('class', 'x axis')
.attr('transform', "translate(0," + height + ")")
.call(price_axis);
// Append SVG to page corresponding to the D3 y-axis
svg.append('g')
.attr('class', 'y axis')
.call(measure_axis);
// add label to y-axis
d3.select(".y.axis")
.append("text")
.attr('class', 'label')
.text("Count")
.attr("transform", "rotate(-90)")
.attr("x", -(height / 2)).attr('y', -40)
.style("text-anchor", "middle")
.style("font-size", "1.5em");
var bin_width = (width / data.length) - 2;
// based on the data bound to each svg circle,
// change its center-x (cx) and center-y (cy)
// coordinates
var bars = d3.selectAll('rect')
.attr('x', function(d) {
return price_scale(d['index']) + 1;
})
.attr('width', bin_width)
.attr('y', function(d) {
return measure_scale(+d[field]);
})
.attr('height', function(d) {
return height - measure_scale(+d[field]);
});
bars.on("mousemove", function(d) {
// get current positions
var pos = d3.mouse(this);
var tool = d3.select("#tooltip");
var format = d3.format('$.2f');
//move the tooltip into position
var xpos = pos[0] + margin.left - parseFloat(tool.style('width')) / 2;
var ypos = pos[1] + margin.top - parseFloat(tool.style('height')) - 15;
// move tooltip div
tool.style("left", xpos + "px")
.style("top", ypos + "px");
// set price display
// tool.select("#key")
// .text(format(d['index']));
//set count value display
tool.select("#value")
.text(d['count']);
// show the tooltip
d3.select("#tooltip").classed("hidden", false);
})
.on("mouseout", function() {
// hide tooltip
d3.select("#tooltip").classed("hidden", true);
});
};
</script>
</head>
<body>
<script type="text/javascript">
// load our data file asynchronously and pass the data
// to the draw function once it is loaded.
d3.json("data.json", draw);
</script>
<div id="tooltip" class='hidden'>
<!-- <p><strong>Price: </strong><span id="key"></span></p> -->
<p style="margin-top: 5px"><strong>Count: </strong><span id="value"></span></p>
</div>
</body>
</html>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment