jzollerneon/.block

## .block
license: mit

## README.md

      
    Raw
  

              README.md
            
          
    Built with blockbuilder.org
When NEON was first starting to offer small mammal box trapping data, I wanted a way to look at capture distributions.  So, I put together a small D3.js visualization that showed a capture distribution heat map by counting trap locations from the capture file.
This code updates the visualization to use the NEON Data API to grab current data from the portal.  The code queries the /products/DP1.10072.001 endpoint to see what data is available for small mammal capture.  It uses that to populate a dropdown with site codes, and requests a list of months available for the currently selected site code.  From there, it requests a list of files for each month, then pulls the capture data file for that month.  For each site, it stitches the file contents together, then splits them up by plot, and creates the heatmaps.
Click the "Open" icon to the right to see this full screen!.

  
## index.html
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Small mammal capture heatmap</title>
    <script src="https://code.jquery.com/jquery-3.1.1.min.js"
            integrity="sha256-hVVnYaiADRTO2PzUGmuLJr8BLUSjGIZsDYGmIJLv2b8=" crossorigin="anonymous"></script>
    <script src="https://d3js.org/d3.v4.js"></script>

    <script src="smammal.js"></script>
</head>
<body>

<select id="sites"></select>
<div id="smammal"></div>

<style>
    .grid {
        display: inline;
    }

    #desc {
        padding: 10px;
    }

    select {
        margin: 20px;
        border: 1px solid #111;
        background: transparent;
        width: 150px;
        padding: 5px;
        font-size: 20px;
        height: 34px;
        /*-webkit-appearance: none;*/
        /*-moz-appearance: none;*/
        /*appearance: none;*/
    }

</style>
<script>
    var SERVER = "http://data.neonscience.org/api/v0/";
    var SMAMMAL = "DP1.10072.001";
    var availableData = {};
    var nestedData = {};
    var smammalGrid = smammal.smammalGrid();

    //when the page loads, grab the list of all small mammal data from the /products endpoint
    var productsUrl = SERVER + "products/" + SMAMMAL;
    $.getJSON(productsUrl, function (result) {
        result.data.siteCodes.forEach(function (d) {
            availableData[d.siteCode] = [];
            d.availableMonths.forEach(function (e) {
                availableData[d.siteCode].push(e);
            });
        });

        //populate the list of sites and months
        var siteOptions = $("#sites");
        $.each(Object.keys(availableData).sort(), function () {
            siteOptions.append($("<option />").val(this).text(this));
        });
        //get the data for the first siteCode on the list
        getData($("#sites").val());
    });

    //get the capture data for a given siteCode
    function getData(siteCode) {
        var combinedData = [];
        var urlList = [];

        //queue up the availability request for each month, and treat all the results in the awaitAll function
        var q = d3.queue();
        availableData[siteCode].forEach(function (month) {
            var requestUrl = SERVER + "data/" + SMAMMAL + "/" + siteCode + "/" + month;
            q.defer(d3.json, requestUrl);
        });

        //handle all of the available data queries
        q.awaitAll(function (availableError, availableResult) {
            availableResult.forEach(function (d, index, arr) {
                for (var i = 0; i < d.data.files.length; i++) {
                    var filename = d.data.files[i].url;
                    if (filename.includes("pertrapnight") && !filename.includes("expanded")) {
                        urlList.push(d.data.files[i].url);
                    }
                }
            });

            //queue up the files to be requested
            var r = d3.queue();
            urlList.forEach(function(e) {
                r.defer(d3.csv, e);
            });

            //handle all of the downloaded files
            r.awaitAll(function(dataError, dataResult) {
                dataResult.forEach(function (fileResult) {
                    fileResult.forEach(function(row) {
                        //trapCoordinates are stored without zero or X padding, so make sure it's in there
                        row.trapCoordinate = pad(row.trapCoordinate);
                        combinedData.push(row);
                    });
                });

                //break the data down by plotID
                nestedData = d3.nest().key(function(f) { return f.plotID; }).entries(combinedData);
                nestedData = nestedData.sort(function(a,b) {
                    if (a.key > b.key) { return 1; } else { return -1;}
                });

                //build a grid for each plotId
                nestedData.forEach(function(dataSet) {
                    var smammalDiv = $("#smammal");
                    $('<div class="grid" id="' + dataSet.key + '"></div>').appendTo(smammalDiv);
                    d3.select('#' + dataSet.key)
                            .datum(dataSet.values)
                            .call(smammalGrid
                                    .width(300)
                                    .height(300)
                                    .title(dataSet.key));
                });

            });

        });
    }

    //pad location with 0 or X as required
    function pad(padstr) {
        if (padstr.length == 2) {
            if (padstr.substring(1, 2) !== 'X') {
                padstr = padstr.substring(0, 1) + '0' + padstr.substring(1, 2);
            } else {
                padstr = padstr.substring(0, 1) + 'X' + padstr.substring(1, 2);
            }
        }
        return padstr;
    }

    //events
    $("#sites").change(function () {
        $("#smammal").empty();
        $("#desc").empty();
        getData(this.value);
    });

</script>
</body>
</html>

## smammal.js
var smammal = {
    smammalGrid: function smammalGrid() {
        var rows = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'X'],
            cols = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', 'X'],
            margin = {top: 30, right: 10, bottom: 10, left: 10},
            width = 600 - margin.left - margin.right,
            height = 600 - margin.top - margin.bottom,
            title = '',
            gridCount = rows.length,
            gridMargin = 2,
            colors = ["#ffffd9", "#edf8b1", "#c7e9b4", "#7fcdbb", "#41b6c4", "#1d91c0", "#225ea8", "#253494", "#081d58"],
            buckets = colors.length;

        function grid(selection) {
            //set up retained values
            var closureWidth = width,
                closureHeight = height,
                closureTitle = title,
                gridSize = Math.floor(closureWidth / gridCount),
                closureGridMargin = gridMargin,
                rowColsObj = {},
                rowColsArr = [];

            //initialize the backing data for each cell
            rows.forEach(function (row, rowindex) {
                cols.forEach(function (col, colindex) {
                    var rc = row + col;
                    rc = pad(rc);
                    rowColsObj[rc] = {};
                    rowColsObj[rc].row = row;
                    rowColsObj[rc].col = col;
                    rowColsObj[rc].count = 0;
                    rowColsObj[rc].trapCoordinate = rc;
                })
            });

            //for each input data item
            selection.each(function (data) {
                //count captures
                data.forEach(function (d) {
                    if (d.trapStatus.indexOf("5") != -1) {
                        rowColsObj[d.trapCoordinate].count = rowColsObj[d.trapCoordinate].count + 1;
                    }
                });

                //map everything out to an array from the object
                rowColsArr = Object.keys(rowColsObj).map(function (key) {
                    return rowColsObj[key];
                });
                rowColsArr.sort(function (a, b) {
                    if (a.trapCoordinate < b.trapCoordinate) { return -1;} else { return 1;}
                });

                //TODO: replace with a constant color scale
                var colorScale = d3.scaleQuantile()
                    .domain([0, buckets - 1, d3.max(rowColsArr, function (d) {
                        return d.count;
                    })])
                    .range(colors);

                //build the top level svg
                var svg = d3.select('#smammal').append('svg')
                    .attr('width', closureWidth + margin.left + margin.right)
                    .attr('height', closureHeight + margin.top + margin.bottom)
                    .append('g')
                    .attr('transform', 'translate(' + margin.left + ', ' + margin.top + ')');

                //build the grid
                var ycount = -1;
                svg.selectAll('rect')
                    .data(rowColsArr)
                    .enter()
                    .append('rect')
                    .attr('x', function (d, i) {
                        return i % gridCount * gridSize;
                    })
                    .attr('y', function (d, i) {
                        if (i % gridCount === 0) ycount = ycount + 1;
                        return ycount * gridSize;
                    })
                    .attr('rx', 3)
                    .attr('ry', 3)
                    .attr('width', gridSize - closureGridMargin)
                    .attr('height', gridSize - closureGridMargin)
                    .style('fill', function (d) {
                        return colorScale(d.count);
                    })
                    .append('svg:title')
                    .text(function (d) {
                        return "Coordinate " + d.trapCoordinate + ': ' + d.count + ' captures';
                    });

                //build the title
                svg.append("text")
                    .attr("x", (width / 2))
                    .attr("y", 0 - (margin.top / 2))
                    .attr("text-anchor", "middle")
                    .style("font-size", "20px")
                    .text(closureTitle);
            });
        };

        grid.width = function (value) {
            if (!arguments.length) return width;
            width = value;
            return grid;
        };

        grid.height = function (value) {
            if (!arguments.length) return height;
            height = value;
            return grid;
        };

        grid.title = function (value) {
            if (!arguments.length) return title;
            title = value;
            return grid;
        };

        grid.gridMargin = function (value) {
            if (!arguments.length) return gridMargin;
            gridMargin = value;
            return grid;
        };

        return grid;
    }

}
	<!DOCTYPE html>
	<html lang="en">
	<head>
	<meta charset="UTF-8">
	<title>Small mammal capture heatmap</title>
	<script src="https://code.jquery.com/jquery-3.1.1.min.js"
	integrity="sha256-hVVnYaiADRTO2PzUGmuLJr8BLUSjGIZsDYGmIJLv2b8=" crossorigin="anonymous"></script>
	<script src="https://d3js.org/d3.v4.js"></script>

	<script src="smammal.js"></script>
	</head>
	<body>

	<select id="sites"></select>
	<div id="smammal"></div>

	<style>
	.grid {
	display: inline;
	}

	#desc {
	padding: 10px;
	}

	select {
	margin: 20px;
	border: 1px solid #111;
	background: transparent;
	width: 150px;
	padding: 5px;
	font-size: 20px;
	height: 34px;
	/-webkit-appearance: none;/
	/-moz-appearance: none;/
	/appearance: none;/
	}

	</style>
	<script>
	var SERVER = "http://data.neonscience.org/api/v0/";
	var SMAMMAL = "DP1.10072.001";
	var availableData = {};
	var nestedData = {};
	var smammalGrid = smammal.smammalGrid();

	//when the page loads, grab the list of all small mammal data from the /products endpoint
	var productsUrl = SERVER + "products/" + SMAMMAL;
	$.getJSON(productsUrl, function (result) {
	result.data.siteCodes.forEach(function (d) {
	availableData[d.siteCode] = [];
	d.availableMonths.forEach(function (e) {
	availableData[d.siteCode].push(e);
	});
	});

	//populate the list of sites and months
	var siteOptions = $("#sites");
	$.each(Object.keys(availableData).sort(), function () {
	siteOptions.append($("<option />").val(this).text(this));
	});
	//get the data for the first siteCode on the list
	getData($("#sites").val());
	});

	//get the capture data for a given siteCode
	function getData(siteCode) {
	var combinedData = [];
	var urlList = [];

	//queue up the availability request for each month, and treat all the results in the awaitAll function
	var q = d3.queue();
	availableData[siteCode].forEach(function (month) {
	var requestUrl = SERVER + "data/" + SMAMMAL + "/" + siteCode + "/" + month;
	q.defer(d3.json, requestUrl);
	});

	//handle all of the available data queries
	q.awaitAll(function (availableError, availableResult) {
	availableResult.forEach(function (d, index, arr) {
	for (var i = 0; i < d.data.files.length; i++) {
	var filename = d.data.files[i].url;
	if (filename.includes("pertrapnight") && !filename.includes("expanded")) {
	urlList.push(d.data.files[i].url);
	}
	}
	});

	//queue up the files to be requested
	var r = d3.queue();
	urlList.forEach(function(e) {
	r.defer(d3.csv, e);
	});

	//handle all of the downloaded files
	r.awaitAll(function(dataError, dataResult) {
	dataResult.forEach(function (fileResult) {
	fileResult.forEach(function(row) {
	//trapCoordinates are stored without zero or X padding, so make sure it's in there
	row.trapCoordinate = pad(row.trapCoordinate);
	combinedData.push(row);
	});
	});

	//break the data down by plotID
	nestedData = d3.nest().key(function(f) { return f.plotID; }).entries(combinedData);
	nestedData = nestedData.sort(function(a,b) {
	if (a.key > b.key) { return 1; } else { return -1;}
	});

	//build a grid for each plotId
	nestedData.forEach(function(dataSet) {
	var smammalDiv = $("#smammal");
	$('<div class="grid" id="' + dataSet.key + '"></div>').appendTo(smammalDiv);
	d3.select('#' + dataSet.key)
	.datum(dataSet.values)
	.call(smammalGrid
	.width(300)
	.height(300)
	.title(dataSet.key));
	});

	});

	});
	}

	//pad location with 0 or X as required
	function pad(padstr) {
	if (padstr.length == 2) {
	if (padstr.substring(1, 2) !== 'X') {
	padstr = padstr.substring(0, 1) + '0' + padstr.substring(1, 2);
	} else {
	padstr = padstr.substring(0, 1) + 'X' + padstr.substring(1, 2);
	}
	}
	return padstr;
	}

	//events
	$("#sites").change(function () {
	$("#smammal").empty();
	$("#desc").empty();
	getData(this.value);
	});

	</script>
	</body>
	</html>
	var smammal = {
	smammalGrid: function smammalGrid() {
	var rows = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'X'],
	cols = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', 'X'],
	margin = {top: 30, right: 10, bottom: 10, left: 10},
	width = 600 - margin.left - margin.right,
	height = 600 - margin.top - margin.bottom,
	title = '',
	gridCount = rows.length,
	gridMargin = 2,
	colors = ["#ffffd9", "#edf8b1", "#c7e9b4", "#7fcdbb", "#41b6c4", "#1d91c0", "#225ea8", "#253494", "#081d58"],
	buckets = colors.length;

	function grid(selection) {
	//set up retained values
	var closureWidth = width,
	closureHeight = height,
	closureTitle = title,
	gridSize = Math.floor(closureWidth / gridCount),
	closureGridMargin = gridMargin,
	rowColsObj = {},
	rowColsArr = [];

	//initialize the backing data for each cell
	rows.forEach(function (row, rowindex) {
	cols.forEach(function (col, colindex) {
	var rc = row + col;
	rc = pad(rc);
	rowColsObj[rc] = {};
	rowColsObj[rc].row = row;
	rowColsObj[rc].col = col;
	rowColsObj[rc].count = 0;
	rowColsObj[rc].trapCoordinate = rc;
	})
	});

	//for each input data item
	selection.each(function (data) {
	//count captures
	data.forEach(function (d) {
	if (d.trapStatus.indexOf("5") != -1) {
	rowColsObj[d.trapCoordinate].count = rowColsObj[d.trapCoordinate].count + 1;
	}
	});

	//map everything out to an array from the object
	rowColsArr = Object.keys(rowColsObj).map(function (key) {
	return rowColsObj[key];
	});
	rowColsArr.sort(function (a, b) {
	if (a.trapCoordinate < b.trapCoordinate) { return -1;} else { return 1;}
	});

	//TODO: replace with a constant color scale
	var colorScale = d3.scaleQuantile()
	.domain([0, buckets - 1, d3.max(rowColsArr, function (d) {
	return d.count;
	})])
	.range(colors);

	//build the top level svg
	var svg = d3.select('#smammal').append('svg')
	.attr('width', closureWidth + margin.left + margin.right)
	.attr('height', closureHeight + margin.top + margin.bottom)
	.append('g')
	.attr('transform', 'translate(' + margin.left + ', ' + margin.top + ')');

	//build the grid
	var ycount = -1;
	svg.selectAll('rect')
	.data(rowColsArr)
	.enter()
	.append('rect')
	.attr('x', function (d, i) {
	return i % gridCount * gridSize;
	})
	.attr('y', function (d, i) {
	if (i % gridCount === 0) ycount = ycount + 1;
	return ycount * gridSize;
	})
	.attr('rx', 3)
	.attr('ry', 3)
	.attr('width', gridSize - closureGridMargin)
	.attr('height', gridSize - closureGridMargin)
	.style('fill', function (d) {
	return colorScale(d.count);
	})
	.append('svg:title')
	.text(function (d) {
	return "Coordinate " + d.trapCoordinate + ': ' + d.count + ' captures';
	});

	//build the title
	svg.append("text")
	.attr("x", (width / 2))
	.attr("y", 0 - (margin.top / 2))
	.attr("text-anchor", "middle")
	.style("font-size", "20px")
	.text(closureTitle);
	});
	};

	grid.width = function (value) {
	if (!arguments.length) return width;
	width = value;
	return grid;
	};

	grid.height = function (value) {
	if (!arguments.length) return height;
	height = value;
	return grid;
	};

	grid.title = function (value) {
	if (!arguments.length) return title;
	title = value;
	return grid;
	};

	grid.gridMargin = function (value) {
	if (!arguments.length) return gridMargin;
	gridMargin = value;
	return grid;
	};

	return grid;
	}

	}