biovisualize/README.md

## README.md

      
    Raw
  

              README.md
            
          
    Scraping Gists with paging as CSV, for uploading to a spreadsheet like this list of Mike Bostock examples, as a first step to including batches of bl.ocks examples to the alternative D3 gallery

  
## gist_scraper.html
<!DOCTYPE html>
<html>
<head>
    <meta http-equiv="content-type" content="text/html; charset=UTF-8">
    <title></title>
    <script src="http://d3js.org/d3.v3.min.js"></script>
    <script src="http://d3js.org/queue.v1.min.js"></script>
</head>
<body>

<div id="grid"></div>

<script type="text/javascript">

var authorID = 'mbostock';
var clientID = 'YOUR_CLIENT_ID';
var clientSecret = 'YOUR_CLIENT_SECRET';

d3.json('https://api.github.com/users/'+authorID, query)

function query(error, results){
    var gistNum = results.public_gists;
    var pageNum = Math.ceil(gistNum/30);
    var q = queue(1);
    d3.range(pageNum).forEach(function(d, i) {
        q.defer(d3.json, 'https://api.github.com/users/'+authorID+'/gists?page='+(d)+'&client_id='+clientID+'&client_secret='+clientSecret);
    });
    q.awaitAll(formatResult);
}

function formatResult(error, results){
    var cleaned = d3.merge(results).map(function(d, i){
        obj = [];
        obj.push('"'+d.description+'"');
        obj.push('"'+'http://bl.ocks.org/'+authorID+'/'+d.id+'"');
        if(d.files['thumbnail.png']) obj.push('"'+d.files['thumbnail.png'].raw_url+'"');
        else obj.push('')
        return obj.join(',')
    });
    console.log(cleaned.join('\n'));
}

</script>

</body>
</html>
	<!DOCTYPE html>
	<html>
	<head>
	<meta http-equiv="content-type" content="text/html; charset=UTF-8">
	<title></title>
	<script src="http://d3js.org/d3.v3.min.js"></script>
	<script src="http://d3js.org/queue.v1.min.js"></script>
	</head>
	<body>

	<div id="grid"></div>

	<script type="text/javascript">

	var authorID = 'mbostock';
	var clientID = 'YOUR_CLIENT_ID';
	var clientSecret = 'YOUR_CLIENT_SECRET';

	d3.json('https://api.github.com/users/'+authorID, query)

	function query(error, results){
	var gistNum = results.public_gists;
	var pageNum = Math.ceil(gistNum/30);
	var q = queue(1);
	d3.range(pageNum).forEach(function(d, i) {
	q.defer(d3.json, 'https://api.github.com/users/'+authorID+'/gists?page='+(d)+'&client_id='+clientID+'&client_secret='+clientSecret);
	});
	q.awaitAll(formatResult);
	}

	function formatResult(error, results){
	var cleaned = d3.merge(results).map(function(d, i){
	obj = [];
	obj.push('"'+d.description+'"');
	obj.push('"'+'http://bl.ocks.org/'+authorID+'/'+d.id+'"');
	if(d.files['thumbnail.png']) obj.push('"'+d.files['thumbnail.png'].raw_url+'"');
	else obj.push('')
	return obj.join(',')
	});
	console.log(cleaned.join('\n'));
	}

	</script>

	</body>
	</html>