Skip to content

Instantly share code, notes, and snippets.

@kissrobber
Last active August 29, 2015 14:14
Show Gist options
  • Save kissrobber/6c40c7337f3e70487d82 to your computer and use it in GitHub Desktop.
Save kissrobber/6c40c7337f3e70487d82 to your computer and use it in GitHub Desktop.
// HOBBY AND CRAFTS - WEB ONLY
// var sites = [
// {
// name: "Michaels Stores",
// url: "http://www.michaels.com/"
// }, {
// name: "Hobby Lobby",
// url: "http://www.hobbylobby.com/home.cfm"
// }, {
// name: "Jo-Ann Fabric and Craft Stores",
// url: "http://www.joann.com/"
// }, {
// name: "A.C. Moore Arts & Crafts",
// url: "http://www.acmoore.com/"
// }
// ];
var casper = require("casper").create();
casper.on('remote.message', function(msg) {
this.echo('remote message caught: ' + msg);
});
casper.on( 'page.error', function (msg, trace) {
this.echo( 'Error: ' + msg, 'ERROR' );
});
casper.start();
casper.then(function(){
sites.forEach(function(site){
casper.thenOpen("http://builtwith.com/?" + site.url, function() {
site.analytics = casper.evaluate(function(){
var analytics = document.querySelectorAll('.techItem a[href^="//trends.builtwith.com/analytics/"]');
analytics = Array.prototype.filter.call(analytics, function(a){ return a.text.indexOf('Usage Statistics') == -1 });
return analytics.map(function(a){ return { name: a.innerText, url: a.href }; });
});
site.ads = casper.evaluate(function(){
var ads = document.querySelectorAll('.techItem a[href^="//trends.builtwith.com/ads/"]');
ads = Array.prototype.filter.call(ads, function(a){ return a.text.indexOf('Usage Statistics') == -1 });
return ads.map(function(a){ return { name: a.innerText, url: a.href }; });
});
});
});
});
casper.then(function(){
sites.forEach(function(site){
site.analytics.forEach(function(a){
casper.echo("... " + a['url']);
casper.thenOpen(a['url'], function() {
a['categories'] = casper.evaluate(function(){
var ps = document.querySelectorAll('.titleBox > .row > div > p');
ps = Array.prototype.filter.call(ps, function(d){ return d.innerHTML.indexOf('Categories') > 0 });
return Array.prototype.map.call(ps[0].children, function(a){ return a.innerText; });
});
});
});
});
});
casper.then(function(){
sites.forEach(function(site){
site.ads.forEach(function(a){
casper.echo("... " + a['url']);
casper.thenOpen(a['url'], function() {
a['categories'] = casper.evaluate(function(){
var ps = document.querySelectorAll('.titleBox > .row > div > p');
ps = Array.prototype.filter.call(ps, function(d){ return d.innerHTML.indexOf('Categories') > 0 });
return Array.prototype.map.call(ps[0].children, function(a){ return a.innerText; });
});
});
});
});
});
casper.run(function(){
// this.echo(JSON.stringify(sites));
this.echo('| name | url | analytics | analytics categries | ads | ad categries | AB testing |')
this.echo("|---|---|---|---|---|---|---|")
sites.forEach(function(s){
var line = "|" + s.name + "|" + s.url + "|";
var analyticsCates = [];
line += s.analytics.map(function(a){
a.categories.forEach(function(c){ analyticsCates.push(c) });
return a.name;
}).join(', ');
line += "|";
line += analyticsCates.filter(function (x, i, self) {
return self.indexOf(x) === i;
}).join(', ');
line += "|";
var adCates = [];
line += s.ads.map(function(a){
a.categories.forEach(function(c){ adCates.push(c) });
return a.name;
}).join(', ');
line += "|";
line += adCates.filter(function (x, i, self) {
return self.indexOf(x) === i;
}).join(', ');
line += "|";
line += analyticsCates.some(function(c){ return c == 'A/B Testing'}) ? "○" : "";
line += "|";
casper.echo(line);
});
this.echo('done.').exit();
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment