Crawl the all sites page and check each site is up

govcms-all-sites-check.js

// Run with:
// casperjs govcms-all-sites-check.js


// First will get all links from a page
// then save 'href' attributes to an array,
// then will iterate over this array and then open each link one by one
// and echo the response code, url and title


var casper = require('casper').create({
    // logLevel:"debug",
    // verbose: true
    
    // lessen the number of requests we make
    pageSettings: {
      loadImages:  false,        // do not load images
      loadPlugins: false         // do not load NPAPI plugins (Flash, Silverlight, ...)
    }
});
var links;

// count up the totals
var total200             = 0;
var total301             = 0;
var totalError           = 0;
var totalUndefined       = 0;
var totalOther           = 0;
var totalPipeTitleIssues = 0;

casper.start('https://www.govcms.gov.au/our-customers');

casper.then(function getLinks(){
    links = this.evaluate(function(){
        var randQuery = "?casperjs" + Math.floor((Math.random() * 100) + 1) + "_";
        var links = document.querySelectorAll('a.card');
        links = Array.prototype.map.call(links,function(link){
          return link.getAttribute('href').replace(/ /g,'').concat(randQuery);   // strip out extra white spaces from href and append random query string for cachebusting
        });
        return links;
    });
});

// Loop through each site
casper.then(function(){

  this.echo("");
  this.echo("Checking all live, public govCMS sites (both SaaS + PaaS)...");
  this.echo("");
  this.echo("---");

  this.each(links,function(self,link){
    self.thenOpen(link,function(a){
      this.echo("");

      var res = this.status(false);
      var response = res.currentHTTPStatus;

      if (response == 200) {                // all good
        this.echo("✅  "+response);
        total200++;
      } else if (response == 301) {         // perm redirect
        this.echo("🔂  " + response);
        total301++;
      } else if (response >= 400) {         // problem
        this.echo("⛔️  " + response);
        totalError++;
      } else if (response == undefined) {   // small problem
        this.echo("❓  " + response);
        totalUndefined++;
      } else {                              // probably still fine, but just check (could be a 302 for example)
        this.echo("⚠️  " + response);
        totalOther++;
      }

      var origLink = this.getCurrentUrl();
      origLink = origLink.split("?")[0];
      this.echo(origLink);

      this.echo(this.getTitle());

      // Check for old govCMS "|" title issue...
      if (this.getTitle().trim().slice(-1) == "|") {
        totalPipeTitleIssues++;
      }

      this.echo("");
      this.echo("---");
    });
  });
});

// Totals
casper.then(function(){
  this.echo("");
  this.echo("Totals:");
  this.echo("✅  " + total200);
  this.echo("🔂  " + total301);
  this.echo("⛔️  " + totalError);
  this.echo("⚠️  " + totalOther);
  this.echo("❓  " + totalUndefined);
  if (totalPipeTitleIssues > 0) {
    this.echo("btw... 😒  " + totalPipeTitleIssues + " sites still have the page title pipe issue (https://github.com/govCMS/govCMS/issues/303)");
  }
  this.echo("");
});

casper.run(function(){
  this.exit();
});