Reads a list of repositories from a gitweb HTML page and performs a 'git clone' or a 'git pull' on each repository

gitweb-html-code-download.js

/*
 * Reads a list of repositories from a gitweb HTML page and performs a 'git clone' or a 'git pull' on each repository
 * Useful when wanting to do a bulk download of code
 *
 * Author: Chad Maughan
 * Date: 5/24/13
 */
var http = require('http'),
  htmlparser = require('htmlparser2'),
  fs = require('fs'),
  shell = require('shelljs');


// check to see if git is installed, exit if it isn't
if (!shell.which('git')) {
  console.log('Sorry, this script requires git');
  shell.exit(1);
}

var domain = 'git.example.com';

// define the parser
var parser = new htmlparser.Parser({

  ontext: function (text) {
    if (text.substr(-4) === ".git") {
      console.log('-->', text);

      var directory = text.substr(0,text.length-4);
      var fullpath = __dirname + '/' + directory;

      // do a git pull if it exists, git clone otherwise
      if(fs.existsSync(fullpath)) {
        console.log('    exists');
        shell.exec('cd ' + directory + '; git pull');
        shell.exec('cd ..');
      }
      else {
        shell.exec('git clone gitosis@' + domain + ':' + text);
      }
    }
  }
});


// define the request options
var options = {
  hostname: domain,
  port: 80,
  path: '/gitweb/',
  method: 'GET'
};

var request = http.request(options, function (response) {

  console.log('STATUS: ' + response.statusCode);
  console.log('HEADERS: ' + JSON.stringify(response.headers));

  response.setEncoding('utf8');
  response.on('data', function (chunk) {
    parser.write(chunk);
    parser.end();
    // console.log('BODY: ' + chunk);
  });
});

request.on('error', function (e) {
  console.trace();
});

// write data to request body
request.write('data\n');
request.write('data\n');
request.end();