Reads a list of repositories from a gitweb TEXT page (gitweb/?a=project_index) and performs a 'git clone' or a 'git pull' on each repository

gitweb-text-code-download.js

/*
 * Reads a list of repositories from a gitweb TEXT page (gitweb/?a=project_index)
 * and performs a 'git clone' or a 'git pull' on each repository
 *
 * Author: Chad Maughan
 * Date: 5/24/13
 */
var http = require('http'),
  fs = require('fs'),
  path = require('path'),
  shell = require('shelljs');

if (!shell.which('git')) {
  console.log('Sorry, this script requires git');
  shell.exit(1);
}

var domain = 'git.example.com';

var process = function(line) {

  var text = line.split(' ')[0];

  // make sure it's a repository
  if (text.substr(-4) === ".git") {
    console.log('-->', text);

    var repository = text.substr(0, text.length-4);
    var workingDirectory = __dirname;

    // if the repository has a path delimiter, create the directory (if it doesn't exist)
    if(repository.indexOf(path.sep) != -1) {
      workingDirectory = __dirname + path.sep + repository.substring(0, repository.lastIndexOf(path.sep));
      if(!fs.existsSync(workingDirectory)) {
        console.log('creating target directory: ' + workingDirectory);
        shell.mkdir('-p', workingDirectory);
      }
    }

    // default command is to clone
    var command = 'git clone gitosis@' + domain + ':' + repository;

    // change to the checked out directory and do a git pull
    if(fs.existsSync(__dirname + path.sep + repository)) {
      console.log('     repository exists, updating');
      workingDirectory = __dirname + path.sep + repository;
      command = 'git pull';
    }

    shell.pushd(workingDirectory);
    shell.exec(command);
    shell.popd();

  }
}

// define the request options
var options = {
  hostname: domain,
  port: 80,
  path: '/gitweb/?a=project_index',
  method: 'GET'
};

var body = '';

// perform the HTTP GET
var request = http.request(options, function (response) {

  console.log('STATUS: ' + response.statusCode);
  console.log('HEADERS: ' + JSON.stringify(response.headers));

  response.setEncoding('utf8');

  response.on('data', function (chunk) {
    body += chunk + '\n';
  });

  response.on('end', function () {
    body.split(/\n/g).forEach(process);
  });
});

request.on('error', function (e) {
  console.trace(e);
});
request.end();