Extract and index html pages into a Solr Cloud stack.

gistfile1.txt

var request = require('request');
var fs = require('fs');

var solr_host = 'http://<>-us-east-1.sr-dev.acquia.com';

var collection;
var host;

if(process.argv[2] && process.argv[3]){
	collection = process.argv[2];
	host = solr_host.replace(/<>/, process.argv[3]);
	console.log(collection, host);
	index();
}else{
	console.log('Usage: node solr.js <collection> <stack>');
}


function index(){
	fs.readdir('posts', function(err, data){
		extract_rec(0, data, function(){});
	});
}

function extract_rec(cur, files, callback){

	if(cur == files.length){
		console.log('All done.');
		return;
	}

	console.log(files[cur]);

	extract_and_index(collection, files[cur], function(err, data){
		if(err){
			console.log(err);
		}else{
			console.log(data);
		}
		extract_rec(cur + 1, files, callback);
	});
}

function extract_and_index(collection, file, callback){
	fs.readFile('posts/' + file, function(err, data){
		request.post({
			url: host + '/solr/' + collection + '/update/extract?literal.id=' + file + '&commit=true&wt=json',
			headers: {
				'content-type' : 'text/html'
			},
			body: data
		}, function (err, resp, body) {
			if(err){
				callback(err, null);
			}else{
				callback(null, body);
			}
		});
	});
}

function query(collection){
	request.get({
		url: solr_host + '/solr/' + collection + '/select?q=*:*&start=0&rows=0&wt=json'
	}, function (err, resp, body) {
		if(err){
			console.log(err);
		}else{
			console.log(body);
		}
	});
}