backupbrain

Tokenizer.prototype.padTokens = function(text) {
	tokenized = text.replace(/ ?([,\(\)\/\[\]\+\*\-]+) ?/g, " $1 ");
	tokenized = tokenized.replace(/ ?([,]) ?/g, " $1 ");
	tokenized = tokenized.replace(/- >/g, "->")
	tokenized = tokenized.replace(/ ?(==) ?/g, " $1 ");
	tokenized = tokenized.replace(/;/g, " ;");
	tokenized = tokenized.replace(/\s{2,}/g, " ");
	return tokenized;
}

backupbrain

Tokenizer.prototype.resetNewlines = function(text) {
	nlText = text.replace(/\n/g, "").replace(/\;/g, ";\n");
	nlText = nlText.replace(/\}/g, "}\n").replace(/\{/g, "\n{\n");
	nlText = nlText.replace(/\n[ ]+/g, "\n");
	nlText = nlText.replace(/\n$/, "");
	return nlText;
}

backupbrain

Tokenizer.prototype.trimWhitespace = function(text) {
	cleanedText = text.replace(/\n+/g, "\n");
	cleanedText = cleanedText.replace(/[ ]+/g, " ");
	cleanedText = cleanedText.trim();
	return cleanedText;
}

backupbrain

Tokenizer.prototype.stripComments = function(text) {
	return text.replace(/\/\/[^\n]+/g, "");
}

backupbrain

// Use Gists to store code you would like to remember later on
console.log(window); // log the "window" object to the console