Fast module hacking

loadModule.js

// var fs = require("fs");
// var Module = require("module");

// var pathCache = JSON.parse(fs.readFileSync("/sv/test/pathCache.json").toString());
// Object.keys(pathCache).forEach(function(val, i) {
// 	Module._pathCache[val] = pathCache[val];
// });

// console.time("load");
// // var mongolayer = require("/sv/node_modules/npm/mongolayer/1/node_modules/mongolayer/");
// var siteLib = require("/sv/node_modules/sv/siteLib/1/");
// console.timeEnd("load");

// var Client = require("/sv/node_modules/sv/siteLib/1/Client.js");

var fs = require("fs");
var vm = require("vm");
// var native_module = require("native_module");
var Module = require("module");
// var content = fs.readFileSync("/sv/node_modules/sv/siteLib/1/index.js").toString();

// var temp = new vm.Script(native_module.wrap(content), { produceCachedData : true });
// fs.writeFileSync("/tmp/cachedData", temp.cachedData);

// var newRequire = function() {
// 	var buffer = fs.readFileSync("/tmp/cachedData");
// 	var temp = new vm.Script(content, { cachedData : buffer });
// 	// var module = { exports : {} };
// 	temp.runInThisContext(native_module.wrap(temp));
// 	// temp.runInNewContext({ exports : module.exports, exports : exports, require : require, __filename : "/sv/node_modules/sv/siteLib/1/index.js", __dirname : "/sv/node_modules/sv/siteLib/1" });
// 	return module.exports;
// }

var fileCache = {};

var readDir = 0;

// cache file reads
var oldRead = fs.readFileSync.bind(fs);
fs.readFileSync = function(filename, options) {
	var start = Date.now();
	var content = oldRead.apply(null, arguments);
	readDir += Date.now() - start;
	
	fileCache[filename] = content;
	
	return content;
}

///// LOAD MODULE WORKING
var numRun = 0;
var runCount = 0;

var contextCache = {};

// cache code compilation
var old = vm.runInThisContext.bind(vm);
vm.runInThisContext = function(code, options) {
	var cacheKey = options.filename + "_" + code;
	
	var start = Date.now();
	var ret = old.apply(null, arguments);
	runCount += Date.now() - start;
	
	if (options.filename.indexOf(".node") === -1 && options.filename.indexOf("bcrypt") === -1) {
		var temp = new vm.Script(code, { produceCachedData : true });
		contextCache[cacheKey] = temp.cachedData;
	}
	
	return ret;
}

var siteLib = require("/sv/node_modules/sv/siteLib/1/");

// write the file cache
fs.writeFileSync("/tmp/fileCache", JSON.stringify(fileCache));
// context cache
fs.writeFileSync("/tmp/cachedData", JSON.stringify(contextCache));
// path cache
fs.writeFileSync("/tmp/pathCache", JSON.stringify(Module._pathCache));

// Object.keys(require.cache).forEach(function(val) {
// 	if (val.indexOf(".node") > -1) {
// 		return;
// 	}
	
// 	delete require.cache[val];
// });

console.log("readDir", readDir);
// console.log("runCount", runCount);


// var siteLib = require("/sv/node_modules/sv/siteLib/1/");

// console.log("runCount", runCount);

// console.log("siteLib", siteLib);

// var fs = require("fs");
// var vm = require("vm");
// console.time("read file");
// var content = fs.readFileSync("/sv/node_modules/sv/scriptsLib/1/index.js").toString();
// console.timeEnd("read file");

// console.time("test");
// var temp = new vm.Script(content, { produceCachedData : true });
// console.timeEnd("test");

// console.log(content.length, temp.cachedData.length);

// console.time("test2");
// var test = new vm.Script(content, { cachedData : temp.cachedData });
// console.timeEnd("test2");

// // console.log(temp.cachedData);

// // var context = vm.createContext({ require : require, module : module });

// console.time("run context");
// var maybe = test.runInNewContext({ require : require, module : module });
// console.timeEnd("run context");

// console.time("native");
// var scriptsLib = require("/sv/node_modules/sv/scriptsLib/1/index.js");
// console.timeEnd("native");

// console.log(Object.keys(maybe));

loadModuleCache.js

var vm = require("vm");
var fs = require("fs");
var Module = require("module");

// var fileCache = JSON.parse(fs.readFileSync("/tmp/fileCache").toString());

// var pathCache = JSON.parse(fs.readFileSync("/tmp/pathCache").toString());
// Object.keys(pathCache).forEach(function(val, i) {
// 	Module._pathCache[val] = pathCache[val];
// });

// var contextCache = JSON.parse(fs.readFileSync("/tmp/cachedData"));
// Object.keys(contextCache).forEach(function(val) {
// 	contextCache[val] = Buffer.from(contextCache[val], "utf-8");
// });

// var oldRead = fs.readFileSync.bind(fs);
// fs.readFileSync = function(filename, options) {
// 	if (fileCache[filename]) {
// 		return fileCache[filename];
// 	}
	
// 	return oldRead.apply(null, arguments);
// }

// var old = vm.runInThisContext.bind(vm);
// vm.runInThisContext = function(code, options) {
// 	var cacheKey = options.filename + "_" + code;
// 	var prev = contextCache[cacheKey];
// 	if (prev) {
// 		// console.log("loading from cache", options.filename, code.length);
// 		// var start = Date.now();
// 		var temp = new vm.Script(code, { cachedData : prev });
// 		var ret = temp.runInThisContext(options);
// 		// runCount += Date.now() - start;
		
// 		return ret;
// 	}
	
// 	return old.apply(null, arguments);
// }

console.time("load");
var siteLib = require("/sv/node_modules/sv/siteLib/1/");
console.timeEnd("load");

maybe_worse_read.js

var vm = require("vm");
var fs = require("fs");
var Module = require("module");
var path = require("path");
var internalModule = require("./internalModule.js");

var oldFs = fs.readFileSync;
fs.readFileSync = function(path) {
	console.log("reading", path);
	return oldFs.apply(null, arguments);
}

console.time("dat");
var blob = fs.readFileSync("/tmp/cache.dat");
console.timeEnd("dat");

console.time("raw");
var raw = fs.readFileSync("/tmp/cache.json");
console.timeEnd("raw");

console.time("parse");
var cache = JSON.parse(raw);
console.timeEnd("parse");

for(var i in cache.pathCache) {
	Module._pathCache[i] = cache.pathCache[i];
}

var oldJs = Module._extensions[".js"];
Module._extensions[".js"] = function(myModule, filename) {
	// console.log("filename", filename);
	
	if (cache.modules[filename] === undefined) {
		var newArgs = [];
		for(var i = 0; i < arguments.length; i++) {
			newArgs.push(arguments[i]);
		}
		console.log("unrecognized", filename);
		return oldJs(...newArgs);
	}
	
	var entry = cache.modules[filename];
	
	var script = new vm.Script(entry.raw, {
		cachedData : blob.slice(entry.breaks[0], entry.breaks[1]) // Buffer.from(entry.cachedData, "hex")
	});
	
	var compiled = script.runInThisContext();
	
	var myRequire = internalModule.makeRequireFunction(myModule);
	var dirname = path.dirname(filename);
	
	var result = compiled.call(myModule.exports, myModule.exports, myRequire, myModule, filename, dirname);
	
	return result;
}

var caches = [
	"/sv/node_modules/npm/mongolayer/1/node_modules/mongolayer/",
	"/sv/node_modules/npm/moment/1/node_modules/moment/",
	"/sv/node_modules/npm/moment-timezone/1/node_modules/moment-timezone/",
	"/sv/node_modules/npm/express/1/node_modules/express/",
	"/sv/node_modules/sv/cloudinaryLib/1/",
	"/sv/node_modules/sv/siteLib/1/",
	"/sv/node_modules/plugins/nav/index.js",
	"/sv/node_modules/plugins/events/index.js",
	"/sv/node_modules/plugins/listings/index.js"
];

console.time("all");
caches.forEach(function(val, i) {
	console.time(val);
	require(val);
	console.timeEnd(val);
});
console.timeEnd("all");

maybe_worse_write.js

var vm = require("vm");
var fs = require("fs");
var Module = require("module");
var internalModule = require("./internalModule.js");

var caches = [
	"/sv/node_modules/npm/mongolayer/1/node_modules/mongolayer/",
	"/sv/node_modules/npm/moment/1/node_modules/moment/",
	"/sv/node_modules/npm/moment-timezone/1/node_modules/moment-timezone/",
	"/sv/node_modules/npm/express/1/node_modules/express/",
	"/sv/node_modules/sv/cloudinaryLib/1/",
	"/sv/node_modules/sv/siteLib/1/",
	"/sv/node_modules/plugins/nav/index.js",
	"/sv/node_modules/plugins/events/index.js",
	"/sv/node_modules/plugins/listings/index.js"
];

var cacheData = {
	modules : {},
	breaks : {}
};

caches.forEach(function(val, i) {
	console.time(val);
	require(val);
	console.timeEnd(val);
});

cacheData.pathCache = Module._pathCache;

var buffers = [];

for(var i in Module._cache) {
	if (i.match(/\.(json|node)$/)) { continue; }
	
	var fileStr = internalModule.stripShebang(fs.readFileSync(i).toString());
	var codeStr = `
		(function(exports, require, module, __filename, __dirname) {
			${fileStr}
		})
	`;
	
	var script = new vm.Script(codeStr, {
		filename : i,
		// lineOffset : 0,
		// displayErrors : true,
		produceCachedData : true
	});
	
	cacheData.modules[i] = {
		// cachedData : script.cachedData.toString("hex"),
		buffer : script.cachedData,
		raw : codeStr
	};
}

var start = 0;
for(var i in cacheData.modules) {
	var newEnd = start + cacheData.modules[i].buffer.length;
	cacheData.modules[i].breaks = [start, newEnd];
	buffers.push(cacheData.modules[i].buffer);
	delete cacheData.modules[i].buffer;
	start = newEnd;
}

var buf = Buffer.concat(buffers);
fs.writeFileSync("/tmp/cache.dat", buf);

var temp = JSON.stringify(cacheData);

fs.writeFileSync("/tmp/cache.json", JSON.stringify(cacheData));