README

Usage:

./sync --local
This will update the code directly through a browser hook. This works much faster
while editing code in simulation mode, and can even be used while offline.

cat sync.crx.b64 | base64 -D > sync.crx
This is an extension you can add to Chrome to avoid having to paste the bootstrap
line in each time.

./sync --auth 'username@example.com:password'
This will sync code via the Screeps Grunt API

sync.crx.b64

Q3IyNAIAAAAmAQAAAAEAADCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANyvUR9ka45nrBXU55cztO7LKQCTfZxcEFDiKxGkqD2NAHa+e8tA8qH6X3C72qjXa6mKKXXXTGIiu9O3JLYpWMWR1mbsy7LLMX+UytzymAbQlNVWdMESl3lMhqy1DFSvrEeGw/DLEemYGUtGeeEcKBrSj+12IulIuXOaEkshFcKkwnZi5LgvD1EhGzulXJqbanYDfXO4hlOvjt0xg2ZUHyf4jhG6LwmcA3r7VwUqyQR/hof+NqK1znwlXJmu7xl2UCwhk4eVK4/FvvB14WQDKO5sFYMO64fzAhslNhfp0SjH4uiy6bZPyw/LYKmU/74iUMFCeaHBn/DxEiOKSEEe47kCAwEAARyfVE54Vj90vhL2jg/2K05RHKkAnv7ZTvYRdXjn31bHhY9OzrZW9yxYbjiM+v8r9Xb16JpLkvgdsR6x6KvfHMS+D7YqbU6QYxysBmNIMqaCD7lZns94hwqMmnCSuOVdoVbymwnG+awnhghE6k+2Ac0SmIwBmDagBc9KDjV+azJ2Ukth8Dit0BQxEUB3mqFczmJgh067/3Xz/Ugky+Vdv11OIN8YojP+YW0Gp5oJNSADKYd9nXbkrmPWGrLP1Liry4IL5TGgFgVXm9b0ZjQ0eOiV5J3PZq3iJujFcQg9PwUGN26vNY2Q17g46SlXoTM8u4aqfCYZZr32pNnIZZmw3CFQSwMEFAAACAgAbI8oSY30KTFiAgAA7QIAAAcAAABmb28uemlwC/BmZhFhYGDgYLjUo+G5X+njISZGBoY2IOZkkGHIzMtKTS7RyyoODeFkYO5muRTOwXkpvLSCm4GR5StQDVir74TzWYcNeI6dPxncEfPIqXBSrFt0kZT1mw3B35nmLTbT68zVSa3XqPxumRfcZuhxzX/+15UzOBotJM4eVTy/muveG1bT5o1vHqxgXqT+Ys48ty08H9/28Fj53FqhaKbl9iPo4IrzjJrXawuUvf4kbkmZM7VlW7W7Uqlb4dztBbPWPIsQYDxp7b2jMXue4Kd2yYu+OmEM+6YmTlw107IxQn0CY+j3ap//23K5W8sWy0/YuGNjZE/jgtoencWtHPv5dqQsUpHteV5UHJEwSaQk+tuZed9Cr9rYnraal7Z/dXlp6iW1Uy52T6490NOY+v3iY+/8Nr+AoLmM7TuumJy1T+9eqGvytffk5p/zznz8xxgAD0SDfg1Pzjo+7pVAnjYwgHiBgZibmJeZlloMCsb8PHBA/uHAGpC5fae5DhkIuH7/2zUhdeKSXW/yd0wJcpK5LWLMOVPD4+X+2e6BAubMM4vfzV3DmywzlfWecd01xgPC1vsDPjur+XOkyKRViMx98eqalon5Aou6jysbtCfwHTtuG/Sbx8/UzJo7ztfA7pDtY0s3/71TQs5zvc0RtWXJ1jAP/qRX77sy5fTNgJq3BkzHNsrpz9ySlqO7ucbilWTdqVv+joXTZKVjHNwnzsvRyPEuztj2jDfAm5FJjhlXspFgAAEgk+FtI4iFlIhYIYkIzd/IxmEGILJxrkAKLThZIcGJYSQrG0gbExAuAtKmTCAeAFBLAwQUAAAICADRjChJvyLxwgIBAACGAQAACQAAAGluamVjdC5qc02Qz2rDMAzGz8lTiFziQnGSXUZbcho77LBT9wKeozYuiW0sZX8oefc5blOGMUjWT5/1qZgIgTgYzcUhz6sK3uwFNYOx7OCoA6In6JyeRrQM8e2MDDpM2qghNipG+FLBqM8BKdd9cCNK/GG0ZJyVhLZ7RyJ1RnGdt3CarOZYEAHJO0u4gWueEfKHGdFNLFYAvpVhkaqZOYFYJ5ABVfd7TP+2bQuFdqMfkLG4sVmMgaB9jCyjhQi/DrhkoiQdjOdyc1hgkhR0W/bMnvZV1Tw9yzqeZr+rd3Vl0ibKRD7k1uAuKJX30eNLb4ZOUFKdAYe41DTNP2eLoS009Y3Js/mezPH+AVBLAwQUAAAICAAwjyhJCX4OC6kAAAArAQAADQAAAG1hbmlmZXN0Lmpzb25tjssKwjAQRff9ipBlkaS67G+4lFJCHNsUMwmZKEjpv5tHURA3A5lz7p2sDWMclQXeM37WAcATO79Q80MmTwhkHGZ4FJ3o6tYqNDegOH7xqYArkA7Gxz1S+wxONTY7C15NMD7CPeM5Rk+9lFTPCu1sFT0Eayg3U/Iuf02pZMvZUHztMALGsR4vmbRmbC2zfDjqGX7K2k9BcZYdG1xAR5GebChsS3NotuYNUEsBAgAAFAAACAgAbI8oSY30KTFiAgAA7QIAAAcAAAAAAAAAAAAAAAAAAAAAAGZvby56aXBQSwECAAAUAAAICADRjChJvyLxwgIBAACGAQAACQAAAAAAAAABAAAAAACHAgAAaW5qZWN0LmpzUEsBAgAAFAAACAgAMI8oSQl+DgupAAAAKwEAAA0AAAAAAAAAAQAAAAAAsAMAAG1hbmlmZXN0Lmpzb25QSwUGAAAAAAMAAwCnAAAAhAQAAAAA

sync.js

#!/usr/bin/env node
"use strict";
let https = require('https');
let path = require('path');
let fs = require('fs');
let URL = require('url');
let child_process = require('child_process');

// Read args
let argv = {};
for (let ii = 2; ii < process.argv.length; ++ii) {
	if (process.argv[ii].substr(0, 2) === '--') {
		if (process.argv[ii + 1] === undefined || process.argv[ii + 1].substr(0, 2) === '--') {
			argv[process.argv[ii].substr(2)] = true;
		} else {
			argv[process.argv[ii].substr(2)] = process.argv[ii + 1];
		}
	}
}

// Usage
if (argv.local !== true && !argv.auth) {
	console.log('usage: '+ path.basename(process.argv[0])+ ' '+ path.basename(process.argv[1])+ ' --local | --auth "email:password"');
	process.exit();
}

// Watch git
function parseBranch(HEAD) {
	return HEAD === 'ref: refs/heads/master\n' ? 'master' : 'dev';
}
let branch = parseBranch(fs.readFileSync('.git/HEAD', 'utf8'));
!argv.local && fs.watch('.git', function(ev, file) {
	if (file === 'HEAD') {
		let tmp = parseBranch(fs.readFileSync('.git/HEAD', 'utf8'));
		if (tmp !== branch) {
			branch = tmp;
			console.log('Switching to: '+ branch);
		}
	}
});

// Read local code from disk
let modules = {};
function refreshLocalBranch() {
	modules = {};
	fs.readdirSync('.').forEach(function(file) {
		if (file !== 'sync.js' && /\.js$/.test(file)) {
			modules[file.replace( /\.js$/, '')] = fs.readFileSync(file, 'utf8');
		}
	});
	modules['last-push'] = 'module.exports='+ Date.now()+ ';';
}

// Watch for local changes
let pushTimeout;
fs.watch('.', function(ev, file) {
	if (file !== 'sync.js' && /\.js$/.test(file)) {
		try {
			modules[file.replace(/\.js$/, '')] = fs.readFileSync(file, 'utf8');
		} catch (err) {
			delete modules[file.replace(/\.js$/, '')];
		}
		modules['last-push'] = 'module.exports='+ Date.now()+ ';';
		schedulePush();
	}
});

// Push changes to screeps.com
let writeListener;
function schedulePush() {
	if (pushTimeout) {
		clearTimeout(pushTimeout);
	}
	pushTimeout = setTimeout(function() {
		pushTimeout = undefined;
		writeListener && writeListener();
	}, 50);
}

if (argv.local) {

	// Auto-generation of self-signed certificate
	let sslKey = new Promise(function(resolve, reject) {
		function generate() {
			child_process.execFile('openssl', [ 'genrsa', '-des3', '-out', 'sync.key', '-passout', 'pass:password', 2048 ], function(err, stdout, stderr) {
				if (err) {
					fs.unlink('sync.key', () => 0);
					return reject(err);
				}
				child_process.execFile('openssl', [ 'req', '-new', '-batch', '-subj', '/commonName=127.0.0.1', '-key', 'sync.key', '-out', 'sync.csr', '-passin', 'pass:password' ], function(err, stdout, stderr) {
					if (err) {
						fs.unlink('sync.key', () => 0);
						fs.unlink('sync.csr', () => 0);
						return reject(err);
					}
					child_process.execFile('openssl', [ 'x509', '-req', '-days', 3650, '-in', 'sync.csr', '-signkey', 'sync.key', '-out', 'sync.crt', '-passin', 'pass:password' ], function(err, stdout, stderr) {
						fs.unlink('sync.csr', () => 0);
						if (err) {
							fs.unlink('sync.key', () => 0);
							fs.unlink('sync.crt', () => 0);
							return reject(err);
						}
						fs.readFile('sync.key', function(err, key) {
							if (err) return reject(err);
							fs.readFile('sync.crt', function(err, cert) {
								if (err) return reject(err);
								resolve({ key, cert });
							});
						});
					});
				});
			});
		}
		fs.readFile('sync.key', function(err, key) {
			if (err) return generate();
			fs.readFile('sync.crt', function(err, cert) {
				if (err) return generate();
				resolve({ key, cert });
			});
		});
	});

	// This all runs in the browser
	let clientSide = function() {
		function wait() {
			if (document.evaluate("//div[contains(@class, 'console-messages-list')]", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue) {
				// Grab reference to the commit button
				let buttons = Array.prototype.slice.call(document.body.getElementsByTagName('button')).filter(function(el) {
					return el.getAttribute('ng:disabled') === '!Script.dirty';
				});
				let commitButton = buttons[0];

				// Override lodash's cloneDeep which is called from inside the internal reset method
				let modules;
				_.cloneDeep = function(cloneDeep) {
					return function(obj) {
						if (obj && typeof obj.main === 'string' && modules) {
							// Monkey patch!
							return modules;
						}
						return cloneDeep.apply(this, arguments);
					};
				}(_.cloneDeep);

				// Wait for changes to local filesystem
				function update(now) {
					let req = new XMLHttpRequest;
					req.onreadystatechange = function() {
						if (req.readyState === 4) {
							if (req.status === 200) {
								modules = JSON.parse(req.responseText);
								commitButton.disabled = false;
								commitButton.click();
							}
							setTimeout(update.bind(this, false), req.status === 200 ? 0 : 1000);
						}
					};
					req.open('GET', '//127.0.0.1:9090/'+ (now ? 'get' : 'wait'), true);
					req.send();
				};
				update(true);

				// Look for console messages
				let sconsole = document.body.getElementsByClassName('console-messages-list')[0];
				let lastMessage;
				setInterval(function() {
					let nodes = sconsole.getElementsByClassName('console-message');
					let messages = [];
					let found = false;
					for (let ii = nodes.length - 1; ii >= 0; --ii) {
						let el = nodes[ii];
						let ts = el.getElementsByClassName('timestamp')[0];
						ts = ts && ts.firstChild.nodeValue;
						let msg = el.getElementsByTagName('span')[0].childNodes;
						let txt = '';
						for (let jj = 0; jj < msg.length; ++jj) {
							if (msg[jj].tagName === 'BR') {
								txt += '\n';
							} else if (msg[jj].tagName === 'ANONYMOUS') {
								msg = msg[jj].childNodes;
								jj = -1;
							} else {
								txt += msg[jj].nodeValue;
							}
						}
						if (lastMessage && txt === lastMessage[1] && ts === lastMessage[0]) {
							break;
						}
						messages.push([ts, txt]);
					}
					if (messages.length) {
						let req = new XMLHttpRequest;
						req.open('GET', '//127.0.0.1:9090/log?log='+ encodeURIComponent(JSON.stringify(messages.reverse())), true);
						req.send();
						lastMessage = messages[messages.length - 1];
					}
				}, 100);
			} else {
				setTimeout(wait, 100);
			}
		}
		wait();
	};

	// Localhost HTTP server
	sslKey.then(function(key) {
		let server = https.createServer({
			key: key.key,
			cert: key.cert,
			passphrase: 'password',
		}, function(req, res) {
			let path = URL.parse(req.url, true);
			switch (path.pathname) {
				case '/inject':
					res.writeHead(200, { 'Content-Type': 'text/javascript' });
					res.end('~'+ clientSide.toString()+ '()');
					break;

				case '/get':
				case '/wait':
					if (writeListener) {
						writeListener();
					}
					writeListener = function() {
						res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
						res.end(JSON.stringify(modules));
						writeListener = undefined;
					};
					if (req.url === '/get') {
						writeListener();
					}
					break;

				case '/log':
					res.writeHead(200, { 'Access-Control-Allow-Origin': '*' });
					res.end();
					let messages = JSON.parse(path.query.log);
					for (let ii = 0; ii < messages.length; ++ii) {
						if (messages[ii][0]) {
							let prefix = ' ';
							for (let jj = messages[ii][0].length; jj > 0; --jj) {
								prefix += ' ';
							}
							console.log(
								messages[ii][0],
								messages[ii][1].split(/\n/g).map(function(line, ii) {
									return (ii ? prefix : '')+ line;
								}).join('\n')
							);
						} else {
							console.log(messages[ii][1]);
						}
					}
					break;

				default:
					res.writeHead(400);
					res.end();
					break;
			}
		});
		server.timeout = 0;
		server.listen(9090);
		console.log(
			"If you haven't done this already, run this (for OS X):\n"+
			"sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain sync.crt\n\n"+
			"Paste this into JS debug console in Screeps (*not* the Screeps console):\n"+
			"var s = document.createElement('script');s.src='https://127.0.0.1:9090/inject';document.body.appendChild(s);"
		);
	}).catch(function(err) {
		process.nextTick(() => { throw err });
	});
} else {

	// Push new code via Screeps API
	writeListener = function() {
		let req = https.request({
			hostname: 'screeps.com',
			port: 443,
			path: '/api/user/code',
			method: 'POST',
			auth: argv.auth,
			headers: {
				'Content-Type': 'application/json; charset=utf-8'
			},
		});
		req.end(JSON.stringify({ branch: branch, modules: modules }));
		req.on('response', function(res) {
			console.log('HTTP Status '+ res.statusCode);
		});
	};
}

// Sync current code
refreshLocalBranch();
schedulePush();

Great work! I love the way you solved this problem :)

On line 37:

setTimeout(update.bind(this, false), req.status === 200 ? 0 : 1000);

This would trigger a HTTP GET every 4ms if the localhost server is down / blocked by firewall, etc.

I recommend an exponential re-check system kind of like:

    // Wait for changes to local filesystem
    function update(now, failureTimeout) {
        var req = new XMLHttpRequest;
        req.onreadystatechange = function() {
            if (req.readyState === 4) {
                var updateBound = update.bind(this, false);
                if (req.status === 200) {
                    modules = JSON.parse(req.responseText);
                    commitButton.disabled = false;
                    commitButton.click();
                    timeout = 1000;
                } else {
                    if (failureTimeout === undefined) {
                        failureTimeout = 10;
                    }
                    timeout = failureTimeout;
                    updateBound.bind(this, failureTimeout * 2);
                }
                setTimeout(updateBound, timeout);
            }
        };
        req.open('GET', 'http://localhost:9090/'+ (now ? 'get' : 'wait'), true);
        req.send();
    };
    update(true);

It defaults to check again in 10ms after the first failure, doubling the delay on each subsequent failure. It does this with the second bind call: updateBound.bind(this, failureTimeout * 2);. As soon as there is a successful response, it resets the timeout to 1000ms again, then starts the cycle over once a response is received.

This doesn't appear to be working- scripts in screeps just aren't being updated. Any hints on how to debug this?

I'm not really clear how this interfaces with the scripts in screeps. Is this relying on screeps calling lodash.cloneDeep at a particular point?

@jesstelford if the host is down how will it return a 200 status code?

@SystemParadox the secret sauce is the monkey patch to _.cloneDeep. Most of the Screeps state seems to be buried in closures that we can't access. But at some point they call _.cloneDeep to clone the state of the user code, so I override that to return our updated code instead of a clone of the old code.

@laverdet Derp, my bad. Having said that, I still think the exponential re-check would be a better solution in the case where connection is lost.

@SystemParadox If you're interested in some cool history behind this type of monkey patch, @laverdet is essentially doing a JSON Hijacking attack on the browser: http://www.thespanner.co.uk/2011/05/30/json-hijacking/ But instead of hooking into Array's constructor, he hooks into the global underscore/lodash variable and method for copying arrays. Excellent bit of work!

Works great on windows using node.js :) Thanks for your work.

Nice script ! 👍 Thanks for sharing. 😄

Thanks, this is great! Nice idea!

It didin't work for me (ubuntu 14.04), because of line 90:

modules[file.replace(/\.js$/, '')] = fs.readFileSync(file, 'utf8');

at that point the file doesn’t exist (due to something in my vim configuration I presume), so I added a timeout as a quick-and-dirty fix.

Line 19 is spitting a lot of errors because obj is null

It should become

if (obj && typeof obj.main === 'string' && modules) {

Edit: This issue is fixed

Updated gist with fix for null clones (fixes survival) and also fixed console.log() redirection with longer messages

Thanks!

For anyone else who is confused, this script does not cause the scripts to change in the screeps UI.