Skip to content

Instantly share code, notes, and snippets.

@laverdet

laverdet/README Secret

Last active Jun 5, 2021
Embed
What would you like to do?
Usage:
./sync --local
This will update the code directly through a browser hook. This works much faster
while editing code in simulation mode, and can even be used while offline.
cat sync.crx.b64 | base64 -D > sync.crx
This is an extension you can add to Chrome to avoid having to paste the bootstrap
line in each time.
./sync --auth 'username@example.com:password'
This will sync code via the Screeps Grunt API
Q3IyNAIAAAAmAQAAAAEAADCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANyvUR9ka45nrBXU55cztO7LKQCTfZxcEFDiKxGkqD2NAHa+e8tA8qH6X3C72qjXa6mKKXXXTGIiu9O3JLYpWMWR1mbsy7LLMX+UytzymAbQlNVWdMESl3lMhqy1DFSvrEeGw/DLEemYGUtGeeEcKBrSj+12IulIuXOaEkshFcKkwnZi5LgvD1EhGzulXJqbanYDfXO4hlOvjt0xg2ZUHyf4jhG6LwmcA3r7VwUqyQR/hof+NqK1znwlXJmu7xl2UCwhk4eVK4/FvvB14WQDKO5sFYMO64fzAhslNhfp0SjH4uiy6bZPyw/LYKmU/74iUMFCeaHBn/DxEiOKSEEe47kCAwEAARyfVE54Vj90vhL2jg/2K05RHKkAnv7ZTvYRdXjn31bHhY9OzrZW9yxYbjiM+v8r9Xb16JpLkvgdsR6x6KvfHMS+D7YqbU6QYxysBmNIMqaCD7lZns94hwqMmnCSuOVdoVbymwnG+awnhghE6k+2Ac0SmIwBmDagBc9KDjV+azJ2Ukth8Dit0BQxEUB3mqFczmJgh067/3Xz/Ugky+Vdv11OIN8YojP+YW0Gp5oJNSADKYd9nXbkrmPWGrLP1Liry4IL5TGgFgVXm9b0ZjQ0eOiV5J3PZq3iJujFcQg9PwUGN26vNY2Q17g46SlXoTM8u4aqfCYZZr32pNnIZZmw3CFQSwMEFAAACAgAbI8oSY30KTFiAgAA7QIAAAcAAABmb28uemlwC/BmZhFhYGDgYLjUo+G5X+njISZGBoY2IOZkkGHIzMtKTS7RyyoODeFkYO5muRTOwXkpvLSCm4GR5StQDVir74TzWYcNeI6dPxncEfPIqXBSrFt0kZT1mw3B35nmLTbT68zVSa3XqPxumRfcZuhxzX/+15UzOBotJM4eVTy/muveG1bT5o1vHqxgXqT+Ys48ty08H9/28Fj53FqhaKbl9iPo4IrzjJrXawuUvf4kbkmZM7VlW7W7Uqlb4dztBbPWPIsQYDxp7b2jMXue4Kd2yYu+OmEM+6YmTlw107IxQn0CY+j3ap//23K5W8sWy0/YuGNjZE/jgtoencWtHPv5dqQsUpHteV5UHJEwSaQk+tuZed9Cr9rYnraal7Z/dXlp6iW1Uy52T6490NOY+v3iY+/8Nr+AoLmM7TuumJy1T+9eqGvytffk5p/zznz8xxgAD0SDfg1Pzjo+7pVAnjYwgHiBgZibmJeZlloMCsb8PHBA/uHAGpC5fae5DhkIuH7/2zUhdeKSXW/yd0wJcpK5LWLMOVPD4+X+2e6BAubMM4vfzV3DmywzlfWecd01xgPC1vsDPjur+XOkyKRViMx98eqalon5Aou6jysbtCfwHTtuG/Sbx8/UzJo7ztfA7pDtY0s3/71TQs5zvc0RtWXJ1jAP/qRX77sy5fTNgJq3BkzHNsrpz9ySlqO7ucbilWTdqVv+joXTZKVjHNwnzsvRyPEuztj2jDfAm5FJjhlXspFgAAEgk+FtI4iFlIhYIYkIzd/IxmEGILJxrkAKLThZIcGJYSQrG0gbExAuAtKmTCAeAFBLAwQUAAAICADRjChJvyLxwgIBAACGAQAACQAAAGluamVjdC5qc02Qz2rDMAzGz8lTiFziQnGSXUZbcho77LBT9wKeozYuiW0sZX8oefc5blOGMUjWT5/1qZgIgTgYzcUhz6sK3uwFNYOx7OCoA6In6JyeRrQM8e2MDDpM2qghNipG+FLBqM8BKdd9cCNK/GG0ZJyVhLZ7RyJ1RnGdt3CarOZYEAHJO0u4gWueEfKHGdFNLFYAvpVhkaqZOYFYJ5ABVfd7TP+2bQuFdqMfkLG4sVmMgaB9jCyjhQi/DrhkoiQdjOdyc1hgkhR0W/bMnvZV1Tw9yzqeZr+rd3Vl0ibKRD7k1uAuKJX30eNLb4ZOUFKdAYe41DTNP2eLoS009Y3Js/mezPH+AVBLAwQUAAAICAAwjyhJCX4OC6kAAAArAQAADQAAAG1hbmlmZXN0Lmpzb25tjssKwjAQRff9ipBlkaS67G+4lFJCHNsUMwmZKEjpv5tHURA3A5lz7p2sDWMclQXeM37WAcATO79Q80MmTwhkHGZ4FJ3o6tYqNDegOH7xqYArkA7Gxz1S+wxONTY7C15NMD7CPeM5Rk+9lFTPCu1sFT0Eayg3U/Iuf02pZMvZUHztMALGsR4vmbRmbC2zfDjqGX7K2k9BcZYdG1xAR5GebChsS3NotuYNUEsBAgAAFAAACAgAbI8oSY30KTFiAgAA7QIAAAcAAAAAAAAAAAAAAAAAAAAAAGZvby56aXBQSwECAAAUAAAICADRjChJvyLxwgIBAACGAQAACQAAAAAAAAABAAAAAACHAgAAaW5qZWN0LmpzUEsBAgAAFAAACAgAMI8oSQl+DgupAAAAKwEAAA0AAAAAAAAAAQAAAAAAsAMAAG1hbmlmZXN0Lmpzb25QSwUGAAAAAAMAAwCnAAAAhAQAAAAA
#!/usr/bin/env node
"use strict";
let https = require('https');
let path = require('path');
let fs = require('fs');
let URL = require('url');
let child_process = require('child_process');
// Read args
let argv = {};
for (let ii = 2; ii < process.argv.length; ++ii) {
if (process.argv[ii].substr(0, 2) === '--') {
if (process.argv[ii + 1] === undefined || process.argv[ii + 1].substr(0, 2) === '--') {
argv[process.argv[ii].substr(2)] = true;
} else {
argv[process.argv[ii].substr(2)] = process.argv[ii + 1];
}
}
}
// Usage
if (argv.local !== true && !argv.auth) {
console.log('usage: '+ path.basename(process.argv[0])+ ' '+ path.basename(process.argv[1])+ ' --local | --auth "email:password"');
process.exit();
}
// Watch git
function parseBranch(HEAD) {
return HEAD === 'ref: refs/heads/master\n' ? 'master' : 'dev';
}
let branch = parseBranch(fs.readFileSync('.git/HEAD', 'utf8'));
!argv.local && fs.watch('.git', function(ev, file) {
if (file === 'HEAD') {
let tmp = parseBranch(fs.readFileSync('.git/HEAD', 'utf8'));
if (tmp !== branch) {
branch = tmp;
console.log('Switching to: '+ branch);
}
}
});
// Read local code from disk
let modules = {};
function refreshLocalBranch() {
modules = {};
fs.readdirSync('.').forEach(function(file) {
if (file !== 'sync.js' && /\.js$/.test(file)) {
modules[file.replace( /\.js$/, '')] = fs.readFileSync(file, 'utf8');
}
});
modules['last-push'] = 'module.exports='+ Date.now()+ ';';
}
// Watch for local changes
let pushTimeout;
fs.watch('.', function(ev, file) {
if (file !== 'sync.js' && /\.js$/.test(file)) {
try {
modules[file.replace(/\.js$/, '')] = fs.readFileSync(file, 'utf8');
} catch (err) {
delete modules[file.replace(/\.js$/, '')];
}
modules['last-push'] = 'module.exports='+ Date.now()+ ';';
schedulePush();
}
});
// Push changes to screeps.com
let writeListener;
function schedulePush() {
if (pushTimeout) {
clearTimeout(pushTimeout);
}
pushTimeout = setTimeout(function() {
pushTimeout = undefined;
writeListener && writeListener();
}, 50);
}
if (argv.local) {
// Auto-generation of self-signed certificate
let sslKey = new Promise(function(resolve, reject) {
function generate() {
child_process.execFile('openssl', [ 'genrsa', '-des3', '-out', 'sync.key', '-passout', 'pass:password', 2048 ], function(err, stdout, stderr) {
if (err) {
fs.unlink('sync.key', () => 0);
return reject(err);
}
child_process.execFile('openssl', [ 'req', '-new', '-batch', '-subj', '/commonName=127.0.0.1', '-key', 'sync.key', '-out', 'sync.csr', '-passin', 'pass:password' ], function(err, stdout, stderr) {
if (err) {
fs.unlink('sync.key', () => 0);
fs.unlink('sync.csr', () => 0);
return reject(err);
}
child_process.execFile('openssl', [ 'x509', '-req', '-days', 3650, '-in', 'sync.csr', '-signkey', 'sync.key', '-out', 'sync.crt', '-passin', 'pass:password' ], function(err, stdout, stderr) {
fs.unlink('sync.csr', () => 0);
if (err) {
fs.unlink('sync.key', () => 0);
fs.unlink('sync.crt', () => 0);
return reject(err);
}
fs.readFile('sync.key', function(err, key) {
if (err) return reject(err);
fs.readFile('sync.crt', function(err, cert) {
if (err) return reject(err);
resolve({ key, cert });
});
});
});
});
});
}
fs.readFile('sync.key', function(err, key) {
if (err) return generate();
fs.readFile('sync.crt', function(err, cert) {
if (err) return generate();
resolve({ key, cert });
});
});
});
// This all runs in the browser
let clientSide = function() {
function wait() {
if (document.evaluate("//div[contains(@class, 'console-messages-list')]", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue) {
// Grab reference to the commit button
let buttons = Array.prototype.slice.call(document.body.getElementsByTagName('button')).filter(function(el) {
return el.getAttribute('ng:disabled') === '!Script.dirty';
});
let commitButton = buttons[0];
// Override lodash's cloneDeep which is called from inside the internal reset method
let modules;
_.cloneDeep = function(cloneDeep) {
return function(obj) {
if (obj && typeof obj.main === 'string' && modules) {
// Monkey patch!
return modules;
}
return cloneDeep.apply(this, arguments);
};
}(_.cloneDeep);
// Wait for changes to local filesystem
function update(now) {
let req = new XMLHttpRequest;
req.onreadystatechange = function() {
if (req.readyState === 4) {
if (req.status === 200) {
modules = JSON.parse(req.responseText);
commitButton.disabled = false;
commitButton.click();
}
setTimeout(update.bind(this, false), req.status === 200 ? 0 : 1000);
}
};
req.open('GET', '//127.0.0.1:9090/'+ (now ? 'get' : 'wait'), true);
req.send();
};
update(true);
// Look for console messages
let sconsole = document.body.getElementsByClassName('console-messages-list')[0];
let lastMessage;
setInterval(function() {
let nodes = sconsole.getElementsByClassName('console-message');
let messages = [];
let found = false;
for (let ii = nodes.length - 1; ii >= 0; --ii) {
let el = nodes[ii];
let ts = el.getElementsByClassName('timestamp')[0];
ts = ts && ts.firstChild.nodeValue;
let msg = el.getElementsByTagName('span')[0].childNodes;
let txt = '';
for (let jj = 0; jj < msg.length; ++jj) {
if (msg[jj].tagName === 'BR') {
txt += '\n';
} else if (msg[jj].tagName === 'ANONYMOUS') {
msg = msg[jj].childNodes;
jj = -1;
} else {
txt += msg[jj].nodeValue;
}
}
if (lastMessage && txt === lastMessage[1] && ts === lastMessage[0]) {
break;
}
messages.push([ts, txt]);
}
if (messages.length) {
let req = new XMLHttpRequest;
req.open('GET', '//127.0.0.1:9090/log?log='+ encodeURIComponent(JSON.stringify(messages.reverse())), true);
req.send();
lastMessage = messages[messages.length - 1];
}
}, 100);
} else {
setTimeout(wait, 100);
}
}
wait();
};
// Localhost HTTP server
sslKey.then(function(key) {
let server = https.createServer({
key: key.key,
cert: key.cert,
passphrase: 'password',
}, function(req, res) {
let path = URL.parse(req.url, true);
switch (path.pathname) {
case '/inject':
res.writeHead(200, { 'Content-Type': 'text/javascript' });
res.end('~'+ clientSide.toString()+ '()');
break;
case '/get':
case '/wait':
if (writeListener) {
writeListener();
}
writeListener = function() {
res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
res.end(JSON.stringify(modules));
writeListener = undefined;
};
if (req.url === '/get') {
writeListener();
}
break;
case '/log':
res.writeHead(200, { 'Access-Control-Allow-Origin': '*' });
res.end();
let messages = JSON.parse(path.query.log);
for (let ii = 0; ii < messages.length; ++ii) {
if (messages[ii][0]) {
let prefix = ' ';
for (let jj = messages[ii][0].length; jj > 0; --jj) {
prefix += ' ';
}
console.log(
messages[ii][0],
messages[ii][1].split(/\n/g).map(function(line, ii) {
return (ii ? prefix : '')+ line;
}).join('\n')
);
} else {
console.log(messages[ii][1]);
}
}
break;
default:
res.writeHead(400);
res.end();
break;
}
});
server.timeout = 0;
server.listen(9090);
console.log(
"If you haven't done this already, run this (for OS X):\n"+
"sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain sync.crt\n\n"+
"Paste this into JS debug console in Screeps (*not* the Screeps console):\n"+
"var s = document.createElement('script');s.src='https://127.0.0.1:9090/inject';document.body.appendChild(s);"
);
}).catch(function(err) {
process.nextTick(() => { throw err });
});
} else {
// Push new code via Screeps API
writeListener = function() {
let req = https.request({
hostname: 'screeps.com',
port: 443,
path: '/api/user/code',
method: 'POST',
auth: argv.auth,
headers: {
'Content-Type': 'application/json; charset=utf-8'
},
});
req.end(JSON.stringify({ branch: branch, modules: modules }));
req.on('response', function(res) {
console.log('HTTP Status '+ res.statusCode);
});
};
}
// Sync current code
refreshLocalBranch();
schedulePush();
@jesstelford
Copy link

jesstelford commented Nov 24, 2014

Great work! I love the way you solved this problem :)

On line 37:

setTimeout(update.bind(this, false), req.status === 200 ? 0 : 1000);

This would trigger a HTTP GET every 4ms if the localhost server is down / blocked by firewall, etc.

I recommend an exponential re-check system kind of like:

    // Wait for changes to local filesystem
    function update(now, failureTimeout) {
        var req = new XMLHttpRequest;
        req.onreadystatechange = function() {
            if (req.readyState === 4) {
                var updateBound = update.bind(this, false);
                if (req.status === 200) {
                    modules = JSON.parse(req.responseText);
                    commitButton.disabled = false;
                    commitButton.click();
                    timeout = 1000;
                } else {
                    if (failureTimeout === undefined) {
                        failureTimeout = 10;
                    }
                    timeout = failureTimeout;
                    updateBound.bind(this, failureTimeout * 2);
                }
                setTimeout(updateBound, timeout);
            }
        };
        req.open('GET', 'http://localhost:9090/'+ (now ? 'get' : 'wait'), true);
        req.send();
    };
    update(true);

It defaults to check again in 10ms after the first failure, doubling the delay on each subsequent failure. It does this with the second bind call: updateBound.bind(this, failureTimeout * 2);. As soon as there is a successful response, it resets the timeout to 1000ms again, then starts the cycle over once a response is received.

@SystemParadox
Copy link

SystemParadox commented Nov 24, 2014

This doesn't appear to be working- scripts in screeps just aren't being updated. Any hints on how to debug this?

I'm not really clear how this interfaces with the scripts in screeps. Is this relying on screeps calling lodash.cloneDeep at a particular point?

@laverdet
Copy link
Author

laverdet commented Nov 24, 2014

@jesstelford if the host is down how will it return a 200 status code?

@SystemParadox the secret sauce is the monkey patch to _.cloneDeep. Most of the Screeps state seems to be buried in closures that we can't access. But at some point they call _.cloneDeep to clone the state of the user code, so I override that to return our updated code instead of a clone of the old code.

@jesstelford
Copy link

jesstelford commented Nov 25, 2014

@laverdet Derp, my bad. Having said that, I still think the exponential re-check would be a better solution in the case where connection is lost.

@SystemParadox If you're interested in some cool history behind this type of monkey patch, @laverdet is essentially doing a JSON Hijacking attack on the browser: http://www.thespanner.co.uk/2011/05/30/json-hijacking/ But instead of hooking into Array's constructor, he hooks into the global underscore/lodash variable and method for copying arrays. Excellent bit of work!

@Gvdpoel
Copy link

Gvdpoel commented Nov 29, 2014

Works great on windows using node.js :) Thanks for your work.

@ceebeel
Copy link

ceebeel commented Dec 14, 2014

Nice script ! 👍 Thanks for sharing. 😄

@sheoak
Copy link

sheoak commented Dec 14, 2014

Thanks, this is great! Nice idea!

It didin't work for me (ubuntu 14.04), because of line 90:

modules[file.replace(/\.js$/, '')] = fs.readFileSync(file, 'utf8');

at that point the file doesn’t exist (due to something in my vim configuration I presume), so I added a timeout as a quick-and-dirty fix.

@avdg
Copy link

avdg commented Dec 15, 2014

Line 19 is spitting a lot of errors because obj is null

It should become

if (obj && typeof obj.main === 'string' && modules) {

Edit: This issue is fixed

@laverdet
Copy link
Author

laverdet commented Dec 15, 2014

Updated gist with fix for null clones (fixes survival) and also fixed console.log() redirection with longer messages

@avdg
Copy link

avdg commented Dec 15, 2014

Thanks!

@SystemParadox
Copy link

SystemParadox commented Jan 8, 2015

For anyone else who is confused, this script does not cause the scripts to change in the screeps UI.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment