Navigation Menu

Skip to content

Instantly share code, notes, and snippets.

@bryangingechen
Last active January 12, 2022 12:19
Show Gist options
  • Star 4 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save bryangingechen/9d86f1e5ec01674a32dd9c54bdc13947 to your computer and use it in GitHub Desktop.
Save bryangingechen/9d86f1e5ec01674a32dd9c54bdc13947 to your computer and use it in GitHub Desktop.
(WARNING, do not use old versions, they have a security issue!! I recommend changing your github password if you used any version from before Feb. 24, 2019) Observable API backup + restore (see https://gist.github.com/mootari/511b751e325db8316bb3138dcb0a7393 and https://talk.observablehq.com/t/backup-method/1628/10)
// https://gist.github.com/mootari/511b751e325db8316bb3138dcb0a7393
const request = require('superagent');
const cache = require('flat-cache').load('cacheId');
// https://stackoverflow.com/a/33500118/
const readline = require('readline');
const Writable = require('stream').Writable;
const fs = require('fs');
const crypto = require('crypto');
class ObservableAPI {
constructor() {
this.SITE_URL = 'https://observablehq.com';
this.API_URL = 'https://api.observablehq.com';
this.GITHUB_CLIENT_ID = '1a8619df27715d9d2c97';
// Cookie jar access info.
this.accessInfo = {
domain: 'observablehq.com',
path: '/',
secure: true,
};
this.agent = request.agent().withCredentials();
if (cache.getKey('S')) {
console.log(cache.getKey('S'));
this.agent.jar.setCookie(cache.getKey('S'), this.accessInfo.domain, this.accessInfo.path);
console.log('set cookie S from cache');
}
}
async get(route, data = null) { return this.request('get', route, data); }
async post(route, data = null) { return this.request('post', route, data); }
async put(route, data = null) { return this.request('put', route, data); }
async patch(route, data = null) { return this.request('patch', route, data); }
async delete(route) { return this.request('delete', route); }
async request(method, route, data = null) {
const m = method.toLowerCase();
const path = this.API_URL + '/' + route.replace(/^\//, '');
this.ensureToken();
let r;
switch(m) {
case 'get':
r = this.agent.get(path);
if(data !== null) r = r.query(data);
break;
case 'post':
case 'put':
case 'patch':
r = this.agent[m](path);
// append token
Object.assign(data, {"token":this.getToken().value});
if(data !== null) r = r.send(data);
break;
case 'delete':
r = this.agent[m](path);
break;
default:
throw Error(`Invalid request method "${method}"`);
}
r = r.set({
Accept: 'application/json',
Origin: this.SITE_URL,
});
try {
return (await r).body;
}
catch(e) {
// todo: throw?
console.log(e);
return false;
}
}
async isAuthorized() {
return !!(await this.request('get', '/user'));
}
getToken() {
const c = this.agent.jar.getCookie('T', this.accessInfo);
return c && c.value !== '' ? c : null;
}
getSession() {
return this.agent.jar.getCookie('S', this.accessInfo) || null;
}
ensureToken(regenerate = false) {
if (!regenerate && this.getToken()) return;
const {domain, path, secure} = this.accessInfo;
const n = 16;
const token = Array.from(crypto.randomFillSync(new Uint8Array(n)), e => e.toString(16).padStart(2, '0')).join('');
const expires = new Date(Date.now() + 1728e5).toUTCString();
const cookie = `T=${token}; Domain=${domain}; Path=${path}; Expires=${expires}; ${secure ? 'Secure' : ''}`;
console.log(cookie);
this.agent.jar.setCookie(cookie, domain, path);
}
async authorizeWithGithub(refresh = false) {
this.ensureToken();
if(!refresh && await this.isAuthorized()) return true;
const res = await this.agent.get('https://github.com/login/oauth/authorize').query({
client_id: this.GITHUB_CLIENT_ID,
state: this.getToken().value,
redirect_uri: `${this.API_URL}/github/oauth?path=/`
});
const token = res.text.match(/ name="authenticity_token" value="([^"]+)"/)[1];
// https://stackoverflow.com/a/33500118/
const mutableStdout = new Writable({
write: function(chunk, encoding, callback) {
if (!this.muted)
process.stdout.write(chunk, encoding);
callback();
}
});
mutableStdout.muted = false;
const rl = readline.createInterface({
input: process.stdin,
output: mutableStdout,
terminal: true
});
const login = await new Promise((resolve) => rl.question("Please enter github username:", name => resolve(name)));
const password = await new Promise((resolve) => {
rl.question("Please enter github password:", pass => resolve(pass));
mutableStdout.muted = true;
});
const res2 = await this.agent.post('https://github.com/session')
.set('Content-Type', 'application/x-www-form-urlencoded')
.send({
authenticity_token: token,
login,
password,
});
if (res2.request.url.match('two-factor')) {
mutableStdout.muted = false;
const token2 = res2.text.match(/ name="authenticity_token" value="([^"]+)"/)[1];
const res3 = this.agent.post('https://github.com/sessions/two-factor');
const otp = await new Promise((resolve) => rl.question("Please enter 2FA:", otp => resolve(otp)));
await res3
.set('Content-Type', 'application/x-www-form-urlencoded')
.send({
authenticity_token: token2,
otp,
});
}
rl.close();
if (this.getSession()) {
cache.setKey('S', this.agent.jar.getCookie('S', this.accessInfo).toString());
cache.save();
console.log('saved cookies to cache');
return true;
}
console.log('failed to authenticate!')
return false;
}
}
(async () => {
const refresh = false; // set to true to force relogin
const api = new ObservableAPI();
const ghWorked = await api.authorizeWithGithub(refresh);
if(ghWorked) {
switch (process.argv[2]) {
case 'download': {
// download notebook-name [outputFile]
if (process.argv[3]) {
const nbdat = await api.get(`/document/${process.argv[3]}`);
if (process.argv[4]) {
console.log(`writing "nodes" array to ${process.argv[4]} as JSON`);
fs.writeFileSync(process.argv[4], JSON.stringify(nbdat.nodes), {flag:'wx'});
} else console.log(nbdat.nodes);
break;
}
}
case 'downloadFull': {
// downloadFull notebook-name [outputFile]
if (process.argv[3]) {
const nbdat = await api.get(`/document/${process.argv[3]}`);
if (process.argv[4]) {
console.log(`writing notebook object to ${process.argv[4]} as JSON`);
fs.writeFileSync(process.argv[4], JSON.stringify(nbdat), {flag:'wx'});
} else console.log(nbdat);
break;
}
}
case 'upload': {
// upload title version inputFile
const [title, versionString, fileName] = process.argv.slice(3);
const version = parseInt(versionString);
if (title && !isNaN(version) && fileName) {
const nodes = JSON.parse(fs.readFileSync(fileName, 'utf8'));
if (!Array.isArray(nodes))
return console.log(`${fileName} could not be parsed into an Array`);
if (nodes.some(d => Object.keys(d).some(e => !['id', 'value', 'pinned'].includes(e))))
return console.log(`the objects in ${fileName} may only have 'id', 'version' and 'pinned' properties`);
if (nodes.length >= version)
return console.log(`the length of the Array in ${fileName} is ${nodes.length}, but it must be less than the version ${version}`);
if (new Set(nodes.map(({id}) => id)).size !== nodes.length)
return console.log(`the 'id' values in ${fileName} must be present and distinct`);
if (nodes.some(({id}) => parseInt(id) !== id || id < 0 || id > version))
return console.log(`the 'id' values in ${fileName} must be nonnegative integers less than or equal to the version ${version}`);
if (nodes.some(d => d.hasOwnProperty('value') && typeof d.value !== 'string') ||
nodes.some(d => d.hasOwnProperty('pinned') && typeof d.pinned !== 'boolean'))
return console.log(`the 'value' and 'pinnned' values in ${fileName}, if present, must be Strings and Booleans, respectively`);
console.log(`creating new notebook with nodes array from ${fileName}`);
console.log('response:\n', await api.post('/document/new', {title, version, nodes}));
break;
}
}
default: {
console.log(`run with:
To save the "nodes" array as JSON:
node index.js download notebook-name [outputFile]
-- OR --
To save the full notebook object as JSON:
node index.js downloadFull notebook-name [outputFile]
-- OR --
To upload a "nodes" array as a new notebook:
node index.js upload title version inputFile`);
}
}
}
})();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment