Instantly share code, notes, and snippets.

Embed
What would you like to do?
Code for pulling a document from Quip and Posting it to Hexo
let path = require('path');
let fs = require('fs');
let readline = require('readline');
let request = require('request');
let TurndownService = require('turndown');
let TurndownPluginGfm = require('turndown-plugin-gfm')
let _ = require('lodash');
let async = require('async');
let chalk = require('chalk');
let sanitizeHtml = require('sanitize-html');
let HtmlEntities = require('html-entities').AllHtmlEntities;
let shell = require('shelljs');
let slugify = require('slugify');
let moment = require('moment');
let folderId = '<insert folder ID here>';
let authorization = 'Bearer <insert personal api token here>';
let baseDir = path.dirname(__dirname);
let postsDir = baseDir + '/source/_posts';
class Importer
{
constructor() {
this.threadIds = null;
this.threads = [];
this.answer = null;
this.markdown = null;
this.title = null;
this.fullSlug = null;
this.markdownFile = null;
this.imagesDir = null;
this.turndown = new TurndownService();
this.turndown.use(TurndownPluginGfm.gfm);
}
run()
{
return this.stepGetFolderIds().then(() => {
return this.stepGetThreads();
}).then(() => {
return this.stepPromptSelection();
}).then(() => {
return this.stepConvertMarkdown();
}).then(() => {
return this.stepCreatePage();
}).then(() => {
return this.stepDownloadImages();
}).then(() => {
return this.stepUpdateMarkdownFile();
}).then(() => {
return this.stepOpenDirAndFile();
}).then(() => {
console.log(chalk.blue("Done!"));
});
}
stepGetFolderIds() {
return new Promise((resolve, reject) => {
let options = {
url: 'https://platform.quip.com/1/folders/' + folderId,
headers: {
'Authorization': authorization
}
};
request(options, (error, response, body) => {
if(error) {
return reject(error);
}
let data = JSON.parse(body);
let threadIds = [];
_.forEach(data.children, (child) => {
if(child && child.thread_id) {
threadIds.push(child.thread_id);
}
});
this.threadIds = threadIds;
resolve();
});
});
}
stepGetThreads() {
return new Promise((resolve, reject) => {
let jobs = [];
_.forEach(this.threadIds, (threadId) => {
jobs.push((cb) => {
this.getThreadData(threadId).then((threadData) => {
this.threads.push(threadData);
cb();
})
});
});
async.parallelLimit(jobs, 1, () => {
resolve();
});
});
}
stepPromptSelection() {
return new Promise((resolve, reject) => {
console.log(chalk.keyword('orange')('Which post do you wish to publish?'));
for(let i = 0; i < this.threads.length; i++) {
let thread = this.threads[i];
let line = chalk.bold.blue(i + 1) +
' ) ' +
chalk.blue(thread.thread.title);
console.log(line);
}
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
rl.question('Select post: ', (answer) => {
// TODO: Log the answer in a database
this.answer = parseInt(answer) - 1;
rl.close();
resolve();
});
});
}
stepConvertMarkdown() {
return new Promise((resolve, reject) => {
let thread = this.threads[this.answer];
let markdown = this.processHtml(thread.html);
this.markdown = markdown;
this.title = thread.thread.title;
resolve();
});
}
stepCreatePage() {
return new Promise((resolve, reject) => {
console.log(chalk.blue("Creating page..."));
this.fullSlug = moment().format('YYYY-MM-DD') + '-' +
slugify(this.title);
this.markdownFile = postsDir + '/' + this.fullSlug + '.md';
this.imagesDir = postsDir + '/' + this.fullSlug;
shell.exec('hexo new post "' + this.title + '"');
if(!shell.test('-f', this.markdownFile)) {
return reject("Whoa, the post file isn't where I expected it: " + this.markdownFile)
}
if(!shell.test('-d', this.imagesDir)) {
return reject("Whoa, the images directory isn't where I expected it: " + this.imagesDir)
}
resolve();
});
}
stepDownloadImages() {
return new Promise((resolve, reject) => {
// We need to scan for the blob match
let blobMatches = this.markdown.match(/\/blob\/[a-zA-Z0-9]+\/[a-zA-Z0-9]+/g);
if(blobMatches.length <= 0) {
console.log(chalk.yellowBright("No images found to download"));
return resolve();
}
let jobs = [];
let count = 0;
_.forEach(blobMatches, (blobStr) => {
count++;
jobs.push((cb) => {
this.downloadBlob(count, blobStr).then(() => {
cb();
});
});
});
async.parallelLimit(jobs, 1, () => {
resolve();
});
});
}
stepUpdateMarkdownFile() {
return new Promise((resolve, reject) => {
fs.appendFile(this.markdownFile, "\n" + this.markdown, function (err) {
if(err) {
return reject(err);
}
console.log(chalk.blue("Markdown File Updated"));
resolve();
});
});
}
stepOpenDirAndFile() {
return new Promise((resolve, reject) => {
shell.exec("open " + this.imagesDir);
shell.exec("code " + this.markdownFile);
resolve();
});
}
getThreadData(threadId) {
return new Promise((resolve, reject) => {
let options = {
url: 'https://platform.quip.com/1/threads/' + threadId,
headers: {
'Authorization': authorization
}
};
request(options, (error, response, body) => {
if(error) {
return reject(error);
}
let data = JSON.parse(body);
resolve(data);
});
});
}
processHtml(dirtyHtml) {
// Handle <br/> manually since Quip is weird and puts these in the <pre>
dirtyHtml = dirtyHtml.replace(/\<br\/\>/g, "\n");
// First remove IDs since they are safer to remove
dirtyHtml = dirtyHtml.replace(/\ id\=\'[a-zA-Z0-9]+?\'/g, '');
// Remove this string for the <pre>statements</pre>
dirtyHtml = dirtyHtml.split(" class='prettyprint'").join("");
let preMatches = dirtyHtml.match(/\<pre\>.*?<\/pre\>/gms);
preMatches = preMatches || [];
for(let i = 0; i < preMatches.length; i++) {
let str = preMatches[i];
let placeholder = '{{{ pre ' + i + ' }}}';
dirtyHtml = dirtyHtml.replace(str, placeholder);
}
let cleanHtml = sanitizeHtml(dirtyHtml, {
allowedTags: ['h2','h3','h4','h5','h6', 'p', 'ul', 'ol', 'li', 'a', 'img', 'pre'],
allowedAttributes: {
'a': [ 'href' ],
'img': [ 'src' ]
}
});
let markdown = this.turndown.turndown(cleanHtml);
// Put back preMatches
let entities = new HtmlEntities();
for(let i = 0; i < preMatches.length; i++) {
let str = preMatches[i];
let placeholder = '{{{ pre ' + i + ' }}}';
// Lets replace <pre> and </pre> with ```
str = str.replace('<pre>', '```');
str = str.replace('</pre>', '```');
str = entities.decode(str);
markdown = markdown.replace(placeholder, str);
}
return markdown;
}
downloadBlob(count, blobStr) {
return new Promise((resolve, reject) => {
let options = {
// Note: since the blobStr matches the API request perfect, we'll just use it
url: 'https://platform.quip.com/1' + blobStr,
headers: {
'Authorization': authorization
},
encoding: null
};
console.log(chalk.blue("Downloading Blog: " + blobStr + "..."));
request(options, (error, response, body) => {
if(error) {
return reject(error);
}
let contentType = response.headers['content-type'];
let extension = null;
if(contentType == 'image/jpeg') {
extension = 'jpg';
} else if(contentType == 'image/gif') {
extension = 'gif';
} else if(contentType == 'image/png') {
extension = 'png';
} else {
console.log(chalk.bold.redBright("ERROR: Unknown contentType of " + contentType));
return reject();
}
let fileName = './image' + count + '.' + extension;
let fullFileName = this.imagesDir + '/' + fileName;
console.log(chalk.blue("Saving blob " + blobStr + " as " + fileName));
fs.writeFile(fullFileName, body, (err) => {
if(err) {
return reject(err);
}
this.markdown = this.markdown.replace(blobStr, fileName);
resolve();
});
});
});
}
}
let importer = new Importer();
importer.run();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment