Skip to content

Instantly share code, notes, and snippets.

@jankeromnes

jankeromnes/spectrum-to-discourse.js

Last active July 8, 2021 12:45
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jankeromnes/dcd9bd8cd1a3df2d32d8541adb7758f2 to your computer and use it in GitHub Desktop.
Save jankeromnes/dcd9bd8cd1a3df2d32d8541adb7758f2 to your computer and use it in GitHub Desktop.
Download ZIP
Quick-and-dirty Node.js hack to export Spectrum threads & import them to Discourse
Raw
spectrum-to-discourse.js
const fs = require('fs');
const https = require('https');
const querystring = require('querystring');
// All image URLs, in case you want to re-import them to Discourse (this script just links Discourse to the Spectrum images)
let images = [];
let comments = {};
// Map Spectrum usernames to existing Discourse usernames, in order to post comments as actual users (otherwise this script posts comments as user 'discobot', but mentions the Spectrum username in first line of comment. You can also fix it manually afterwards.)
const discourseUsers = {
'spectrum-username': 'discourse-username',
}
function fetchDiscourseAPI(path, data, method = 'POST', username = 'discobot') {
var postData = querystring.stringify(data);
var options = {
/* TODO: Replace all occurrences of 'community.theia-ide.org' by your own Discourse domain) */
hostname: 'community.theia-ide.org',
port: 443,
path: path,
method: method,
headers: {
'Api-Key': '[REDACTED: create one in https://community.theia-ide.org/admin/api/keys]',
'Api-Username': username,
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': postData.length
}
};
return new Promise((resolve, reject) => {
var req = https.request(options, (res) => {
let body = '';
res.on('data', chunk => {
body += chunk;
});
res.on('error', error => reject);
res.on('end', () => { resolve({res, body}); });
});
req.on('error', reject);
req.write(postData);
req.end();
});
}
async function postDiscourseThread(thread) {
const first = thread.shift();
const {res, body} = await fetchDiscourseAPI('/posts.json', {
"title": first.title,
"raw": first.raw,
"archetype": "regular",
"created_at": first.created_at
}, 'POST', first.username);
const parsed = JSON.parse(body);
if (res.statusCode >= 400) {
if (parsed.errors && parsed.errors[0] === "Title has already been used") {
console.log('Duplicate post, skipping!', res.statusCode, body, first.title);
return;
}
console.log('Rate limit!', res.statusCode, body, first.title);
await new Promise(resolve => setTimeout(resolve, 35000));
thread.unshift(first);
return postDiscourseThread(thread);
}
//console.log('parsed', parsed);
const topic_id = parsed.topic_id;
console.log('post Discourse Thread', res.statusCode, topic_id);
if (!topic_id) {
console.error('Duplicate topic! aborting');
return;
}
if (!threadUrls[first.id]) {
threadUrls[first.id] = {};
}
threadUrls[first.id].discourse = 'https://community.theia-ide.org/t/' + topic_id;
postDiscourseThreadMessages(topic_id, thread);
}
async function postDiscourseThreadMessages(topic_id, messages) {
for (const message of messages) {
const payload = {
"topic_id": topic_id,
"raw": message.raw,
"archetype": "regular",
"created_at": message.created_at,
};
//await new Promise(resolve => setTimeout(resolve, 10000));
let {res: re, body: bo} = await fetchDiscourseAPI('/posts.json', payload, 'POST', message.username);
while (re.statusCode >= 400) {
console.log('Comment rate limit!', re.statusCode, bo, message.raw);
await new Promise(resolve => setTimeout(resolve, 10000));
const a = await fetchDiscourseAPI('/posts.json', payload, 'POST', message.username);
re = a.res; bo = a.body;
}
console.log('post Discourse comment', re.statusCode, bo, payload, message.username);
}
}
const topicWhitelist = [];
async function verifySpectrumThreadIsOnDiscourse(thread) {
let {res, body} = await fetchDiscourseAPI(`/search/query.json?term=${encodeURIComponent(`"${thread[0].title.replace(/[ \.!?]+$/,'')}" in:title`)}`, {}, 'GET');
const results = JSON.parse(body);
if (res.statusCode === 429) {
console.log('Rate limit!', res.statusCode, body);
await new Promise(resolve => setTimeout(resolve, 1000 + 1000 * (results && results.extras && results.extras.wait_seconds || 35)));
return verifySpectrumThreadIsOnDiscourse(thread);
}
if (!results.topics) {
console.log(res.statusCode);
console.error(thread);
console.error(results);
throw new Error('No results!');
}
if (results.topics.length !== 1) {
console.error(thread);
console.error(results.topics);
throw new Error('Expected exactly 1 Discourse search result for Spectrum thread');
}
if (!topicWhitelist.includes(results.topics[0].id) && results.topics[0].posts_count !== thread[0].messageCount + 1) {
console.error(thread);
console.error(results.topics[0]);
throw new Error('Discourse/Spectrum comments count mismatch! https://community.theia-ide.org/t/' + results.topics[0].id);
}
console.log('Verified thread:', thread[0].title);
}
function fetchSpectrumAPI(data) {
var postData = JSON.stringify(data);
var options = {
hostname: 'spectrum.chat',
port: 443,
path: '/api',
method: 'POST',
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:73.0) Gecko/20100101 Firefox/73.0',
'Content-Type': 'application/json',
'Content-Length': postData.length,
'Origin': 'https://spectrum.chat',
/* TODO: Replace all occurrencces of 'https://spectrum.chat/theia' with your own Spectrum URL */
'Referer': 'https://spectrum.chat/theia?tab=posts',
/* TODO: Copy your own cookies below, e.g. sign in with your browser and copy cookies from your browser devtools */
'Cookie': '_now_no_cache=1; _ga=[REDACTED]; amplitude_id_undefinedspectrum.chat=[REDACTED]; session=[REDACTED]; session.sig=[REDACTED]; _gid=[REDACTED]; _gat=1'
}
};
return new Promise((resolve, reject) => {
var req = https.request(options, (res) => {
let body = '';
res.on('data', chunk => {
body += chunk;
});
res.on('error', error => reject);
res.on('end', () => { resolve({res, body}); });
});
req.on('error', reject);
req.write(postData);
req.end();
});
}
// Operational Transformation algorithm (insert)
function adjustRanges(ranges, index, insert) {
if (!ranges) {
return;
}
for (const range of ranges.reverse()) {
if (range.offset > index) {
range.offset += insert.length;
} else if (range.offset + range.length >= index) {
range.length += insert.length;
} else {
continue;
}
}
}
// Blocks is an internal Spectrum representation. Raw is a Discourse HTML post.
function blocksToRaw(blocks, entityMap) {
return blocks.map(b => {
if (b.text && b.text.length > 0) {
if (b.entityRanges.length > 0) {
for (const range of b.entityRanges.reverse()) {
let i = range.offset + range.length;
const entity = entityMap[range.key];
if (entity.type === 'IMAGE') {
images.push({src: entity.data.src, alt: entity.data.alt});
const insert = `<img src="${entity.data.src}" alt="${entity.data.alt || 'image'}"/>`;
b.text = b.text.slice(0, i) + insert + b.text.slice(i);
adjustRanges(b.inlineStyleRanges, i, insert);
continue;
}
if (entity.type === 'LINK' || entity.data.type === 'youtube') {
const insert2 = '</a>';
b.text = b.text.slice(0, i) + insert2 + b.text.slice(i);
adjustRanges(b.inlineStyleRanges, i, insert2);
i -= range.length;
const insert1 = `<a href="${entity.data.url}">`;
b.text = b.text.slice(0, i) + insert1 + b.text.slice(i);
adjustRanges(b.inlineStyleRanges, i, insert1);
continue;
}
if (entity.data.id) {
const urls = threadUrls[entity.data.id.split('?')[0]];
if (entity.data.entity === 'thread' && urls) {
const insert = urls.discourse || urls.spectrum;
b.text = b.text.slice(0, i) + insert + b.text.slice(i);
adjustRanges(b.inlineStyleRanges, i, insert);
continue;
}
}
console.error('unsupported entity!', entity);
}
}
// Looks like: "inlineStyleRanges":[{"offset":63,"length":37,"style":"CODE"},{"offset":112,"length":16,"style":"CODE"}]
for (const range of (b.inlineStyleRanges || []).reverse()) {
let i = range.offset + range.length;
switch (range.style) {
case 'CODE':
b.text = b.text.slice(0, i) + '</code>' + b.text.slice(i);
i -= range.length;
b.text = b.text.slice(0, i) + '<code>' + b.text.slice(i);
break;
case 'ITALIC':
b.text = b.text.slice(0, i) + '</em>' + b.text.slice(i);
i -= range.length;
b.text = b.text.slice(0, i) + '<em>' + b.text.slice(i);
break;
case 'BOLD':
b.text = b.text.slice(0, i) + '</strong>' + b.text.slice(i);
i -= range.length;
b.text = b.text.slice(0, i) + '<strong>' + b.text.slice(i);
break;
default:
console.error('unknown style range', range);
}
}
switch (b.type) {
case 'unstyled':
break;
case 'atomic':
b.text = b.text.replace(/^ /, '');;
break;
case 'blockquote':
b.text = '> ' + b.text;
break;
case 'ordered-list-item':
b.text = '1. ' + b.text;
break;
case 'unordered-list-item':
b.text = '- ' + b.text;
break;
case 'code-block':
b.text = b.text.replace(/^(<code>)?/, '```\n').replace(/(<\/code>)?$/, '\n```');
break;
case 'header-one':
b.text = '# ' + b.text;
break;
case 'header-two':
b.text = '## ' + b.text;
break;
case 'header-three':
b.text = '### ' + b.text;
break;
case 'header-four':
b.text = '#### ' + b.text;
break;
case 'header-five':
b.text = '##### ' + b.text;
break;
default:
console.error('unknown text block type:', b.type, b);
}
return b.text;
}
console.error('unsupported block!', b);
return '';
}).join('\n\n');
}
async function threadConnectionToThreads(threadConnection) {
let threads = threadConnection.edges.map(edge => {
const {blocks, entityMap} = JSON.parse(edge.node.content.body);
const url = `https://spectrum.chat/theia/${edge.node.channel.slug}/${edge.node.content.title.toLowerCase().replace(/[^a-z\s]/g,'').replace(/\s+/g,'-')}~${edge.node.id}`;
if (!threadUrls[edge.node.id]) {
threadUrls[edge.node.id] = {};
}
threadUrls[edge.node.id].spectrum = url;
let thread = [{
id: edge.node.id,
messageCount: edge.node.messageCount,
created_at: edge.node.createdAt,
title: edge.node.content.title,
raw: blocksToRaw(blocks, entityMap) + `\n\n<em>[<a href="${url}">original thread</a> by ${edge.node.author.user.name}]</em>`,
username: discourseUsers[edge.node.author.user.username] || 'discobot'
}];
comments[edge.node.author.user.username] = (comments[edge.node.author.user.username] || 0) + 1;
return thread;
});
for (const thread of threads) {
if (thread[0].messageCount > 0) {
let messages = await fetchSpectrumMessages(thread[0].id);
for (const message of messages) { thread.push(message); }
}
}
return threads;
}
async function fetchSpectrumThreads() {
// Note: I reverse-engineered these Spectrum payloads by using Spectrum in my browser and inspecting the API requests.
const {res, body} = await fetchSpectrumAPI({"operationName":"getCommunityThreadConnection","variables":{"id":"34dfd41a-dbbb-4432-ad9b-4bc6575b0e98","after":null,"sort":"latest"},"query":"query getCommunityThreadConnection($id: ID, $after: String, $sort: CommunityThreadConnectionSort) {\n community(id: $id) {\n ...communityInfo\n ...communityThreadConnection\n __typename\n }\n}\n\nfragment threadInfo on Thread {\n id\n messageCount\n createdAt\n modifiedAt\n lastActive\n receiveNotifications\n currentUserLastSeen\n editedBy {\n ...threadParticipant\n __typename\n }\n author {\n ...threadParticipant\n __typename\n }\n channel {\n ...channelInfo\n __typename\n }\n community {\n ...communityInfo\n ...communityMetaData\n __typename\n }\n isPublished\n isLocked\n isAuthor\n type\n content {\n title\n body\n __typename\n }\n attachments {\n attachmentType\n data\n __typename\n }\n watercooler\n metaImage\n reactions {\n count\n hasReacted\n __typename\n }\n __typename\n}\n\nfragment threadParticipant on ThreadParticipant {\n user {\n ...userInfo\n __typename\n }\n isMember\n isModerator\n isBlocked\n isOwner\n roles\n reputation\n __typename\n}\n\nfragment userInfo on User {\n id\n profilePhoto\n coverPhoto\n name\n firstName\n description\n website\n username\n isOnline\n timezone\n totalReputation\n betaSupporter\n __typename\n}\n\nfragment channelInfo on Channel {\n id\n name\n slug\n description\n isPrivate\n createdAt\n isArchived\n channelPermissions {\n isMember\n isPending\n isBlocked\n isOwner\n isModerator\n receiveNotifications\n __typename\n }\n community {\n ...communityInfo\n ...communityMetaData\n __typename\n }\n __typename\n}\n\nfragment communityInfo on Community {\n id\n createdAt\n name\n slug\n description\n website\n profilePhoto\n coverPhoto\n pinnedThreadId\n isPrivate\n watercoolerId\n lastActive\n communityPermissions {\n isMember\n isBlocked\n isOwner\n isPending\n isModerator\n reputation\n lastSeen\n __typename\n }\n brandedLogin {\n isEnabled\n message\n __typename\n }\n __typename\n}\n\nfragment communityMetaData on Community {\n metaData {\n members\n onlineMembers\n __typename\n }\n __typename\n}\n\nfragment communityThreadConnection on Community {\n pinnedThread {\n ...threadInfo\n __typename\n }\n watercooler {\n ...threadInfo\n __typename\n }\n threadConnection(first: 10, after: $after, sort: $sort) {\n pageInfo {\n hasNextPage\n hasPreviousPage\n __typename\n }\n edges {\n cursor\n node {\n ...threadInfo\n __typename\n }\n __typename\n }\n __typename\n }\n __typename\n}\n"});
console.log('Spectrum API posts', res.statusCode);
let {data} = JSON.parse(body);
let threadConnection = data.community.threadConnection;
let threads = await threadConnectionToThreads(threadConnection);
while (threadConnection.pageInfo.hasNextPage) {
const {res: re, body: bo} = await fetchSpectrumAPI({"operationName":"loadMoreCommunityThreads","variables":{"after":threadConnection.edges[threadConnection.edges.length-1].cursor,"id":"34dfd41a-dbbb-4432-ad9b-4bc6575b0e98"},"query":"query loadMoreCommunityThreads($after: String, $id: ID, $sort: CommunityThreadConnectionSort) {\n community(id: $id) {\n ...communityInfo\n ...communityThreadConnection\n __typename\n }\n}\n\nfragment threadInfo on Thread {\n id\n messageCount\n createdAt\n modifiedAt\n lastActive\n receiveNotifications\n currentUserLastSeen\n editedBy {\n ...threadParticipant\n __typename\n }\n author {\n ...threadParticipant\n __typename\n }\n channel {\n ...channelInfo\n __typename\n }\n community {\n ...communityInfo\n ...communityMetaData\n __typename\n }\n isPublished\n isLocked\n isAuthor\n type\n content {\n title\n body\n __typename\n }\n attachments {\n attachmentType\n data\n __typename\n }\n watercooler\n metaImage\n reactions {\n count\n hasReacted\n __typename\n }\n __typename\n}\n\nfragment threadParticipant on ThreadParticipant {\n user {\n ...userInfo\n __typename\n }\n isMember\n isModerator\n isBlocked\n isOwner\n roles\n reputation\n __typename\n}\n\nfragment userInfo on User {\n id\n profilePhoto\n coverPhoto\n name\n firstName\n description\n website\n username\n isOnline\n timezone\n totalReputation\n betaSupporter\n __typename\n}\n\nfragment channelInfo on Channel {\n id\n name\n slug\n description\n isPrivate\n createdAt\n isArchived\n channelPermissions {\n isMember\n isPending\n isBlocked\n isOwner\n isModerator\n receiveNotifications\n __typename\n }\n community {\n ...communityInfo\n ...communityMetaData\n __typename\n }\n __typename\n}\n\nfragment communityInfo on Community {\n id\n createdAt\n name\n slug\n description\n website\n profilePhoto\n coverPhoto\n pinnedThreadId\n isPrivate\n watercoolerId\n lastActive\n communityPermissions {\n isMember\n isBlocked\n isOwner\n isPending\n isModerator\n reputation\n lastSeen\n __typename\n }\n brandedLogin {\n isEnabled\n message\n __typename\n }\n __typename\n}\n\nfragment communityMetaData on Community {\n metaData {\n members\n onlineMembers\n __typename\n }\n __typename\n}\n\nfragment communityThreadConnection on Community {\n pinnedThread {\n ...threadInfo\n __typename\n }\n watercooler {\n ...threadInfo\n __typename\n }\n threadConnection(first: 10, after: $after, sort: $sort) {\n pageInfo {\n hasNextPage\n hasPreviousPage\n __typename\n }\n edges {\n cursor\n node {\n ...threadInfo\n __typename\n }\n __typename\n }\n __typename\n }\n __typename\n}\n"});
console.log('Spectrum API more posts', re.statusCode);
if (re.statusCode === 503) {
console.log('Rate limit!', re.statusCode, bo);
await new Promise(resolve => setTimeout(resolve, 30000));
continue;
}
threadConnection = JSON.parse(bo).data.community.threadConnection;
threads = threads.concat(await threadConnectionToThreads(threadConnection));
}
return threads;
}
async function messageConnectionToMessages(messageConnection) {
let messages = messageConnection.edges.map(edge => {
let message = {
created_at: edge.node.timestamp,
username: discourseUsers[edge.node.author.user.username] || 'discobot',
raw: ''
};
if (message.username === 'discobot') {
message.raw += `<em>[${edge.node.author.user.name}]</em>\n\n`;
}
comments[edge.node.author.user.username] = (comments[edge.node.author.user.username] || 0) + 1;
try {
const {blocks, entityMap} = JSON.parse(edge.node.content.body);
message.raw += blocksToRaw(blocks, entityMap);
if (!message.raw) {
console.log('message', message, 'edge', edge);
}
} catch (error) {
console.error(error, edge.node.content.body);
images.push({src: edge.node.content.body, alt: 'image'});
message.raw += edge.node.content.body;
}
return message;
}).sort((a,b) => {
if(a.created_at < b.created_at) { return -1; }
if(a.created_at > b.created_at) { return 1; }
return 0;
});
return messages;
}
async function fetchSpectrumMessages(threadId, includeFirst = false) {
const {res, body} = await fetchSpectrumAPI( {"operationName":"getThreadMessages","variables":{"id":threadId,"first":25},"query":"query getThreadMessages($id: ID!, $after: String, $first: Int, $before: String, $last: Int) {\n thread(id: $id) {\n ...threadInfo\n ...threadMessageConnection\n __typename\n }\n}\n\nfragment threadInfo on Thread {\n id\n messageCount\n createdAt\n modifiedAt\n lastActive\n receiveNotifications\n currentUserLastSeen\n editedBy {\n ...threadParticipant\n __typename\n }\n author {\n ...threadParticipant\n __typename\n }\n channel {\n ...channelInfo\n __typename\n }\n community {\n ...communityInfo\n ...communityMetaData\n __typename\n }\n isPublished\n isLocked\n isAuthor\n type\n content {\n title\n body\n __typename\n }\n attachments {\n attachmentType\n data\n __typename\n }\n watercooler\n metaImage\n reactions {\n count\n hasReacted\n __typename\n }\n __typename\n}\n\nfragment threadParticipant on ThreadParticipant {\n user {\n ...userInfo\n __typename\n }\n isMember\n isModerator\n isBlocked\n isOwner\n roles\n reputation\n __typename\n}\n\nfragment userInfo on User {\n id\n profilePhoto\n coverPhoto\n name\n firstName\n description\n website\n username\n isOnline\n timezone\n totalReputation\n betaSupporter\n __typename\n}\n\nfragment channelInfo on Channel {\n id\n name\n slug\n description\n isPrivate\n createdAt\n isArchived\n channelPermissions {\n isMember\n isPending\n isBlocked\n isOwner\n isModerator\n receiveNotifications\n __typename\n }\n community {\n ...communityInfo\n ...communityMetaData\n __typename\n }\n __typename\n}\n\nfragment communityInfo on Community {\n id\n createdAt\n name\n slug\n description\n website\n profilePhoto\n coverPhoto\n pinnedThreadId\n isPrivate\n watercoolerId\n lastActive\n communityPermissions {\n isMember\n isBlocked\n isOwner\n isPending\n isModerator\n reputation\n lastSeen\n __typename\n }\n brandedLogin {\n isEnabled\n message\n __typename\n }\n __typename\n}\n\nfragment communityMetaData on Community {\n metaData {\n members\n onlineMembers\n __typename\n }\n __typename\n}\n\nfragment threadMessageConnection on Thread {\n messageConnection(after: $after, first: $first, before: $before, last: $last) {\n pageInfo {\n hasNextPage\n hasPreviousPage\n __typename\n }\n edges {\n cursor\n node {\n ...messageInfo\n __typename\n }\n __typename\n }\n __typename\n }\n __typename\n}\n\nfragment messageInfo on Message {\n id\n timestamp\n modifiedAt\n messageType\n bot\n parent {\n id\n timestamp\n messageType\n author {\n ...threadParticipant\n __typename\n }\n content {\n body\n __typename\n }\n __typename\n }\n author {\n ...threadParticipant\n __typename\n }\n reactions {\n count\n hasReacted\n __typename\n }\n content {\n body\n __typename\n }\n __typename\n}\n"});
console.log('Spectrum API thread', res.statusCode);
if (res.statusCode === 503) {
console.log('Rate limit!', res.statusCode, threadId, body);
await new Promise(resolve => setTimeout(resolve, 30000));
return fetchSpectrumMessages(threadId, includeFirst);
}
if (res.statusCode !== 200) {
console.log('failed to fetch thread', threadId, body);
}
let thread = JSON.parse(body).data.thread;
let messageConnection = thread.messageConnection;
if (includeFirst) {
messageConnection.edges.unshift({node: thread});
}
let messages = await messageConnectionToMessages(messageConnection);
if (includeFirst) {
// This is a hack to include the original Spectrum comment in the list of replies, if needed.
messages[0].id = thread.id;
messages[0].messageCount = thread.messageCount;
messages[0].created_at = thread.createdAt;
messages[0].title = thread.content.title;
}
while (messageConnection.pageInfo.hasNextPage) {
const {res: re, body: bo} = await fetchSpectrumAPI( {"operationName":"getThreadMessages","variables":{"after":messageConnection.edges[messageConnection.edges.length-1].cursor,"id":threadId,"first":25},"query":"query getThreadMessages($id: ID!, $after: String, $first: Int, $before: String, $last: Int) {\n thread(id: $id) {\n ...threadInfo\n ...threadMessageConnection\n __typename\n }\n}\n\nfragment threadInfo on Thread {\n id\n messageCount\n createdAt\n modifiedAt\n lastActive\n receiveNotifications\n currentUserLastSeen\n editedBy {\n ...threadParticipant\n __typename\n }\n author {\n ...threadParticipant\n __typename\n }\n channel {\n ...channelInfo\n __typename\n }\n community {\n ...communityInfo\n ...communityMetaData\n __typename\n }\n isPublished\n isLocked\n isAuthor\n type\n content {\n title\n body\n __typename\n }\n attachments {\n attachmentType\n data\n __typename\n }\n watercooler\n metaImage\n reactions {\n count\n hasReacted\n __typename\n }\n __typename\n}\n\nfragment threadParticipant on ThreadParticipant {\n user {\n ...userInfo\n __typename\n }\n isMember\n isModerator\n isBlocked\n isOwner\n roles\n reputation\n __typename\n}\n\nfragment userInfo on User {\n id\n profilePhoto\n coverPhoto\n name\n firstName\n description\n website\n username\n isOnline\n timezone\n totalReputation\n betaSupporter\n __typename\n}\n\nfragment channelInfo on Channel {\n id\n name\n slug\n description\n isPrivate\n createdAt\n isArchived\n channelPermissions {\n isMember\n isPending\n isBlocked\n isOwner\n isModerator\n receiveNotifications\n __typename\n }\n community {\n ...communityInfo\n ...communityMetaData\n __typename\n }\n __typename\n}\n\nfragment communityInfo on Community {\n id\n createdAt\n name\n slug\n description\n website\n profilePhoto\n coverPhoto\n pinnedThreadId\n isPrivate\n watercoolerId\n lastActive\n communityPermissions {\n isMember\n isBlocked\n isOwner\n isPending\n isModerator\n reputation\n lastSeen\n __typename\n }\n brandedLogin {\n isEnabled\n message\n __typename\n }\n __typename\n}\n\nfragment communityMetaData on Community {\n metaData {\n members\n onlineMembers\n __typename\n }\n __typename\n}\n\nfragment threadMessageConnection on Thread {\n messageConnection(after: $after, first: $first, before: $before, last: $last) {\n pageInfo {\n hasNextPage\n hasPreviousPage\n __typename\n }\n edges {\n cursor\n node {\n ...messageInfo\n __typename\n }\n __typename\n }\n __typename\n }\n __typename\n}\n\nfragment messageInfo on Message {\n id\n timestamp\n modifiedAt\n messageType\n bot\n parent {\n id\n timestamp\n messageType\n author {\n ...threadParticipant\n __typename\n }\n content {\n body\n __typename\n }\n __typename\n }\n author {\n ...threadParticipant\n __typename\n }\n reactions {\n count\n hasReacted\n __typename\n }\n content {\n body\n __typename\n }\n __typename\n}\n"});
console.log('Spectrum API more messages', re.statusCode);
if (re.statusCode === 503) {
console.log('Rate limit!', re.statusCode, bo);
await new Promise(resolve => setTimeout(resolve, 30000));
continue;
}
messageConnection = JSON.parse(bo).data.thread.messageConnection;
messages = messages.concat(await messageConnectionToMessages(messageConnection));
}
return messages;
}
// THE MAIN FUNCTION
// Here you can see how the various functions are used, and you can comment out what you don't want to run.
// I suggest first working through the fetching of all Spectrum threads, then saving the result in a backup file.
// Next, I suggest loading all threads from the backup file, and importing them all into Discourse.
// And finally, you can (also load all threads from the backup file) and verify that all threads have been imported successfully.
(async () => {
try {
// Transfer a single Spectrum thread
/*const thread = await fetchSpectrumMessages('c2d31366-b6c4-4f9b-9181-eabe5288b6cf', true);
console.log('thread', thread);
await postDiscourseThread(thread);*/
// Transfer only the comments from a Spectrum thread to an existing Discourse topic
/*const thread = await fetchSpectrumMessages('c2d31366-b6c4-4f9b-9181-eabe5288b6cf');
await postDiscourseThreadMessages('196', thread);*/
// Here I bulk-renamed a series of Discourse topics:
/* const toRename = [191,186,183,79,188,176,177]; // all Discourse topicIds to process
for (const topic_id of toRename) {
let {res, body} = await fetchDiscourseAPI(`/t/${topic_id}.json`, {}, 'GET', 'discobot');
if (res.statusCode > 400) {
await new Promise(resolve => setTimeout(resolve, 35000));
continue;
}
const topic = JSON.parse(body);
console.log('GET topic', topic_id, res.statusCode, topic.title);
if (topic.title.startsWith('[Imported] ')) {
let {res: re, body: bo} = await fetchDiscourseAPI(`/t/-/${topic_id}.json`, { title: topic.title.replace('[Imported] ', '') }, 'PUT', 'discobot');
console.log('PUT topic', topic_id, re.statusCode, topic.title.replace('[Imported] ',''), bo);
if (re.statusCode > 400) {
await new Promise(resolve => setTimeout(resolve, 35000));
}
}
// This is how to reset the Dicourse bump date, e.g. if you modify an old Discourse topic but don't want it to come back up to the top of the list.
await fetchDiscourseAPI(`/t/${topic_id}/reset-bump-date`, {}, 'PUT', 'discobot');
}*/
// Fetch all Spectrum threads and import them all into Discourse
// (Skips duplicate threads, but doesn't add new comments on existing threads -- delete a Discourse thread to fully re-import it.)
const threads = await fetchSpectrumThreads();
console.log('images = ' + JSON.stringify(images, null, 2));
console.log('threads', threads);
// Save fetched threads into a backup file (useful for later verification):
fs.writeFileSync('./spectrum-threads.json', JSON.stringify(threads, null, 2), 'utf-8');
for (const thread of threads) {
await postDiscourseThread(thread);
}
// Then, after all threads were already fetched once, restore them from the backup file, then shuffle them (to be faster at skipping already imported threads while verifying all threads):
//const threads = shuffle(JSON.parse(fs.readFileSync('./spectrum-threads.json', 'utf-8')));
//console.log('Threads to verify:', threads.length);
// Verify that all fetched Spectrum threads are indeed on Discourse
// (verification sometimes fails even though the import was successful -- in that case, add the thread ID to the whitelist and restart the verification process.
const threadWhitelist = ['5ef2fc0d-376e-4ee8-b3fe-067003d37842'];
for (const thread of threads.filter(t => !threadWhitelist.includes(t[0].id))) {
await verifySpectrumThreadIsOnDiscourse(thread);
//await postDiscourseThread(thread);
}
} catch (error) {
console.error(error);
}
fs.writeFileSync('./spectrum-images.json', JSON.stringify({images}, null, 2), 'utf-8');
fs.writeFileSync('./spectrum-thread-urls.json', JSON.stringify(threadUrls, null, 2), 'utf-8');
})();
/**
* Shuffles array in place. ES6 version
* @param {Array} a items An array containing the items.
*/
function shuffle(a) {
for (let i = a.length - 1; i > 0; i--) {
const j = Math.floor(Math.random() * (i + 1));
[a[i], a[j]] = [a[j], a[i]];
}
return a;
}
var threadUrls = {}; // JSON.parse(fs.readFileSync('./spectrum-thread-urls.json', 'utf-8'));
@sdirix
Copy link

sdirix commented Jul 8, 2021

Thanks @jankeromnes! This script helped us a lot to successfully migrate our JSON Forms community from Spectrum to Discourse.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment