Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save aflashyrhetoric/eaeda5b2bad55e4604f18f7d2e71e9a0 to your computer and use it in GitHub Desktop.
Save aflashyrhetoric/eaeda5b2bad55e4604f18f7d2e71e9a0 to your computer and use it in GitHub Desktop.
/*
* This script converts API data from Backlog Syntax to Markdown.
* It can also output to Markdown and JSON.
*/
// Lib Imports
const fs = require('fs');
const path = require('path');
const chalk = require('chalk');
const readline = require('readline');
// App configuration imports
const backlogConfig = require('./backlog-config').en;
const topicsConfig = require('./topics-config');
// List of top-level topics that get their own section
let backlogTopLevelTopics = backlogConfig.topLevelTopics;
// CLI arguments
const outputFormat = process.argv[2].toLowerCase();
const inputPathArgument = process.argv[3];
const outputPathArgument = process.argv[4];
// bwiki libs
const STATUS = require('./status-codes');
const reportStatus = STATUS.reportStatus;
// Readline
let rl;
reportStatus(STATUS.STARTING_HUGO);
/**
* Extracts data, adds Hugo-specific front-matter
* @param {string} inputPathArgument - path to unconverted file
*/
fs.readFile(inputPathArgument, 'utf8', (err, data) => {
switch (outputFormat) {
case "md":
case "markdown":
saveAsMarkdown(data);
break;
case "json":
saveAsJson(data);
break;
case "hugo":
buildForHugo(data);
break;
default:
console.error(STATUS.ERROR_PARSE);
}
});
/*
* Save input string as outputPathArgument in markdown
*/
function saveAsMarkdown(data) {
reportStatus(STATUS.BEGIN_MARKDOWN_CONVERT);
convertToMarkdownSyntax(data)
.then((markdownData) => {
reportStatus(STATUS.FINISH_CONVERT);
reportStatus(STATUS.WRITING_TO_FILE);
fs.writeFile(outputPathArgument, markdownData, 'utf8', function(err) {
if (err) {
return console.log(chalk.red(err))
}
});
});
}
/*
* Save input string as outputPathArgument as JSON
*/
function saveAsJson(data) {
reportStatus(STATUS.BEGIN_MARKDOWN_INTERIM_CONVERT);
convertToMarkdownSyntax(data)
.then((markdownData) => {
reportStatus(STATUS.FINISH_CONVERT);
reportStatus(STATUS.BEGIN_JSON_CONVERT);
let lines = markdownData.split('\n');
extractTopics(lines).then((topics) => {
reportStatus(STATUS.FINISH_CONVERT);
reportStatus(STATUS.WRITING_TO_FILE);
fs.writeFile(outputPathArgument, JSON.stringify(topics), 'utf8', function(err) {
if (err) {
return console.log(chalk.red(err))
}
});
});
});
}
/**
* Extracts data, adds Hugo-specific front-matter
* @param {string} data - text string containing all API data
*/
function buildForHugo(data) {
reportStatus(STATUS.BEGIN_HUGO_BUILD);
convertToMarkdownSyntax(data)
.then((markdownData) => {
let lines = markdownData.split('\n');
extractTopics(lines).then((topics) => {
// Hard code value for now
let app = "backlog";
let now = Date.now();
// For each topic,
for (let topicNumber in topics) {
let currentTopic = topics[topicNumber];
let topicTitle = currentTopic.topicName;
let topicFilename =
topicTitle
.toLowerCase()
.replace(/ /g, "-")
.replace(/\?/g, "")
.replace(/\//g, "-");
let urlParameter = "";
// Conditionally add urlParameter
if (backlogConfig.urlMaps.hasOwnProperty(topicFilename)) {
urlParameter = `url = "${backlogConfig.urlMaps[topicFilename]}"`;
} else {
urlParameter = `url = /docs/backlog/api/2/${topicFilename}`;
}
// Don't change indentation,
// it seems to break ES6 template string's newlines.
let topicFrontMatter =
`
+++
isApi = false
weight = ${currentTopic.topicWeight}
title = "${topicTitle}"
fileName = "${topicFilename}"
apps = "${app}"
${urlParameter}
+++\n
`.substring(1);
let topicContent = currentTopic.topicContent.trim();
let newTopicItem = "";
newTopicItem = newTopicItem + topicFrontMatter + topicContent;
if (topicsConfig.blacklist.includes(topicFilename)) {
console.log(`"${topicFilename}.md" has been skipped, since it should not be included. Check topics-config.js for details.`)
} else {
fs.writeFile(`content/${app}/${topicFilename}.md`, newTopicItem, 'utf8', function(err) {
if (err) {
return console.log(chalk.red(err))
}
});
}
for (let subtopicNumber in currentTopic.subtopics) {
let currentSubtopic = currentTopic.subtopics[subtopicNumber];
let subtopicTitle = currentSubtopic.subtopicName;
let subtopicFilename =
subtopicTitle
.toLowerCase()
.replace(/ /g, "-")
.replace(/\?/g, "")
.replace(/\./g, "")
.replace(/\//g, "-");
// Remove parenthesis, if exists
subtopicFilename =
subtopicFilename
.replace(/\(/g, '')
.replace(/\)/g, '');
let subtopicUrlParameter = `url = "/docs/backlog/api/2/${subtopicFilename}"`;
let subtopicFrontMatter =
`
+++
isApi = ${currentSubtopic.isApi}
weight = ${currentSubtopic.subtopicWeight}
group = "${topicTitle}"
title = "${subtopicTitle}"
fileName = "${subtopicFilename}"
apps = "${app}"
${subtopicUrlParameter}
+++\n
`.substring(1);
let subtopicContent = currentSubtopic.subtopicContent.trim();
let newSubtopicItem = "";
newSubtopicItem = newSubtopicItem + subtopicFrontMatter + subtopicContent;
if (topicTitle == "Version history" && subtopicTitle != "API Libraries") {
console.log(`Version History individual versions shouldn't be saved to disk, so one such file has been skipped: ${subtopicTitle}`);
} else if (topicsConfig.blacklist.includes(subtopicFilename)) {
console.log(`"${subtopicFilename}.md" has been skipped, since it should not be included. Check topics-config.js for details.`)
} else {
fs.writeFile(`content/${app}/${subtopicFilename}.md`, newSubtopicItem, 'utf8', function(err) {
if (err) {
return console.log(chalk.red(err))
}
});
}
}
}
});
});
reportStatus(STATUS.WRITING_TO_FILE);
reportStatus(STATUS.FINISH_CONVERT);
}
/**
* Extract text and add semantic markers by converting to JSON
* @param {string} lines - text string containing all API data
*/
function extractTopics(lines) {
return new Promise((resolve, reject) => {
// Use pointers to build object inside for loop
let topics = [];
let newTopic = {};
let newTopicName = "";
let newTopicContent = "";
let newTopicWeight = 0;
let subtopics = {};
let newSubtopic = {};
let newSubtopicName = "";
let newSubtopicContent = "";
let newSubtopicWeight = 0;
let recordingTopic = false;
let recordingSubtopic = false;
let currentLineIsNewTopic = false;
let currentLineIsNewSubtopic = false;
let nextLineIsNewTopic = false;
let nextLineIsNewSubtopic = false;
let completed = false;
try {
// For each line
for (let i = 0; i < lines.length; i++) {
let currentLine = lines[i];
let nextIndex = i + 1;
let nextLine = lines[nextIndex];
// Check "nextLine" status
if (currentLine != null && nextLine != null) {
currentLineIsNewTopic = currentLine.substring(0, 2) == "# ";
currentLineIsNewSubtopic = currentLine.substring(0, 3) == "## ";
nextLineIsNewTopic = nextLine.substring(0, 2) == "# ";
nextLineIsNewSubtopic = nextLine.substring(0, 3) == "## ";
}
/***************************
* Check whether to capture
***************************/
if (currentLineIsNewTopic) {
let topicName = currentLine.substring(2);
newTopicName = topicName;
recordingTopic = true;
}
if (nextLineIsNewSubtopic) {
recordingTopic = false;
}
if (currentLineIsNewSubtopic) {
let subTopicName = currentLine.substring(3);
newSubtopicName = subTopicName;
recordingSubtopic = true;
}
/******************************
* Capture Lines (conditionally)
******************************/
if ((recordingTopic && !currentLineIsNewSubtopic) || backlogTopLevelTopics.includes(newTopicName)) {
newTopicContent += `${currentLine} \n`;
}
if (recordingSubtopic && !currentLineIsNewTopic) {
newSubtopicContent += `${currentLine} \n`;
}
/*****************
* Record content
*****************/
// Save SUBTOPIC
if ((nextLineIsNewSubtopic || nextLineIsNewTopic) && newSubtopicName.length > 0) {
// build and record subtopic as property of newTopic
let keyName = newSubtopicName.toLowerCase().replace(/ /g, "-").replace(/\?/g, "");
newSubtopic.subtopicName = newSubtopicName;
newSubtopic.subtopicContent = newSubtopicContent.trim();
newSubtopic.subtopicWeight = newSubtopicWeight;
// Only API items have the keyword "Role" as a field
if (newSubtopicContent.includes("### Method \n")) {
newSubtopic.isApi = true;
} else {
newSubtopic.isApi = false;
}
subtopics[keyName.toString()] = newSubtopic;
// Wipe data for next subtopic
newSubtopic = {};
newSubtopicName = "";
newSubtopicContent = "";
nextLineIsNewSubtopic = false;
// Increment subtopic weight (don't reset)
newSubtopicWeight++;
}
// Save TOPIC.
let isLastLine = (lines[i + 1] == null);
if ((nextLineIsNewTopic || isLastLine) && i > 1) {
// Build newTopic to be returned
newTopic.topicName = newTopicName;
newTopic.subtopics = subtopics;
newTopic.topicContent = newTopicContent.trim();
newTopic.topicWeight = newTopicWeight;
topics.push(newTopic);
// Reset newTopic and relevant fields
newTopic = {};
newTopicName = "";
newSubtopic = {};
newTopicContent = "";
subtopics = {};
nextLineIsNewTopic = false;
// Increment topic weight (don't reset)
newTopicWeight++;
}
}
} catch (err) {
console.error(`${STATUS.ERROR_PARSE}. \n ${err}`);
}
// Resolve
resolve(topics);
// Reject
reject(new Error(STATUS.ERROR_PARSE));
});
}
/**
* Converts Backlog syntax STRING to valid Markdown
* @param {string} data - path to raw data from file
*/
function convertToMarkdownSyntax(data) {
return new Promise((resolve, reject) => {
// Convert * to #
let result = data.replace(/\*/g, '#');
// Convert {code} to ```
result = result.replace(/({code})/g, '```');
result = result.replace(/({\/code})/g, '```\n');
// Convert [[a:b]] to [a](b)
result = result.replace(/(\[\[)([^\:]+)\:(([^\s]+))(\]\])/g, '[$2]($3)');
result = result.replace(/(&br;)/g, "<br>");
let lines = result.split('\n');
/**
* CUSTOM HARD-CODED PARSERS FOR BACKLOG SYNTAX
*/
// PARSE CODE KEYWORDS
let codeKeywords = [
"role", "url", "method", "scope"
];
// PARSE RESPONSE EXAMPLES
let responseExampleKeywords = [
"response example"
];
// URL PARAMETERS + QUERY PARAMETERS
let formKeywords = [
"url parameters",
"query parameters",
"response description",
"custom fields (text)",
"custom fields (numeric)",
"custom fields (date)",
"custom fields (list)"
];
// FORM PARAMETERS
let formParametersKeywords = [
"form parameters"
];
let keywords = codeKeywords.concat(responseExampleKeywords, formKeywords, formParametersKeywords);
for (let i = 0; i < lines.length; i++) {
let prevLine = (i > 0) ? lines[i] : null;
let currentLine = lines[i];
let lineContent = currentLine.substring(2).toLowerCase().trim();
let nextLine = lines[i + 1];
let nextNextLine = lines[i + 2];
// Some quick validation because I noticed inconsistencies
// if ( (currentLine.substring(0, 3) === "## ")
// && (i - 1 > 0)
// && (lines[i-1].length > 0)) {
// // lines.splice(i-1, 0, "\n");
// lines[i-1] = "\n";
// }
// Set section headers to <h4>
if (keywords.includes(lineContent)) {
if (nextLine.length > 0) {
// If this section has content, trim and markdownify
lines[i] = `### ${currentLine.substring(2).trim()}`;
} else {
// Otherwise, remove array entries to keep markdown files clean
lines.splice(i - 1, 3, "");
i -= 2;
}
}
if (codeKeywords.includes(lineContent) && nextLine.length > 0) {
lines.splice(i + 1, 0, `\`\`\``);
// Account for multiple
let offset = 2;
while (lines[i + offset].length > 0) {
offset += 1;
}
lines.splice(i + offset, 0, `\`\`\``);
}
if (responseExampleKeywords.includes(lineContent) && nextLine.length > 0) {
// Restructure
lines.splice(i + 1, 0, `#### Status Line / Response Header`);
// Account for potential multiple lines
let offset = 5;
while (lines[i + offset] != '```') {
offset += 1;
}
lines.splice(i + offset + 1, 0, `#### Response Body`);
}
if (formKeywords.includes(lineContent) && nextLine.length > 0) {
// If there's a table
lines.splice(i + 1, 0, 'Parameter Name | Type | Description');
lines.splice(i + 2, 0, '---|---|---');
let offset = 3;
// while the first character is "|", trim to make valid syntax
while (lines[i + offset].substring(0, 1) == "|") {
lines[i + offset] = lines[i + offset].slice(1, -1).trim();
offset += 1;
}
}
if (formParametersKeywords.includes(lineContent) && nextLine.length > 0) {
lines.splice(i + 1, 0, `\`\`\``);
lines.splice(i + 3, 0, `\`\`\``);
lines.splice(i + 4, 0, ``);
if (lines[i + 5].substring(0, 1) == "|") {
// Remove first and last | character
let offset = 5;
// wipe lines until you hit a blank line
while (lines[i + offset].length != 0) {
lines[i + offset] = lines[i + offset].slice(1, -1).trim();
offset += 1;
}
lines.splice(i + 6, 0, `\-\-\-\- | \-\-\-\- | \-\-\-\-`);
}
}
}
// Reassemble
result = lines.join('\n');
// Return markdown-ified data
if (result.length > 1) {
resolve(result);
} else {
reject(new Error(STATUS.ERROR_PARSE));
}
});
}
"use strict"
/*
* This script converts Japanese API data from Backlog Syntax to Markdown.
* It can also output to Markdown and JSON.
*/
// Lib Imports
const fs = require('fs');
const path = require('path');
const chalk = require('chalk');
const readline = require('readline');
// App configuration imports
const backlogConfig = require('./backlog-config').ja;
const topicsConfig = require('./topics-config');
// List of top-level topics that get their own section
const backlogTopLevelTopics = backlogConfig.topLevelTopics;
// CLI arguments
const outputFormat = process.argv[2].toLowerCase();
const inputPathArgument = process.argv[3];
const outputPathArgument = process.argv[4];
// bwiki libs
const STATUS = require('./status-codes');
const reportStatus = STATUS.reportStatus;
// Readline
let rl;
reportStatus(STATUS.STARTING_HUGO);
/**
* Extracts data, adds Hugo-specific front-matter
* @param {string} inputPathArgument - path to unconverted file
*/
fs.readFile(inputPathArgument, 'utf8', (err, data) => {
switch (outputFormat) {
case "md":
case "markdown":
saveAsMarkdown(data);
break;
case "json":
saveAsJson(data);
break;
case "hugo":
buildForHugo(data);
break;
default:
console.error(STATUS.ERROR_PARSE);
}
});
/*
* Save input string as outputPathArgument in markdown
*/
function saveAsMarkdown(data) {
reportStatus(STATUS.BEGIN_MARKDOWN_CONVERT);
convertToMarkdownSyntax(data)
.then((markdownData) => {
reportStatus(STATUS.FINISH_CONVERT);
reportStatus(STATUS.WRITING_TO_FILE);
fs.writeFile(outputPathArgument, markdownData, 'utf8', function(err) {
if (err) {
return console.log(chalk.red(err))
}
});
});
}
/*
* Save input string as outputPathArgument as JSON
*/
function saveAsJson(data) {
reportStatus(STATUS.BEGIN_MARKDOWN_INTERIM_CONVERT);
convertToMarkdownSyntax(data)
.then((markdownData) => {
reportStatus(STATUS.FINISH_CONVERT);
reportStatus(STATUS.BEGIN_JSON_CONVERT);
let lines = markdownData.split('\n');
extractTopics(lines).then((topics) => {
reportStatus(STATUS.FINISH_CONVERT);
reportStatus(STATUS.WRITING_TO_FILE);
fs.writeFile(outputPathArgument, JSON.stringify(topics), 'utf8', function(err) {
if (err) {
return console.log(chalk.red(err))
}
});
});
});
}
/**
* Extracts data, adds Hugo-specific front-matter
* @param {string} data - text string containing all API data
*/
function buildForHugo(data) {
reportStatus(STATUS.BEGIN_HUGO_BUILD);
convertToMarkdownSyntax(data)
.then((markdownData) => {
let lines = markdownData.split('\n');
let headingsList = JSON.parse(fs.readFileSync('./api-data/headings-list.json', ''));
let subheadingsList = JSON.parse(fs.readFileSync('./api-data/subheadings-list.json', ''));
extractTopics(lines).then((topics) => {
// Hard code value for now
let app = "backlog";
// For each topic,
// for (let topicNumber of topics) {
for (let i = 0; i < topics.length; i++) {
// let currentTopic = topicNumber;
let currentTopic = topics[i];
let topicTitle = currentTopic.topicName;
// console.log(chalk.blue(topicTitle))
let topicFilename =
topicTitle
.toLowerCase()
.replace(/ /g, "-")
.replace(/\?/g, "")
.replace(/\//g, "-");
let urlParameter = "";
// Conditionally add urlParameter
if (backlogConfig.urlMaps.hasOwnProperty(topicFilename)) {
urlParameter = `url = "${backlogConfig.urlMaps[topicFilename]}"`;
} else {
topicFilename =
currentTopic.topicName
.toLowerCase()
.replace(/ /g, "-")
.replace(/\?/g, "")
.replace(/\//g, "-");
urlParameter = `url = "/ja/docs/backlog/api/2/${topicFilename}"`;
}
// Don't change indentation,
// it seems to break ES6 template string's newlines.
let topicFrontMatter =
`
+++
isApi = false
weight = ${currentTopic.topicWeight}
title = "${currentTopic.japaneseTopicName}"
fileName = "${topicFilename}"
${urlParameter}
+++\n
`.substring(1);
let topicContent = currentTopic.topicContent;
let newTopicItem = "";
newTopicItem = newTopicItem + topicFrontMatter + topicContent;
if (topicsConfig.blacklist.includes(topicFilename)) {
console.log(`"${topicFilename}.md" has been skipped, since it should not be included. Check topics-config.js for details.`)
} else {
console.log(chalk.green(`Successfully saved: ${topicFilename}.ja.md`));
fs.writeFile(`content/${app}/${topicFilename}.ja.md`, newTopicItem, 'utf8', function(err) {
if (err) {
return console.log(chalk.red(err))
}
});
}
try {
for (let subtopicKey in currentTopic.subtopics) {
let currentSubtopic = currentTopic.subtopics[subtopicKey];
// let currentSubtopic = subtopicKey;
let subtopicTitle = currentSubtopic.subtopicName;
let subtopicFilename =
subtopicTitle
.toLowerCase()
.replace(/ /g, "-")
.replace(/\?/g, "")
.replace(/\./g, "")
.replace(/\//g, "-")
// Remove parenthesis, if exists
.replace(/\(/g, '')
.replace(/\)/g, '');
let subtopicUrlParameter = `url = "/ja/docs/backlog/api/2/${subtopicFilename}"`;
let subtopicFrontMatter =
`
+++
isApi = ${currentSubtopic.isApi}
weight = ${currentSubtopic.subtopicWeight}
group = "${topicTitle}"
title = "${currentSubtopic.japaneseSubtopicName}"
fileName = "${subtopicFilename}"
${subtopicUrlParameter}
+++\n
`.substring(1);
let subtopicContent = currentSubtopic.subtopicContent.trim();
let newSubtopicItem = "";
newSubtopicItem = newSubtopicItem + subtopicFrontMatter + subtopicContent;
if (topicTitle == "Version history" && subtopicTitle != "API Libraries") {
console.log(`Version History individual versions shouldn't be saved to disk, so one such file has been skipped: ${chalk.red(subtopicTitle)}`);
} else if (topicsConfig.blacklist.includes(subtopicFilename)) {
console.log(`"${subtopicFilename}.md" has been skipped, since it should not be included. Check topics-config.js for details.`)
} else {
fs.writeFile(`content/${app}/${subtopicFilename}.ja.md`, newSubtopicItem, 'utf8', function(err) {
if (err) {
return console.log(chalk.red(err))
}
});
}
}
} catch (e) {
console.error(chalk.red(e));
}
}
});
});
reportStatus(STATUS.WRITING_TO_FILE);
reportStatus(STATUS.FINISH_CONVERT);
}
/**
* Extract text and add semantic markers by converting to JSON
* @param {string} lines - text string containing all API data
*/
function extractTopics(lines) {
return new Promise((resolve, reject) => {
// Use pointers to build object inside for loop
let topics = [];
let newTopic = {};
let newTopicName = "";
let newTopicContent = "";
let newTopicWeight = 0;
let subtopics = {};
let newSubtopic = {};
let newSubtopicName = "";
let newSubtopicContent = "";
let newSubtopicWeight = 0;
let recordingTopic = false;
let recordingSubtopic = false;
let currentLineIsNewTopic = false;
let currentLineIsNewSubtopic = false;
let nextLineIsNewTopic = false;
let nextLineIsNewSubtopic = false;
let completed = false;
let headingsList = JSON.parse(fs.readFileSync('./api-data/headings-list.json', ''));
let subheadingsList = JSON.parse(fs.readFileSync('./api-data/subheadings-list.json', ''));
try {
// For each line
for (let i = 0; i < lines.length; i++) {
let currentLine = lines[i];
let nextIndex = i + 1;
let nextLine = lines[nextIndex];
// Check "nextLine" status
if (currentLine != null && nextLine != null) {
currentLineIsNewTopic = currentLine.substring(0, 2) == "# ";
currentLineIsNewSubtopic = currentLine.substring(0, 3) == "## ";
nextLineIsNewTopic = nextLine.substring(0, 2) == "# ";
nextLineIsNewSubtopic = nextLine.substring(0, 3) == "## ";
}
/***************************
* Check whether to capture
***************************/
if (currentLineIsNewTopic) {
let topicName = currentLine.substring(2);
newTopicName = topicName;
recordingTopic = true;
}
if (nextLineIsNewSubtopic) {
recordingTopic = false;
}
if (currentLineIsNewSubtopic) {
let subTopicName = currentLine.substring(3);
newSubtopicName = subTopicName;
recordingSubtopic = true;
}
/******************************
* Capture Lines (conditionally)
******************************/
if ((recordingTopic && !currentLineIsNewSubtopic) || backlogTopLevelTopics.includes(newTopicName)) {
newTopicContent += `${currentLine} \n`;
}
if (recordingSubtopic && !currentLineIsNewTopic) {
newSubtopicContent += `${currentLine} \n`;
}
/*****************
* Record content
*****************/
// Save SUBTOPIC
if ((nextLineIsNewSubtopic || nextLineIsNewTopic) && newSubtopicName.length > 0) {
newSubtopic.japaneseSubtopicName = newSubtopicName;
newSubtopic.subtopicName = subheadingsList.shift().toString();
newSubtopic.subtopicContent = newSubtopicContent.trim();
newSubtopic.subtopicWeight = newSubtopicWeight;
let keyName = newSubtopic.subtopicName.toString().toLowerCase().replace(/ /g, "-").replace(/\?/g, "");
// Only API items have the keyword "Role" as a field
if (newSubtopicContent.includes("### メソッド \n")) {
newSubtopic.isApi = true;
} else {
newSubtopic.isApi = false;
}
if(keyName != null) {
subtopics[keyName] = newSubtopic;
}
// Wipe data for next subtopic
newSubtopic = {};
newSubtopicName = "";
newSubtopicContent = "";
nextLineIsNewSubtopic = false;
// Increment subtopic weight (don't reset)
newSubtopicWeight++;
}
// Save TOPIC.
let isLastLine = (lines[i + 1] == null);
if ((nextLineIsNewTopic || isLastLine) && i > 1) {
// Build newTopic to be returned
newTopic.japaneseTopicName = newTopicName;
newTopic.topicName = headingsList.shift();
newTopic.subtopics = subtopics;
newTopic.topicContent = newTopicContent.trim();
newTopic.topicWeight = newTopicWeight;
topics.push(newTopic);
// Reset newTopic and relevant fields
newTopic = {};
newTopicName = "";
newSubtopic = {};
newTopicContent = "";
subtopics = {};
nextLineIsNewTopic = false;
// Increment topic weight (don't reset)
newTopicWeight++;
}
}
} catch (err) {
console.error(`${STATUS.ERROR_PARSE}. \n ${err}`);
}
// Resolve
resolve(topics);
// Reject
reject(new Error(STATUS.ERROR_PARSE));
});
}
/**
* Converts Backlog syntax STRING to valid Markdown
* @param {string} data - path to raw data from file
*/
function convertToMarkdownSyntax(data) {
return new Promise((resolve, reject) => {
// Convert * to #
let result = data.replace(/\*/g, '#');
// Convert {code} to ```
result = result.replace(/({code})/g, '```');
result = result.replace(/({\/code})/g, '```\n');
// Convert [[a:b]] to [a](b)
result = result.replace(/(\[\[)([^\:]+)\:(([^\s]+))(\]\])/g, '[$2]($3)');
result = result.replace(/(&br;)/g, "<br>");
let lines = result.split('\n');
/**
* CUSTOM HARD-CODED PARSERS FOR BACKLOG SYNTAX
*/
// PARSE CODE KEYWORDS
let codeKeywords = [
"権限", "url", "メソッド", "scope"
];
// PARSE RESPONSE EXAMPLES
let responseExampleKeywords = [
"レスポンス例"
];
// URL PARAMETERS + QUERY PARAMETERS
let formKeywords = [
"URL パラメーター",
"リクエストパラメーター",
"レスポンス説明",
"カスタム属性を指定した検索 (テキスト属性)",
"カスタム属性を指定した検索 (数値属性)",
"カスタム属性を指定した検索 (日付属性)",
"カスタム属性を指定した検索 (リスト属性)"
];
// FORM PARAMETERS
let formParametersKeywords = [
"リクエストパラメーター"
];
let keywords = codeKeywords.concat(responseExampleKeywords, formKeywords, formParametersKeywords);
for (let i = 0; i < lines.length; i++) {
let prevLine = (i > 0) ? lines[i] : null;
let currentLine = lines[i];
let lineContent = currentLine.substring(2).toLowerCase().trim();
let nextLine = lines[i + 1];
let nextNextLine = lines[i + 2];
// Set section headers to <h4>
if (keywords.includes(lineContent)) {
if (nextLine.length > 0) {
// If this section has content, trim and markdownify
lines[i] = `### ${currentLine.substring(2).trim()}`;
} else {
// Otherwise, remove array entries to keep markdown files clean
lines.splice(i - 1, 3, "");
i -= 2;
}
}
if (codeKeywords.includes(lineContent) && nextLine.length > 0) {
lines.splice(i + 1, 0, `\`\`\``);
// Account for multiple
let offset = 2;
while (lines[i + offset].length > 0) {
offset += 1;
}
lines.splice(i + offset, 0, `\`\`\``);
}
if (responseExampleKeywords.includes(lineContent) && nextLine.length > 0) {
// Restructure
lines.splice(i + 1, 0, `#### ステータスライン / レスポンスヘッダ`);
// Account for potential multiple lines
let offset = 5;
while (lines[i + offset] != '```') {
offset += 1;
}
lines.splice(i + offset + 1, 0, `#### レスポンスボディ`);
}
if (formKeywords.includes(lineContent) && nextLine.length > 0) {
lines.splice(i + 1, 0, 'パラメーター名 | 型 | 内容');
lines.splice(i + 2, 0, '---|---|---');
// If there's a table
let offset = 3;
// while the first character is "|", trim to make valid syntax
while (lines[i + offset].substring(0, 1) == "|") {
lines[i + offset] = lines[i + offset].slice(1, -1).trim();
offset += 1;
}
}
if (formParametersKeywords.includes(lineContent) && nextLine.length > 0) {
lines.splice(i + 1, 0, `\`\`\``);
lines.splice(i + 3, 0, `\`\`\``);
lines.splice(i + 4, 0, ``);
if (lines[i + 5].substring(0, 1) == "|") {
// Remove first and last | character
let offset = 5;
// wipe lines until you hit a blank line
while (lines[i + offset].length != 0) {
lines[i + offset] = lines[i + offset].slice(1, -1).trim();
offset += 1;
}
lines.splice(i + 6, 0, `\-\-\-\- | \-\-\-\- | \-\-\-\-`);
}
}
}
// Reassemble
result = lines.join('\n');
// Return markdown-ified data
if (result.length > 1) {
resolve(result);
} else {
reject(new Error(STATUS.ERROR_PARSE));
}
});
}
/*
* This script converts Japanese API data from Backlog Syntax to Markdown.
* It can also output to Markdown and JSON.
*/
// Lib Imports
const fs = require('fs');
const path = require('path');
const chalk = require('chalk');
const readline = require('readline');
// App configuration imports
const backlogConfig = require('./backlog-config').en;
const backlogTopLevelTopics = backlogConfig.topLevelTopics;
const topicsConfig = require('./topics-config');
// CLI arguments
const inputPathArgument = process.argv[2];
// bwiki libs
const STATUS = require('./status-codes');
const reportStatus = STATUS.reportStatus;
// Readline
let rl;
let headingsList = [];
let subheadingsList = [];
fs.readFile(inputPathArgument, 'utf8', (err, data) => {
generateHeadingsList(data);
});
/**
* Extracts data, adds Hugo-specific front-matter
* @param {string} data - text string containing all API data
*/
function generateHeadingsList(data) {
reportStatus(STATUS.GENERATING_TRANSLATION_MAP);
convertToMarkdownSyntax(data)
.then((markdownData) => {
let lines = markdownData.split('\n');
extractTopics(lines).then((topics) => {
// For each topic
for (let topicNumber in topics) {
let currentTopic = topics[topicNumber];
let topicTitle = currentTopic.topicName;
headingsList.push(topicTitle);
for (let subtopicNumber in currentTopic.subtopics) {
let currentSubtopic = currentTopic.subtopics[subtopicNumber];
let subtopicTitle = currentSubtopic.subtopicName;
let subtopicFilename =
subtopicTitle
.toLowerCase()
.replace(/\s/g, "-")
.replace(/\?/g, "")
.replace(/\//g, "-")
.replace(/\./g, "")
.replace(/\(/g, "")
.replace(/\)/g, "");
subheadingsList.push(subtopicTitle);
}
}
fs.writeFile('api-data/headings-list.json', JSON.stringify(headingsList), 'utf8');
fs.writeFile('api-data/subheadings-list.json', JSON.stringify(subheadingsList), 'utf8');
});
});
reportStatus(STATUS.GENERATING_TRANSLATION_MAP_DONE);
reportStatus(STATUS.FINISH_CONVERT);
}
/**
* Extract text and add semantic markers by converting to JSON
* @param {string} lines - text string containing all API data
*/
function extractTopics(lines) {
return new Promise((resolve, reject) => {
// Use pointers to build object inside for loop
let topics = [];
let newTopic = {};
let newTopicName = "";
let newTopicContent = "";
let newTopicWeight = 0;
let subtopics = {};
let newSubtopic = {};
let newSubtopicName = "";
let newSubtopicContent = "";
let newSubtopicWeight = 0;
let recordingTopic = false;
let recordingSubtopic = false;
let currentLineIsNewTopic = false;
let currentLineIsNewSubtopic = false;
let nextLineIsNewTopic = false;
let nextLineIsNewSubtopic = false;
let completed = false;
// List of top-level topics that get their own section
let backlogTopLevelTopics = backlogConfig.topLevelTopics;
try {
// For each line
for (let i = 0; i < lines.length; i++) {
let currentLine = lines[i];
let nextIndex = i + 1;
let nextLine = lines[nextIndex];
// Check "nextLine" status
if (currentLine != null && nextLine != null) {
currentLineIsNewTopic = currentLine.substring(0, 2) == "# ";
currentLineIsNewSubtopic = currentLine.substring(0, 3) == "## ";
nextLineIsNewTopic = nextLine.substring(0, 2) == "# ";
nextLineIsNewSubtopic = nextLine.substring(0, 3) == "## ";
}
/***************************
* Check whether to capture
***************************/
if (currentLineIsNewTopic) {
let topicName = currentLine.substring(2);
newTopicName = topicName;
recordingTopic = true;
}
if(nextLineIsNewSubtopic) {
recordingTopic = false;
}
if (currentLineIsNewSubtopic) {
let subTopicName = currentLine.substring(3);
newSubtopicName = subTopicName;
recordingSubtopic = true;
}
/******************************
* Capture Lines (conditionally)
******************************/
if ((recordingTopic && !currentLineIsNewSubtopic) || backlogTopLevelTopics.includes(newTopicName)) {
newTopicContent += `${currentLine} \n`;
}
if (recordingSubtopic && !currentLineIsNewTopic) {
newSubtopicContent += `${currentLine} \n`;
}
/*****************
* Record content
*****************/
// Save TOPIC.
let isLastLine = (lines[i+1] == null);
if ((nextLineIsNewTopic || isLastLine ) && i > 1) {
// Build newTopic to be returned
newTopic.topicName = newTopicName;
newTopic.subtopics = subtopics;
newTopic.topicContent = newTopicContent.trim();
newTopic.topicWeight = newTopicWeight;
topics.push(newTopic);
// Reset newTopic and relevant fields
newTopic = {};
newTopicName = "";
newSubtopic = {};
newTopicContent = "";
subtopics = {};
nextLineIsNewTopic = false;
// Increment topic weight (don't reset)
newTopicWeight++;
}
// Save SUBTOPIC
if (nextLineIsNewSubtopic && newSubtopicName.length > 0) {
// build and record subtopic as property of newTopic
let keyName = newSubtopicName.toLowerCase().replace(/ /g, "-").replace(/\?/g, "");
newSubtopic.subtopicName = newSubtopicName;
newSubtopic.subtopicContent = newSubtopicContent.trim();
newSubtopic.subtopicWeight = newSubtopicWeight;
// Only API items have the keyword "Role" as a field
if (newSubtopicContent.includes("### Method \n")) {
newSubtopic.isApi = true;
} else {
newSubtopic.isApi = false;
}
subtopics[keyName.toString()] = newSubtopic;
// Wipe data for next subtopic
newSubtopic = {};
newSubtopicName = "";
newSubtopicContent = "";
nextLineIsNewSubtopic = false;
// Increment subtopic weight (don't reset)
newSubtopicWeight++;
}
}
} catch (err) {
console.error(`${STATUS.ERROR_PARSE}. \n ${err}`);
}
// Resolve
resolve(topics);
// Reject
reject(new Error(STATUS.ERROR_PARSE));
});
}
/**
* Converts Backlog syntax STRING to valid Markdown
* @param {string} data - path to raw data from file
*/
function convertToMarkdownSyntax(data) {
return new Promise((resolve, reject) => {
// Convert * to #
let result = data.replace(/\*/g, '#');
try {
// Convert {code} to ```
result = result.replace(/({code})/g, '```');
result = result.replace(/({\/code})/g, '```\n');
// Convert [[a:b]] to [a](b)
result = result.replace(/(\[\[)([^\:]+)\:(([^\s]+))(\]\])/g, '[$2]($3)');
result = result.replace(/(&br;)/g, "<br>");
} catch (e) {
console.error("Error while performing regex: " + e);
}
let lines = result.split('\n');
/**
* CUSTOM HARD-CODED PARSERS FOR BACKLOG SYNTAX
*/
// PARSE CODE KEYWORDS
let codeKeywords = [
"role", "url", "method", "scope"
];
// PARSE RESPONSE EXAMPLES
let responseExampleKeywords = [
"response example"
];
// URL PARAMETERS + QUERY PARAMETERS
let formKeywords = [
"url parameters",
"query parameters",
"response description",
"custom fields (text)",
"custom fields (numeric)",
"custom fields (date)",
"custom fields (list)"
];
// FORM PARAMETERS
let formParametersKeywords = [
"form parameters"
];
let keywords = codeKeywords.concat(responseExampleKeywords, formKeywords, formParametersKeywords);
for (let i = 0; i < lines.length; i++) {
let prevLine = (i > 0) ? lines[i] : null;
let currentLine = lines[i];
let lineContent = currentLine.substring(2).toLowerCase().trim();
let nextLine = lines[i + 1];
let nextNextLine = lines[i + 2];
// Set section headers to <h4>
if (keywords.includes(lineContent)) {
if (nextLine.length > 0) {
// If this section has content, trim and markdownify
lines[i] = `### ${currentLine.substring(2).trim()}`;
} else {
// Otherwise, remove array entries to keep markdown files clean
lines.splice(i - 1, 3, "");
i -= 2;
}
}
if (codeKeywords.includes(lineContent) && nextLine.length > 0) {
lines.splice(i + 1, 0, `\`\`\``);
// Account for multiple
let offset = 2;
while (lines[i + offset].length > 0) {
offset += 1;
}
lines.splice(i + offset, 0, `\`\`\``);
}
if (responseExampleKeywords.includes(lineContent) && nextLine.length > 0) {
// Restructure
lines.splice(i + 1, 0, `#### Status Line / Response Header`);
// Account for potential multiple lines
let offset = 5;
while (lines[i + offset] != '```') {
offset += 1;
}
lines.splice(i + offset + 1, 0, `#### Response Body`);
}
if (formKeywords.includes(lineContent) && nextLine.length > 0) {
// If there's a table
lines.splice(i + 1, 0, 'Parameter Name | Type | Description');
lines.splice(i + 2, 0, '---|---|---');
let offset = 3;
// while the first character is "|", trim to make valid syntax
while (lines[i + offset].substring(0, 1) == "|") {
lines[i + offset] = lines[i + offset].slice(1, -1).trim();
offset += 1;
}
}
if (formParametersKeywords.includes(lineContent) && nextLine.length > 0) {
lines.splice(i + 1, 0, `\`\`\``);
lines.splice(i + 3, 0, `\`\`\``);
lines.splice(i + 4, 0, ``);
if (lines[i + 5].substring(0, 1) == "|") {
// Remove first and last | character
let offset = 5;
// wipe lines until you hit a blank line
while (lines[i + offset].length != 0) {
lines[i + offset] = lines[i + offset].slice(1, -1).trim();
offset += 1;
}
lines.splice(i + 6, 0, `\-\-\-\- | \-\-\-\- | \-\-\-\-`);
}
}
}
// Reassemble
result = lines.join('\n');
// Return markdown-ified data
if (result.length > 1) {
resolve(result);
} else {
reject(new Error(STATUS.ERROR_PARSE));
}
});
}
// backlog-config
let backlogConfig = {};
backlogConfig.en = {
// These topics are in the sidebar
topLevelTopics: [
"Version history",
"Backlog API Overview",
"Error Response",
"Authentication & Authorization"
],
urlMaps: {
"backlog-api-overview": "/docs/backlog/",
"authentication-&-authorization": "/docs/backlog/auth",
"error-response": "/docs/backlog/error-response",
"version-history": "/docs/backlog/changes",
"oauth-20" : "/docs/backlog/api/2/oauth2"
}
};
backlogConfig.ja = {
topLevelTopics: [
"変更履歴",
"Backlog API とは",
"エラーレスポンス",
"認証と認可"
],
urlMaps: {
"backlog-api-overview": "/ja/docs/backlog/",
"authentication-&-authorization": "/ja/docs/backlog/auth",
"error-response": "/ja/docs/backlog/error-response",
"version-history": "/ja/docs/backlog/changes",
"oauth-20" : "/ja/docs/backlog/api/1/oauth2"
}
}
module.exports = backlogConfig;
// STATUS CODES
const chalk = require('chalk');
const outputFormat = process.argv[2].toLowerCase();
const inputPathArgument = process.argv[3];
const outputPathArgument = process.argv[4];
let STATUS = {
BEGIN_MARKDOWN_CONVERT: "Converting to Markdown...",
BEGIN_MARKDOWN_INTERIM_CONVERT: "Converting to Markdown first...",
BEGIN_JSON_CONVERT: "Converting to JSON...",
STARTING_HUGO: `Beginning conversion to ${outputFormat.toUpperCase()}...\n`,
BEGIN_HUGO_BUILD: "Adding Hugo Front Matter & saving to disk...",
FINISH_CONVERT: "Success!\n",
ERROR_PARSE: "Error: couldn't parse input. Please check syntax and try again. \n",
WRITING_TO_FILE: `SAVED: "${outputPathArgument}".\n`,
// For generate-translation-map
GENERATING_TRANSLATION_MAP: "Generating headings list for use in Japanese translations...\n\n",
GENERATING_TRANSLATION_MAP_DONE: "Done. \nCheck api-data/ for headings-list.json and subheadings-list.json files. \nIf you want to generate Japanese Markdown content, run `node backlog-syntax-converter-jp.js hugo /path/to/output/folder`"
}
// Change status messages for Hugo
STATUS.WRITING_TO_FILE = outputFormat == "hugo" ? "Files saved in Hugo directories." : STATUS.WRITING_TO_FILE;
// Helper function function reportStatus(message) {
STATUS.reportStatus = function(message) {
console.log(chalk.bold(message));
}
module.exports = STATUS;
// topics-config
let topicsConfig = {
blacklist: [
"200-2014-07-07",
"git",
"group",
"issue",
"notification",
"oauth-2.0",
"priority",
"project",
"pull-request",
"resolution",
"space",
"star",
"status",
"users",
"watching",
"wiki",
"oauth-20",
"スペース",
"ユーザー",
"グループ",
"状態",
"完了理由",
"優先度",
"プロジェクト",
"課題",
"Wiki",
"スター",
"お知らせ",
"Git",
"プルリクエスト",
"ウォッチ",
]
};
module.exports = topicsConfig;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment