markdown.ts

import invariant from 'tiny-invariant';
import 'zx/globals';
import { event, info } from '../logger';

// Patched: use domino-ext instead of domino in turndown.umd.js
// ref: https://github.com/fgnass/domino/issues/146
// @ts-ignore
import TurndownService from 'turndown/lib/turndown.umd.js';

(async () => {
  info(argv);
  invariant(argv.file, 'file is required');

  // why not https://github.com/sindresorhus/clipboardy?
  // since it's only support plain text
  const html = require('pb').get('html');
  info(`html: ${html}`);

  //   const html = `
  // <meta charset='utf-8'><h2 dir="auto" style="box-sizing: border-box; margin-top: 24px; margin-bottom: 16px; font-size: 1.5em; font-weight: 600; line-height: 1.25; padding-bottom: 0.3em; border-bottom: 1px solid var(--color-border-muted); color: rgb(36, 41, 47); font-family: -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, Helvetica, Arial, sans-serif, &quot;Apple Color Emoji&quot;, &quot;Segoe UI Emoji&quot;; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;">Install</h2><div class="highlight highlight-source-shell notranslate position-relative overflow-auto" style="box-sizing: border-box; position: relative !important; overflow: auto !important; margin-bottom: 16px; color: rgb(36, 41, 47); font-family: -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, Helvetica, Arial, sans-serif, &quot;Apple Color Emoji&quot;, &quot;Segoe UI Emoji&quot;; font-size: 16px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;"><pre style="box-sizing: border-box; font-family: ui-monospace, SFMono-Regular, &quot;SF Mono&quot;, Menlo, Consolas, &quot;Liberation Mono&quot;, monospace; font-size: 13.6px; margin-top: 0px; margin-bottom: 0px; overflow-wrap: normal; padding: 16px; overflow: auto; line-height: 1.45; background-color: var(--color-canvas-subtle); border-radius: 6px; word-break: normal;">npm install clipboardy</pre></div><h2 dir="auto" style="box-sizing: border-box; margin-top: 24px; margin-bottom: 16px; font-size: 1.5em; font-weight: 600; line-height: 1.25; padding-bottom: 0.3em; border-bottom: 1px solid var(--color-border-muted); color: rgb(36, 41, 47); font-family: -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, Helvetica, Arial, sans-serif, &quot;Apple Color Emoji&quot;, &quot;Segoe UI Emoji&quot;; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;"><a id="user-content-usage" class="anchor" aria-hidden="true" href="https://github.com/sindresorhus/clipboardy#usage" style="box-sizing: border-box; background-color: transparent; color: var(--color-accent-fg); text-decoration: none; transition: color 80ms cubic-bezier(0.33, 1, 0.68, 1) 0s, background-color, box-shadow, border-color; float: left; padding-right: 4px; margin-left: -20px; line-height: 1;"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path fill-rule="evenodd" d="M7.775 3.275a.75.75 0 001.06 1.06l1.25-1.25a2 2 0 112.83 2.83l-2.5 2.5a2 2 0 01-2.83 0 .75.75 0 00-1.06 1.06 3.5 3.5 0 004.95 0l2.5-2.5a3.5 3.5 0 00-4.95-4.95l-1.25 1.25zm-4.69 9.64a2 2 0 010-2.83l2.5-2.5a2 2 0 012.83 0 .75.75 0 001.06-1.06 3.5 3.5 0 00-4.95 0l-2.5 2.5a3.5 3.5 0 004.95 4.95l1.25-1.25a.75.75 0 00-1.06-1.06l-1.25 1.25a2 2 0 01-2.83 0z"></path></svg></a></h2><br class="Apple-interchange-newline">
  //   `;

  const markdown = new TurndownService({
    headingStyle: 'atx',
    codeBlockStyle: 'fenced',
    // fence: '```````',
  }).turndown(html);
  info(`markdown: ${markdown}`);

  // trim line
  const trimedMarkdown = markdown
    .split('\n')
    .map((line: string) => line.trim())
    .join('\n');

  const file = argv.file;
  const mdFile = path.join(__dirname, `../../docs/${file}.md`);
  fs.writeFileSync(mdFile, trimedMarkdown, 'utf-8');
  event(`✅ DONE`);
})();

translate.ts

import invariant from 'tiny-invariant';
import 'zx/globals';
import { event, info } from '../logger';

const cwd = process.cwd();
const API_URL = 'http://localhost:8080/translate';
const cacheFile = path.join(__dirname, 'translate.cache.json');
const cacheData = fs.existsSync(cacheFile)
  ? JSON.parse(fs.readFileSync(cacheFile, 'utf-8'))
  : {};
const cache = new Map(Object.entries(cacheData));

const saveCache = (key: string, value: string) => {
  cache.set(key, value);
  fs.writeFileSync(
    cacheFile,
    JSON.stringify(Object.fromEntries(cache), null, 2),
    'utf-8',
  );
};

const delay = (ms: number) => new Promise((res) => setTimeout(res, ms));

const translate = async (opts: {
  text: string;
  sourceLang?: string;
  targetLang?: string;
}): Promise<{ data: string; cache: boolean }> => {
  const { text, sourceLang = 'auto', targetLang = 'ZH' } = opts;
  invariant(text.length < 5000, `Too long... ${text.length}`);
  if (cache.has(text)) {
    info(`Cache hit.`);
    return {
      data: cache.get(text) as string,
      cache: true,
    };
  }
  const res = await fetch(API_URL, {
    method: 'POST',
    body: JSON.stringify({
      text,
      source_lang: sourceLang,
      target_lang: targetLang,
    }),
  });
  invariant(res.status === 200, `${res.status} ${res.statusText}`);
  const json = await res.json();
  const { data } = json;
  invariant(data, `data is null, ${JSON.stringify(json)}`);
  saveCache(text, data);
  return { data, cache: false };
};

(async () => {
  info(argv);
  const file = argv._[0];
  const absFile = path.isAbsolute(file) ? file : path.join(cwd, file);
  const SEPARATOR = '\n\n';
  const CHAR_COUNT_LIMIT = 4000;
  const lineMap = new Map<
    string,
    { type: 'CODE_BLOCK' | 'TEXT' | 'IMG'; text?: string }
  >();

  // read file
  invariant(fs.existsSync(absFile), `File not found: ${absFile}`);
  const text = fs.readFileSync(absFile, 'utf-8').trim();
  info(`text length: ${text.length}`);
  const lines = text.split(SEPARATOR);

  // split lines to blocks
  const blocks: string[][] = [];
  let currBlock = [];
  let currCharCount = 0;
  for (const line of lines) {
    let type: 'CODE_BLOCK' | 'TEXT' | 'IMG' = 'TEXT';
    if (line.startsWith('```')) {
      type = 'CODE_BLOCK';
    } else if (/^!\[(.+)?]\(.+?\)$/.test(line)) {
      type = 'IMG';
    }
    lineMap.set(line, { type });
    if (type !== 'TEXT') continue;
    const charCount = line.length;
    if (currCharCount + charCount > CHAR_COUNT_LIMIT) {
      blocks.push(currBlock);
      currBlock = [];
      currBlock.push(line);
      currCharCount = charCount;
    } else {
      currBlock.push(line);
      currCharCount += charCount;
    }
  }
  if (currBlock.length > 0) {
    blocks.push(currBlock);
  }
  info(`block length: ${blocks.length}`);

  // translate
  const blockLength = blocks.length;
  for (const [index, block] of blocks.entries()) {
    const text = block.join(SEPARATOR);
    info(`[${index + 1}/${blockLength}] Translating block...`);
    const { data: translatedText, cache } = await translate({
      text,
      sourceLang: argv.sourceLang,
      targetLang: argv.targetLang,
    });
    const translatedTextArr = translatedText.split(SEPARATOR);
    invariant(
      block.length === translatedTextArr.length,
      `translated length not match`,
    );
    for (let i = 0; i < block.length; i++) {
      const line = block[i];
      const translatedLine = translatedTextArr[i];
      invariant(lineMap.has(line), `line not found in map: ${line}`);
      lineMap.get(line)!.text = translatedLine;
    }
    if (!cache && index < blockLength - 1) {
      const delayTime = 1000 + Math.floor(Math.random() * 2000);
      info(`delay ${delayTime}ms...`);
      await delay(delayTime);
    }
  }

  // merge
  const mergedArr = [];
  for (const [line, { type, text }] of lineMap.entries()) {
    if (type === 'TEXT') {
      mergedArr.push(line);
      mergedArr.push(text);
    } else {
      mergedArr.push(line);
    }
  }
  info(`Merged.`);

  // write new file
  const absNewFile = absFile.replace(/\.md/, '.translated.md');
  fs.writeFileSync(absNewFile, mergedArr.join(SEPARATOR), 'utf-8');
  event(`Translated to ${absNewFile}`);
})();