Utility to rip emojis and generate sprite sheets and meta information for them.

drklo-emoji-ripper.js

/**
 * Emoji sprite and data generator.
 *
 * Data is taken from frankerfacez.com and name->char index is generated.
 *
 * Sprite generator works by parsing some of the code in DrKLO/Telegram (Telegram for Android)
 * and downloading emoji files contained there, while generating code and sprite.
 *
 * Can easily be modified to generate JSON instead of CSS or to download from some other source.
 * Can also be easily ported to TypeScript
 *
 * (c) teidesu 2020. This script is licensed under MIT
 */

const fetch = require('node-fetch')
const { createWriteStream, existsSync, readFileSync, writeFileSync, mkdirSync } = require('fs')
const { createCanvas, loadImage } = require('canvas')

function extractCodeBlock (str, start = 0) {
    let i = start
    const stack = []
    const closers = {
        ')': '(',
        ']': '[',
        '}': '{',
    }
    const openers = {
        '(': ')',
        '[': ']',
        '{': '}',
    }
    let ret = ''
    let inString = ''
    let inStringEscaped = false
    do {
        let chr = str[i]
        if (!inString) {
            if (chr === '\'' || chr === '"' || chr === '`') {
                inString = chr
            }
            if (openers[chr]) {
                stack.push(chr)
            } else if (closers[chr]) {
                if (stack[stack.length - 1] !== closers[chr]) {
                    throw TypeError('Malformed code, expected ' + closers[stack[stack.length - 1]]
                        + ' at position ' + i)
                }
                stack.pop()
            }
        } else {
            if (!inStringEscaped) {
                if (chr === inString) {
                    inString = false
                }
                if (chr === '\\') {
                    inStringEscaped = true
                }
            } else {
                inStringEscaped = false
            }
        }
        i++
        if (stack.length > 0) {
            ret += chr
        } else if (ret !== '') {
            ret += chr
            break // first block code ended
        }
    } while (i < str.length)
    if (stack.length > 0) {
        throw TypeError('Malformed code, expected ' + openers[stack.pop()] + ' at position ' + i)
    }
    return { block: ret, end: i }
}

const toCharCode = (s) => {
    let ret = []
    for (let i = 0; i < s.length; i++) {
        ret.push(s.charCodeAt(i).toString(16))
    }
    return ret.join('_')
}

async function createEmojisFile () {
    let data = await fetch('https://cdn.frankerfacez.com/static/emoji/v3.2.json').then(i => i.json())
    let result = {
        names: [],
        symbols: {},
    }

    for (let it of data.e) {
        let names = it[2]
        let value = String.fromCodePoint(...it[4].split('-').map(i => parseInt(i, 16)))

        if (!Array.isArray(names)) names = [names]
        names.forEach((s) => {
            if (!result.symbols[s]) {
                result.symbols[s] = toCharCode(value)
                result.names.push(s)
            }
        })
    }

    writeFileSync('emoji.json', JSON.stringify(result))
    console.log('[v] Written emoji.json (%d entries)', result.names.length)
}

const EMOJI_CATEGORIES = [
    'faces',
    'nature',
    'food',
    'activity',
    'transport',
    'objects',
    'symbols',
    'flags',
] // as const
// export type EmojiCategory = typeof EMOJI_CATEGORIES[number]
// type EmojiData = Record<EmojiCategory, string[]>

async function createEmojisDataFile () {
    let shittyDrkloJavaCode = await fetch(
        'https://raw.githubusercontent.com/DrKLO/Telegram/master/TMessagesProj/src/main/java/org/telegram/messenger/EmojiData.java',
    ).then(i => i.text())
    let dataBlockStart = shittyDrkloJavaCode.match(/public static final String\[]\[] data = {/)
    if (!dataBlockStart) throw new Error('could not find data block')

    let { block: dataBlock } = extractCodeBlock(
        shittyDrkloJavaCode,
        dataBlockStart.index + dataBlockStart[0].length - 1,
    )
    dataBlock = dataBlock.substring(1, dataBlock.length - 1).trim()
    let dataBlockPos = 0
    let data = []
    while (dataBlockPos < dataBlock.length) {
        try {
            let { block, end } = extractCodeBlock(dataBlock, dataBlockPos)
            dataBlockPos = end
            if (block === '[]') continue
            data.push(JSON.parse('[' + block.substring(1, block.length - 1) + ']'))
        } catch (e) {
            break
        }
    }

    let result = {}
    data.forEach((it, i) => result[EMOJI_CATEGORIES[i]] = it.map(toCharCode))

    writeFileSync('emoji-data.json', JSON.stringify(result))
}

async function downloadEmojiIfNeeded (name) {
    let path = `emoji/${name}.png`
    if (existsSync(path)) return loadImage(path)

    console.log('[i] downloading emoji %s', name)
    let output = createWriteStream(path)
    let res = await fetch(`https://raw.githubusercontent.com/DrKLO/Telegram/master/TMessagesProj/src/main/assets/emoji/${name}.png`)
    let pipe = res.body.pipe(output)

    await new Promise((res, rej) => {
        pipe.on('finish', res)
        pipe.on('error', rej)
    })

    return loadImage(path)
}

const SPRITE_SIZE = 20
const SPRITE_PER_ROW = 50
const SPRITE_MAX_X = SPRITE_PER_ROW - 1

async function createEmojisSpriteAndCss () {
    if (!existsSync('emoji')) mkdirSync('emoji')
    if (!existsSync('emoji-data.json')) {
        console.log('[i] generating emoji-data.json')
        await createEmojisDataFile()
    }
    let emojiData = JSON.parse(readFileSync('emoji-data.json').toString('utf-8'))

    let totalEmojiCount = Object.values(emojiData).reduce((a, b) => a + b.length, 0)

    let x = 0
    let y = 0
    let width = SPRITE_SIZE * SPRITE_PER_ROW
    let height = SPRITE_SIZE * Math.ceil(totalEmojiCount / SPRITE_PER_ROW)

    // modify here to change output format
    const outputCss = createWriteStream('emoji.css')
    outputCss.write(`
/* THIS FILE IS AUTO-GENERATED! */
.emojione {
  font-size: inherit;
  height: 20px;
  width: 20px;
  display: inline-block;
  line-height: normal;
  vertical-align: top;
  background-image: url(emoji.png);
  background-repeat: no-repeat;
  background-size: ${width}px ${height}px;
}
`.trim())

    const canvas = createCanvas(width, height)
    const ctx = canvas.getContext('2d')
    for (let categoryId = 0; categoryId < EMOJI_CATEGORIES.length; categoryId++) {
        const categoryName = EMOJI_CATEGORIES[categoryId]
        const categoryEmojis = emojiData[categoryName]

        for (let emojiId = 0; emojiId < categoryEmojis.length; emojiId++) {
            const emojiCharacter = categoryEmojis[emojiId]
            const fullEmojiId = `${categoryId}_${emojiId}`

            const image = await downloadEmojiIfNeeded(fullEmojiId)
            ctx.drawImage(image, SPRITE_SIZE * x, SPRITE_SIZE * y, SPRITE_SIZE, SPRITE_SIZE)
            // modify here to change output format
            outputCss.write(
                `.emojione-${emojiCharacter}{background-position:${-SPRITE_SIZE * x}px ${-SPRITE_SIZE * y}px}`,
            )

            x += 1
            if (x === SPRITE_MAX_X) {
                y += 1
                x = 0
            }
        }
    }

    const pngStream = canvas.createPNGStream()
    const outputSpriteSheet = createWriteStream('emoji.png')
    await new Promise(res => pngStream.pipe(outputSpriteSheet).on('finish', res))
    console.log('[v] generated emoji sprite sheet')
}

if (require.main === module) {
    createEmojisSpriteAndCss().catch(console.error)
}