A class that provides translation capabilities with support for multiple sources, such as file-based translations and database translations.

MultiSourceTranslator.ts

/**
 * MultiSourceTranslator is a class that provides translation capabilities with support for multiple sources, such as
 * file-based translations and database translations. It allows you to load translations for multiple languages and
 * supports string interpolation using placeholders.
 *
 * @class     MultiSourceTranslator
 * @author    Mohamed Riyad <m@ryad.dev>
 * @link      https://RyadPasha.com
 * @copyright Copyright (C) 2023 RyadPasha. All rights reserved.
 * @license   MIT
 * @version   1.1.8-2023.12.15
 * @see       {@link https://gist.github.com/RyadPasha/ae849c75130b295da0801b0a9099586d} for updates
 */

import * as fs from 'fs'
import { CronJob } from 'cron'
import mysql, { Pool } from 'mysql2'
import logger from './logger'

// Table structure for the translations table
// CREATE TABLE `translations` (
//   `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
//   `language` varchar(6) NOT NULL,
//   `t_key` varchar(255) NOT NULL,
//   `t_value` text NOT NULL,
//   PRIMARY KEY (`id`),
//   UNIQUE KEY `lang_key` (`language`,`t_key`)
// ) ENGINE=InnoDB DEFAULT CHARSET=utf8;

interface TranslationRecord {
    id?: number
    language: string
    t_key: string
    t_value: string
}

interface Translation {
    [key: string]: string | Translation
}

interface MultiSourceTranslatorOptions {
    filePath: string // Path to the directory containing translation files
    defaultLanguage?: string // Default language code
    loadFileTranslations?: boolean // Load translations from files
    loadDbTranslations?: boolean // Load translations from the database
    syncTranslationsToDatabase?: boolean // Sync the file translations to the database (WARNING: This should not be used in production)
    truncateTranslationsTableBeforeSync?: boolean // Truncate the 'translations' table before synchronizing translations
    logsEnabled?: boolean // Enable/disable logging
    refreshCronJob?: string // Cron job to periodically refresh translations
    saveMissingTranslations?: boolean // Save missing translations to a JSON file
    checkConsistency?: boolean // Check if translations are consistent across languages
}

export class MultiSourceTranslator {
    private fileTranslations: { [language: string]: Translation } = {}
    private dbTranslations: { [language: string]: Translation } = {}
    private fileLoaded: boolean = false
    private dbLoaded: boolean = false
    private allLanguageCodes: string[] = []
    private readonly pool: Pool
    private isInitiated: boolean = false

    /**
     * Construct a MultiSourceTranslator instance.
     *
     * @param options - The configuration options.
     * @param dbConfigOrPool - The MySQL connection pool or configuration object.
     */
    constructor(private options: MultiSourceTranslatorOptions, dbConfigOrPool?: Pool | object) {
        if (!options) {
            throw new Error('Missing required argument "options".')
        } else if (!options.filePath) {
            throw new Error('Missing required option "filePath".')
        }

        // Set default values if not provided in options
        options.defaultLanguage = options.defaultLanguage || 'en'
        options.loadFileTranslations = options.loadFileTranslations !== undefined ? options.loadFileTranslations : true
        options.loadDbTranslations = options.loadDbTranslations !== undefined ? options.loadDbTranslations : true
        options.syncTranslationsToDatabase = options.syncTranslationsToDatabase !== undefined ? options.syncTranslationsToDatabase : false
        options.truncateTranslationsTableBeforeSync = options.truncateTranslationsTableBeforeSync !== undefined ? options.truncateTranslationsTableBeforeSync : false
        options.logsEnabled = options.logsEnabled !== undefined ? options.logsEnabled : false
        options.saveMissingTranslations = options.saveMissingTranslations !== undefined ? options.saveMissingTranslations : false
        options.checkConsistency = options.checkConsistency !== undefined ? options.checkConsistency : false

        if (dbConfigOrPool) {
            if ('createPool' in dbConfigOrPool) {
                /**
                 * The MySQL connection pool.
                 *
                 * @type {Pool}
                 */
                this.pool = dbConfigOrPool as Pool
            } else if (typeof dbConfigOrPool === 'object' && !Array.isArray(dbConfigOrPool)) {
                /**
                 * The MySQL connection pool.
                 *
                 * @type {Pool}
                 */
                this.pool = mysql.createPool(dbConfigOrPool as object)
            } else {
                throw new Error('Invalid argument type for dbConfigOrPool. Expected an object or a mysql.Pool instance.')
            }

            if (options.filePath && options.syncTranslationsToDatabase) {
                this.log('WARNING: Translations will be synchronized to the database. This will overwrite any existing translations in the database.')
                this.log('WARNING: This should not be used in production.')
                this.syncTranslationsToDatabase(options.filePath, options.truncateTranslationsTableBeforeSync).then(() => {
                    this.log('Translations have been synchronized to the database.')
                })
            }
        }
    }

    /**
     * Initialize the translator by loading translations from file and/or database based on configuration.
     *
     * @async
     */
    async init(): Promise<void> {
        if (this.isInitiated) return
        await this.loadTranslations()
        this.isInitiated = true

        if (this.options.checkConsistency) {
            this.compareFieldsAcrossLanguages(this.fileTranslations)
        }

        if (this.options.refreshCronJob) {
            // Schedule the refresh cron job to periodically update translations
            new CronJob(this.options.refreshCronJob, () => {
                this.log('Refreshing translations...')
                this.loadTranslations()
            }).start()
        }
    }

    /**
     * Log the given arguments.
     *
     * @param args
     */
    log(...args: any[]) {
        if (this.options.logsEnabled) console.log(...args)
    }

    /**
     * Load translations from file and/or database, based on the configuration.
     *
     * @private
     * @async
     */
    private async loadTranslations(): Promise<void> {
        if (this.options.loadFileTranslations && !this.fileLoaded) {
            this.loadFileTranslationsFromDirectory()
        }

        if (this.options.loadDbTranslations && !this.dbLoaded) {
            await this.loadDbTranslationsFromDatabase()
        }

        // Combine language codes from 'fileTranslations' and 'dbTranslations', remove duplicates using a Set,
        // and store the unique language codes in 'allLanguageCodes' array.
        this.allLanguageCodes = [...new Set([...Object.keys(this.fileTranslations), ...Object.keys(this.dbTranslations)])]
    }

    /**
     * Load translations from files in the specified directory and detects available languages.
     *
     * @private
     */
    private loadFileTranslationsFromDirectory() {
        try {
            this.log(`Loading translations from files in "${this.options.filePath}"...`)
            const files = fs.readdirSync(this.options.filePath)

            for (const file of files) {
                if (file.endsWith('.json')) {
                    const language = file.replace('.json', '')

                    if (language.endsWith('.missing')) {
                        // this.log(`Skipping missing translations file "${file}".`)
                        continue
                    }

                    this.log(`Loading translations from file "${file}" ...`)

                    if (language.length !== 2) {
                        this.log(`Invalid language code "${language}". Skipping file "${file}".`)
                        continue
                    }

                    const data = fs.readFileSync(`${this.options.filePath}/${file}`, 'utf-8')
                    const translations = JSON.parse(data)

                    // Recursively flatten and store translations in the desired format
                    const flattenTranslations = (currentObj, currentKey = '') => {
                        for (const key in currentObj) {
                            const newKey = currentKey ? `${currentKey}.${key}` : key
                            if (typeof currentObj[key] === 'object') {
                                flattenTranslations(currentObj[key], newKey)
                            } else {
                                this.fileTranslations[language] = this.fileTranslations[language] || {}
                                this.fileTranslations[language][newKey] = currentObj[key]
                            }
                        }
                    }

                    flattenTranslations(translations)
                }
            }

            this.log('Translations have been loaded from files.')
            this.fileLoaded = true
        } catch (error) {
            this.log('Error loading translations from files:', error)
        }
    }

    /**
     * Load translations from the database and detects available languages.
     *
     * @private
     * @async
     */
    private async loadDbTranslationsFromDatabase(): Promise<void> {
        try {
            if (!this.pool) {
                console.info('MySQL connection pool is not initialized.')
                return
            }

            const [result] = await this.pool.promise().execute('SELECT language, t_key, t_value FROM translations')
            const rows = result as TranslationRecord[]

            for (const row of rows) {
                const language = row.language
                if (!this.dbTranslations[language]) {
                    this.dbTranslations[language] = {}
                }

                this.dbTranslations[language][row.t_key] = row.t_value
            }

            this.dbLoaded = true
        } catch (error) {
            this.log('Error loading translations from the database:', error)
        }
    }

    /**
     * Translate a given key in the specified language with optional string interpolation using placeholders.
     *
     * @param key - The translation key.
     * @param placeholders - Optional placeholders for string interpolation.
     * @param language - The language for which the translation is needed.
     * @returns A translated string.
     */
    translate(key: string, placeholders?: Record<string, string>, language?: string): string {
        if (!language) language = this.options.defaultLanguage
        if (!placeholders) placeholders = {}

        // Normalize the provided language to a standard format (e.g., "en-US" or "en")
        language = this.normalizeLanguage(language)

        // Load translations from the file in case the class was not initialized (init() is not called)
        if (this.options.loadFileTranslations && !this.fileLoaded) {
            this.loadFileTranslationsFromDirectory()
        }

        // if (this.options.loadDbTranslations && !this.dbLoaded) {
        //     await this.loadDbTranslationsFromDatabase()
        // }

        const dbTranslation = this.findTranslation(this.dbTranslations[language], key)

        if (dbTranslation !== undefined) {
            return this.interpolate(dbTranslation, placeholders)
        }

        const fileTranslation = this.findTranslation(this.fileTranslations[language], key)

        if (fileTranslation !== undefined) {
            return this.interpolate(fileTranslation, placeholders)
        }

        // Fallback to the default language if available
        if (language !== this.options.defaultLanguage) {
            this.log(`Translation for "${key}" in "${language}" not found. Fallback to default language "${this.options.defaultLanguage}".`)

            const defaultDbTranslation = this.findTranslation(this.dbTranslations[this.options.defaultLanguage], key)
            if (defaultDbTranslation !== undefined) {
                return this.interpolate(defaultDbTranslation, placeholders)
            }

            const defaultFileTranslation = this.findTranslation(this.fileTranslations[this.options.defaultLanguage], key)
            if (defaultFileTranslation !== undefined) {
                return this.interpolate(defaultFileTranslation, placeholders)
            }
        }

        // Fallback to the default value if available
        if (placeholders['defaultValue']) {
            return this.interpolate(placeholders['defaultValue'], placeholders)
        }

        // Otherwise, return the key itself
        const lastKey = key.lastIndexOf('.')
        const defaultValue = key.substring(lastKey + 1)

        if (this.options.saveMissingTranslations) {
            this.saveMissingTranslation(key, language, defaultValue)
        }

        const keyBasedTranslation = this.interpolate(key, placeholders)
        return keyBasedTranslation === key ? defaultValue : keyBasedTranslation
    }

    /**
     * Common function to process translation arguments.
     *
     * @param args - Translation arguments.
     * @returns Object containing 'key', 'placeholders' and 'defaultValue'.
     */
    processTranslationArgs(...args: [key: string, placeholders?: Record<string, string> | { defaultValue: string }] | [key: string, defaultValue?: string, placeholders?: Record<string, string>]) {
        let key, defaultValue, placeholders

        if (args.length === 1) {
            ;[key] = args
        } else if (args.length === 2) {
            let arg2
            ;[key, arg2] = args

            if (typeof arg2 === 'string') {
                defaultValue = arg2
            } else {
                if (arg2 && arg2.defaultValue) {
                    defaultValue = arg2.defaultValue
                }
                placeholders = arg2
            }
        } else if (args.length === 3) {
            ;[key, defaultValue, placeholders] = args
        } else {
            throw new Error('Invalid number of arguments')
        }

        if (defaultValue) {
            placeholders = placeholders || {}
            placeholders['defaultValue'] = defaultValue
        }

        return { key, placeholders, defaultValue }
    }

    /**
     * Translate a given key in the specified language with optional string interpolation using placeholders.
     *
     * @param language - The language for which the translation is needed.
     * @param args     - The translation key, optional default value, and optional placeholders for string interpolation.
     * @returns A translated string.
     */
    t_(
        language = null,
        ...args: [key: string, placeholders?: Record<string, string> | { defaultValue: string }] | [key: string, defaultValue?: string, placeholders?: Record<string, string>]
    ): string {
        const { key, placeholders, defaultValue } = this.processTranslationArgs(...args)

        if (!key) return defaultValue

        // Call the translate function with the given language, if any
        return this.translate(key, placeholders, language)
    }

    /**
     * Translate a message for multiple languages and return translations in an object.
     *
     * @param args - The translation key, optional default value, and optional placeholders for string interpolation.
     * @returns Translations for all languages.
     */
    t__(
        ...args: [key: string, placeholders?: Record<string, string> | { defaultValue: string }] | [key: string, defaultValue?: string, placeholders?: Record<string, string>]
    ): Record<string, string> {
        const { key, placeholders, defaultValue } = this.processTranslationArgs(...args)

        const translations: Record<string, string> = {}

        for (const language of this.allLanguageCodes) {
            if (!key) {
                translations[language] = defaultValue
            } else {
                const translation = this.translate(key, placeholders, language)

                if (translation) {
                    translations[language] = translation
                }
            }
        }

        return translations
    }

    /**
     * Recursively searches for a translation key in a translation object.
     *
     * @private
     * @param translations - The translation object.
     * @param key - The translation key (e.g., "common.greeting.hello").
     * @returns The translated string or undefined if the key is not found.
     */
    private findTranslation(translations: Translation, key: string): string | undefined {
        if (!translations) return undefined

        if (key in translations) {
            return translations[key] as string
        }
    }

    /**
     * Interpolate placeholders in a translation string using the provided placeholders.
     *
     * @private
     * @param translation - The translation string containing placeholders.
     * @param placeholders - A dictionary of placeholders and their corresponding values.
     * @returns The interpolated translation string.
     */
    private interpolate(translation: string, placeholders: Record<string, string>): string {
        return translation.replace(/{{(\w+)}}/g, (match, placeholder) => {
            return placeholders[placeholder] || match
        })
    }

    /**
     * Normalize the provided language code to a standard format (e.g., "en-US" or "en").
     *
     * @private
     * @param language - Language code to normalize.
     * @returns The normalized language code.
     */
    private normalizeLanguage(language: string): string {
        return (language + '').substring(0, 2).toLowerCase()
    }

    /**
     * Recursively merge two objects, where object properties are deep-merged.
     *
     * @param {object} obj1 - The first object to merge.
     * @param {object} obj2 - The second object to merge.
     * @returns {object} - The merged object.
     */
    mergeObjects(obj1: object, obj2: object): object {
        for (const key in obj2) {
            if (obj2.hasOwnProperty(key)) {
                if (obj1.hasOwnProperty(key) && typeof obj1[key] === 'object' && typeof obj2[key] === 'object') {
                    obj1[key] = this.mergeObjects(obj1[key], obj2[key])
                } else {
                    obj1[key] = obj2[key]
                }
            }
        }

        return obj1
    }

    /**
     * Convert a dot-separated string into an object structure.
     *
     * @param str - The dot-separated string representing the object structure.
     * @param value - The value to assign to the innermost property.
     * @returns An object with the specified structure.
     */
    createObjectFromString(str: string, value: any): object {
        // Split the input string by dots and reverse the array to process keys in reverse order.
        const keys = str.split('.') //.reverse()
        let result = value

        // Build the object structure from innermost property to outermost property.
        while (keys.length) {
            const key = keys.pop()
            result = { [key]: result }
        }

        return result
    }

    /**
     * Save missing translations to a JSON file with the specified language code.
     *
     * @param key The translation key.
     * @param language The language code for which to save missing translations.
     * @param value The translation value.
     */
    saveMissingTranslation(key: string, language: string, value: string = 'MISSING_TRANSLATION'): void {
        const filePath = `${this.options.filePath}/${language}.missing.json`

        // Merge your object into the existing data
        const objectFromStr = this.createObjectFromString(key, value)

        // Initialize an empty object as the default data
        let existingData = {}

        try {
            // Attempt to read the existing JSON data from the file
            const existingDataString = fs.readFileSync(filePath, 'utf8')
            existingData = JSON.parse(existingDataString)
        } catch (err) {
            if (err.code === 'ENOENT') {
                // Handle the case where the file does not exist
                this.log('File not found. Creating a new one.')
            } else {
                // Handle other file read errors
                this.log('Error reading the file:', err)
            }
        }

        try {
            const mergedData = this.mergeObjects(existingData, objectFromStr)

            // Write the merged data back to the JSON file
            fs.writeFileSync(filePath, JSON.stringify(mergedData, null, 2), 'utf8')

            this.log(`Missing translations for "${key}" in "${language}" have been saved to "${filePath}".`)
        } catch (error) {
            this.log('Error saving missing translations:', error)
        }
    }

    /**
     * Synchronize translations from translation files to the database.
     *
     * @async
     * @param filePath - The path to the directory containing translation files.
     * @param truncateTable - Whether to truncate the 'translations' table before synchronizing translations. Defaults to `false`.
     */
    async syncTranslationsToDatabase(filePath: string, truncateTable: boolean = false): Promise<void> {
        try {
            const files = fs.readdirSync(filePath)

            if (truncateTable) {
                await this.pool.promise().execute('TRUNCATE TABLE translations')
            }

            for (const file of files) {
                if (file.endsWith('.json')) {
                    const language = file.replace('.json', '')

                    if (language.endsWith('.missing')) {
                        // this.log(`Skipping missing translations file "${file}".`)
                        continue
                    }

                    if (language.length !== 2) {
                        this.log(`Invalid language code "${language}". Skipping file "${file}".`)
                        continue
                    }

                    const data = fs.readFileSync(`${filePath}/${file}`, 'utf-8')
                    const translations = JSON.parse(data)
                    const updatedTranslations = {}

                    // Recursively flatten and store translations in the desired format
                    const flattenTranslations = (currentObj, currentKey = '') => {
                        for (const key in currentObj) {
                            const newKey = currentKey ? `${currentKey}.${key}` : key
                            if (typeof currentObj[key] === 'object') {
                                flattenTranslations(currentObj[key], newKey)
                            } else {
                                updatedTranslations[language] = updatedTranslations[language] || {}
                                updatedTranslations[language][newKey] = currentObj[key]
                            }
                        }
                    }

                    flattenTranslations(translations)

                    for (const language in updatedTranslations) {
                        for (const key in updatedTranslations[language]) {
                            await this.insertOrUpdateTranslation(language, key, updatedTranslations[language][key])
                        }
                    }
                }
            }
        } catch (error) {
            this.log('Error syncing translations to the database:', error)
        }
    }

    /**
     * Insert or update a translation in the database.
     *
     * @async
     * @private
     * @param language - The language of the translation.
     * @param key - The translation key.
     * @param value - The translation value.
     */
    private async insertOrUpdateTranslation(language: string, key: string, value: string): Promise<void> {
        try {
            await this.pool.promise().execute('INSERT INTO translations (language, t_key, t_value) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE t_value = VALUES(t_value)', [language, key, value])
        } catch (error) {
            this.log('Error inserting/updating translation:', error)
        }
    }

    /**
     * Compare translations and identifies the keys that are missing in certain languages.
     *
     * @param translations - The translations object containing language-specific key-value pairs.
     * @returns An array of strings indicating the differences in translations.
     */
    compareFieldsAcrossLanguages(translations: { [language: string]: Translation }): string[] {
        const keys = new Set<string>()

        // Collect all keys from all languages
        for (const lang in translations) {
            for (const key in translations[lang]) {
                keys.add(key)
            }
        }

        const differences: string[] = []

        // Check each key for availability in each language
        keys.forEach((key) => {
            const availableIn: string[] = []

            for (const lang in translations) {
                if (translations[lang][key]) {
                    availableIn.push(lang)
                }
            }

            // If a key is missing in a language, add it to differences
            const missingIn = Object.keys(translations).filter((lang) => !availableIn.includes(lang))
            if (missingIn.length > 0) {
                differences.push(`'${key}' is available in '${availableIn.join(`' and '`)}' but not in '${missingIn.join(`' or '`)}'`)
            }
        })

        if (differences.length > 0) {
            logger.warn('Translations are not consistent across languages:')
            differences.forEach((diff) => logger.warn(diff))
        }

        return differences
    }
}

// Example usage
const dbConfig = {
    host: 'host',
    user: 'user',
    password: 'password',
    database: 'database',
    charset: 'utf8mb4'
}

const translator = new MultiSourceTranslator(
    {
        filePath: './src/translations',
        defaultLanguage: 'en',
        loadFileTranslations: true,
        loadDbTranslations: true,
        syncTranslationsToDatabase: true, // WARNING: This should not be used in production
        truncateTranslationsTableBeforeSync: false, // WARNING: This should not be used in production
        checkConsistency: process.env.NODE_ENV === 'development', // WARNING: This should not be used in production
        refreshCronJob: process.env.REFRESH_CRON_JOB
    },
    dbConfig
)

translator.init().then(async () => {
    const translation = await translator.translate('common.greeting.hello', 'en', {
        name: 'Mohamed'
    })

    console.log(translation)
})