Code to add a custom button to the TinyMCE editor (needs WordPress 4.8)

custom-tinymce-button.js

/**
 * Listen to setup of a TinyMCE instance.
 */
jQuery(document).on('tinymce-editor-setup', function (event, editor) {
    /**
     * Add the id of the soon-to-create button to the
     * toolbar1.
     *
     * @type {string}
     */
    editor.settings.toolbar1 += ',schlicht-formats';
 
    /**
     * Create the button.
     */
    editor.addButton('schlicht-formats', {
        /**
         * It is a menubutton.
         */
        type: 'menubutton',
        text: '»Schlicht« Formats',
        icon: false,
 
        /**
         * Add the menu items.
         */
        menu: [
            {
                text: 'Side note',
                onclick: function () {
                    /**
                     * Toggle the custom side note format.
                     */
                    editor.formatter.toggle('schlicht_side_note_format');
                }
            }
        ]
    });
 
    /**
     * Wait until the editor is initialized before
     * registering style.
     *
     * Original hint from http://archive.tinymce.com/forum/viewtopic.php?pid=92762#p92762
     * onInit is deprecated, so https://www.tinymce.com/docs/advanced/migration-guide-from-3.x/#eventhandling
     */
    editor.on('init', function () {
        /**
         * Register the new format for the side note.
         */
        editor.formatter.register('schlicht_side_note_format', {
            /**
             * It is the block element div.
             */
            block: 'div',
 
            /**
             * The div element gets the class side-note.
             */
            classes: 'side-note',
 
            /**
             * The div is the wrapper for other block elements
             * and does not replace them.
             */
            wrapper: true
        });
    });
});

More information about that (in German): https://krautpress.de/2017/editor-um-button-erweitern/