Wordpress TinyMCE Popup-Dialog. German Tutorial on http://dnaber.de/p1477

editor_plugin.js

// js/editor_plugin.js
/**
 * an example tinyMCE Plugin
 */
tinymce.create( 
  'tinymce.plugins.myPlugin', 
  {
    /**
     * @param tinymce.Editor editor
     * @param string url
     */
    init : function( editor, url ) {
      /**
       *  register a new button
       */
      editor.addButton(
        'my_plugin_button', 
        {
          cmd   : 'my_plugin_button_cmd',
          title : editor.getLang( 'myPlugin.buttonTitle', 'My Button' ),
          image : url + '/img/example.gif'
        }
      );
      /**
       * and a new command
       */
      editor.addCommand(
        'my_plugin_cmd', 
        function() {
         /**
          * @link http://www.tinymce.com/wiki.php/API3:method.tinymce.WindowManager.open
          * @param Object Popup settings
          * @param Object Arguments to pass to the Popup
          */
          editor.windowManager.open(
            {
              file   : url + '/dialog.htm',
              width  : 320,
              height : 120,
              inline : 1
            }, 
            {
              plugin_url      : url,
              some_custom_arg : 'custom value'
            }
          );
        }
      );
    }
  }
  // register plugin
  tinymce.PluginManager.add( 'myPlugin', tinymce.plugins.myPlugin );
);

editor_plugin_part.js

// js/editor_plugin.js
/**
 * somewhere in the init-method
 * of the parameter object at 
 * tinymce.create()...
 * 
 * this requires the mce plugin WPDialog
 */
editor.addCommand(
  'my_plugin_button_cmd',
  function() {
    /**
     * @param Object Popup settings
     * @param Object Arguments to pass to the Popup
     */
    editor.windowManager.open(
      {
        // this is the ID of the popups parent element
        id       : 'my_plugin_dialog',
        width    : 480,
        title    : editor.getLang( 'myPlugin.popupTitle', 'Default Title' )
        height   : 'auto',
        wpDialog : true,
      },
      {
        plugin_url : url
      }
    );
  }
);

hook_in_dialog.php

<?php
# somewhere in my_plugin.php
if ( is_admin() ) {
  /**
   * hook in only, if current user can 
   * see the editor and want to have 
   * a rich text editor
   */
  if ( 
       ( current_user_can( 'edit_posts' ) 
       || current_user_can( 'edit_pages' ) ) 
      && 'true' == get_user_option( 'rich_editing' ) 
  ) {
    add_action( 'wp_footer', 'my_plugin_dialog' );
  }
}
function my_plugin_dialog() {
  ?>
  <div style="display:none;">
    <form id="my_plugin_dialog" tabindex="-1" action="">
      <div>
        <input type="text" name="foo" />
      </div>
      <div>
        <input type="submit" class="button-primary" value="Go" />
      </div>
    </form>
  </div>
  <?php
}

mce_hook_in.php

<?php
# somewhere in my_plugin.php

add_filter( 'mce_external_plugins', 'register_my_tinymce_plugin' );
add_filter( 'mce_buttons', 'register_my_tinymce_button' );

/**
 * register_my_tinymce_plugin
 *
 * @param array $mce_plugins
 * @return array $mce_plugins
 */
function register_my_tinymce_plugin( $mce_plugins ) {
  $mce_plugins[ 'myPlugin' ] = plugins_url() . '/my-plugin/js/editor_plugin.js';
  return $mce_plugins;
}

/**
 * register_my_tinymce_button
 *
 * @param array $buttons
 * @return array $buttons
 */
function register_my_tinymce_button( $buttons ) {
  array_push( $buttons, '|', 'my_plugin_button' );
  return $buttons;
}

mce_i18n.php

<?php
# somewhere in my_plugin.php
# remember to load the plugin textdomain

add_filter( 'mce_external_languages', 'my_mce_localisation' );

/**
 * my_mce_localisation
 *
 * @see wp-admin/includes/post.php line 1474
 * @param array $mce_external_languages
 * @return array $mce_external_languages
 */
 function my_mce_localisation( $mce_external_languages ) {
  $mce_external_languages[ 'myPlugin' ] = plugin_dir_path( __FILE__ ) . 'i18n/mce_locale.php';
  return $mce_external_languages;
}

mce_locale.php

<?php
# i18n/mce_locale.php
/**
* @var string $strings a JavaScript snippet to add another language pack to TinyMCE
* @var string $mce_locale an ISO 639-1 formated string of the current language e.g. en, de...
* @deprecated wp_tiny_mce() at wp-admin/includes/post.php (for versions prior WP 3.3)
* @see _WP_Editors::editor_settings in wp-includes/class-wp-editor.php
*/
$strings =
  'tinyMCE.addI18n( 
    "' . $mce_locale .'.myPlugin", 
    {
      buttonTitle : "' . esc_js( __( 'My Button Title:', 'my_textdomain' ) ) . '",
      popupTitle  : "' . esc_js( __( 'My Popup Title', 'my_textdomain' ) ) . '",
    } 
  );';