Example plugin for how to add a button to tinymce in wordpress

editor_plugin.js

(function() 
{
	tinymce.create('tinymce.plugins.BrettsYouTube', 
	{
		init : function(ed, url) {
            ed.addButton('brettsyoutube', 
			{
				title : 'brettsyoutube.youtube',
				image : 'http://localhost/wordpress/wp-includes/js/tinymce/themes/advanced/img/icons.gif',
				onclick : function() 
				{
					idPattern = /(?:(?:[^v]+)+v.)?([^&=]{11})(?=&|$)/;
					var vidId = prompt("YouTube Video", "Enter the id or url for your video");
					var m = idPattern.exec(vidId);
					if (m != null && m != 'undefined')
						ed.execCommand('mceInsertContent', false, '[youtube id="'+m[1]+'"]');
				}
			});
		},
		createControl : function(n, cm) {
			return null;
		},
		getInfo : function() 
		{
			return 
			{
				longname : "Brett's YouTube Shortcode",
				author : 'Brett Terpstra',
				authorurl : 'http://brettterpstra.com/',
				infourl : 'http://brettterpstra.com/',
				version : "1.0"
			};
		}
	});
	tinymce.PluginManager.add( 'brettsyoutube', tinymce.plugins.BrettsYouTube );
})();

tinymce_add_button.php

<?php
/*
Plugin Name:	Add TinyMCE Button Example
Plugin URI:		http://example.com
Description:	Adds an examplary YouTube Button to the TinyMCE WYSIWG Editor
Author:			Franz Josef Kaiser
Author URI: 	http://example.com
Version:		0.1
License:		GPL v2 - http://www.gnu.org/licenses/old-licenses/gpl-2.0.html

	(c) Copyright 2012 - 2011 by Franz Josef Kaiser

	This program is free software; you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation; either version 2 of the License, or
	(at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with this program; if not, write to the Free Software
	Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA

@link http://wordpress.stackexchange.com/questions/15452/wysiwyg-clearboth/15454#15454
@link http://brettterpstra.com/adding-a-tinymce-button/
*/

/**
 * Enter description here ...
 * @param unknown_type $ver
 */
function wpse15452_script_refresh_mce( $ver ) 
{
  $ver += 3;
  return $ver;
}
add_filter( 'tiny_mce_version', 'wpse15452_script_refresh_mce' );

/**
 * add the shortcode handler for YouTube videos
 * 
 * @param unknown_type $atts
 * @param unknown_type $content
 */
function wpse15452_addYouTube( $atts, $content = null ) 
{
	extract( shortcode_atts( array( 'id' => '' ), $atts ) );
	$output = 
		'<p style="text-align:center"> \
		<a href="http://www.youtube.com/v/'.$id.'"> \
		<img src="http://img.youtube.com/vi/'.$id.'/0.jpg" width="400" height="300" class="aligncenter" /> \
		<span>Watch the video</span></a></p>';

	return $output;
}
add_shortcode( 'youtube', 'wpse15452_addYouTube' );

/**
 * Enter description here ...
 */
function wpse15452_add_youtube_button() 
{
	if ( ! current_user_can( 'edit_posts' ) && ! current_user_can( 'edit_pages' ) )
		return;

	if ( get_user_option( 'rich_editing' ) == 'true' ) 
	{
		add_filter( 'mce_external_plugins', 'wpse15452_add_youtube_tinymce_plugin' );
		add_filter( 'mce_buttons', 'wpse15452_register_youtube_button' );
	}
}
add_action( 'init', 'wpse15452_add_youtube_button' );

/**
 * Enter description here ...
 */
function wpse15452_register_youtube_button( $buttons ) 
{
	array_push( $buttons, '|', 'brettsyoutube' );
	return $buttons;
}

/**
 * Enter description here ...
 */
function wpse15452_add_youtube_tinymce_plugin( $plugin_array ) 
{
	$plugin_array[ 'youtube' ] = home_url().'/wp-content/plugins/tinymce-add-button/editor_plugin.js';
	return $plugin_array;
}

Copy into a folder named "tinymce-add-button" inside your wordpress plugins directory.