button styling.css

span.InsertCustomSymbols
{
  background-image:url('/path/to/icon.gif') !important;
}

Custom Drop Field in RTE.cs

using Sitecore;
using Sitecore.Diagnostics;
using Sitecore.Resources;
using Sitecore.Web.UI;
using Telerik.Web.UI;
using Sitecore.Data.Items;

namespace RTEDropList
{
    public class EditorConfiguration : Sitecore.Shell.Controls.RichTextEditor.EditorConfiguration
    {
        public EditorConfiguration(Item profile) : base(profile)
        {
        }

        protected override void SetupToolbar(EditorToolGroup toolbar, Item item)
        {
            Assert.ArgumentNotNull((object)toolbar, "toolbar");
            Assert.ArgumentNotNull((object)item, "item");
            foreach (Sitecore.Data.Items.Item obj in item.Children)
            {
                string str1 = obj["Click"];
                if (!(str1 == "SpellCheck") && !(str1 == "AjaxSpellCheck") || this.SpellcheckEnabled)
                {
                    switch (obj.TemplateName)
                    {
                        case "Html Editor Divider":
                            EditorToolBaseCollection tools = toolbar.Tools;
                            EditorTool editorTool1 = new EditorTool();
                            editorTool1.Type = EditorToolType.Separator;
                            EditorTool editorTool2 = editorTool1;
                            tools.Add((EditorToolBase)editorTool2);
                            continue;
                        case "Html Editor Drop Down":
                            EditorTool editorTool3 = new EditorTool();
                            editorTool3.Type = EditorToolType.DropDown;
                            editorTool3.Name = str1;
                            editorTool3.ShowIcon = !MainUtil.GetBool(obj["Suppress Icon"], false);
                            EditorTool button1 = editorTool3;
                            this.SetProperties(button1, obj);
                            EditorConfiguration.SetSize(button1, obj);
                            toolbar.Tools.Add((EditorToolBase)button1);
                            continue;
                        case "Html Editor Drop Down Button":
                            EditorTool editorTool4 = new EditorTool(str1);
                            editorTool4.ShowIcon = true;
                            editorTool4.Type = EditorToolType.SplitButton;
                            EditorTool button2 = editorTool4;
                            this.SetProperties(button2, obj);
                            toolbar.Tools.Add((EditorToolBase)button2);
                            continue;
                        case "Html Editor Custom Drop Down":
                            EditorDropDown editorDropDown1 = new EditorDropDown();
                            editorDropDown1.Name = str1;
                            editorDropDown1.ShowIcon = !MainUtil.GetBool(obj["Suppress Icon"], false);
                            editorDropDown1.ShowText = true;
                            EditorDropDown editorDropDown2 = editorDropDown1;
                            this.SetProperties((EditorTool)editorDropDown2, obj);
                            EditorConfiguration.SetSize((EditorTool)editorDropDown2, obj);
                            EditorConfiguration.SetChildren(editorDropDown2.Items, obj);
                            toolbar.Tools.Add((EditorToolBase)editorDropDown2);
                            continue;
                        case "Html Editor Custom Drop Down Button":
                            EditorSplitButton editorSplitButton1 = new EditorSplitButton();
                            editorSplitButton1.Name = str1;
                            editorSplitButton1.ShowIcon = true;
                            editorSplitButton1.ItemsPerRow = "8";
                            EditorSplitButton editorSplitButton2 = editorSplitButton1;
                            this.SetProperties((EditorTool)editorSplitButton2, obj);
                            EditorConfiguration.SetChildren(editorSplitButton2.Items, obj);
                            toolbar.Tools.Add((EditorToolBase)editorSplitButton2);
                            continue;
                        default:
                            EditorTool editorTool5 = new EditorTool(str1);
                            if (!string.IsNullOrEmpty(obj[FieldIDs.DisplayName]))
                                editorTool5.Text = obj.DisplayName;
                            string image = obj[FieldIDs.Icon];
                            if (!string.IsNullOrEmpty(image))
                            {
                                string themedImageSource = Images.GetThemedImageSource(image, ImageDimension.id16x16);
                                this.Result.Styles.Append(".{EditorClass} .{ToolName}\r\n                      {\r\n                        background-image:url({IconUrl}) !important;\r\n                        background-position: 1px 1px;\r\n                      }".Replace("{EditorClass}", this.Editor.CssClass).Replace("{ToolName}", str1).Replace("{IconUrl}", themedImageSource));
                            }
                            editorTool5.ShowIcon = true;
                            string str2 = obj["Shortcut"];
                            if (!string.IsNullOrEmpty(str2))
                                editorTool5.ShortCut = str2;
                            toolbar.Tools.Add((EditorToolBase)editorTool5);
                            continue;
                    }
                }
            }
        }

        protected new Telerik.Web.UI.RadEditor Editor
        {
            get { return base.Editor; }
        }

        private static void SetChildren(EditorDropDownItemCollection items, Sitecore.Data.Items.Item parent)
        {
            Assert.ArgumentNotNull((object)items, "items");
            Assert.ArgumentNotNull((object)parent, "parent");
            foreach (Sitecore.Data.Items.Item obj in parent.Children)
            {
                if (obj.TemplateName == "Html Editor List Item")
                    items.Add(obj["Header"], obj["Value"]);
            }
        }

        private void SetProperties(EditorTool button, Sitecore.Data.Items.Item child)
        {
            Assert.ArgumentNotNull((object)button, "button");
            Assert.ArgumentNotNull((object)child, "child");
            string str = child["Shortcut"];
            if (!string.IsNullOrEmpty(str))
                button.ShortCut = str;
            if (!string.IsNullOrEmpty(child[FieldIDs.DisplayName]))
                button.Text = child.DisplayName;
            else if (string.IsNullOrEmpty(this.Editor.Localization.Tools.GetString(child["Click"])))
                button.Text = child.Name;
            if (child["Width"].Length > 0)
                button.PopUpWidth = child["Width"];
            if (child["Height"].Length <= 0)
                return;
            button.PopUpHeight = child["Height"];
        }

        private static void SetSize(EditorTool button, Sitecore.Data.Items.Item child)
        {
            Assert.ArgumentNotNull((object)button, "button");
            Assert.ArgumentNotNull((object)child, "child");
            if (child["Width"].Length > 0)
                button.PopUpWidth = child["Width"];
            if (child["Height"].Length <= 0)
                return;
            button.PopUpHeight = child["Height"];
        }
 
    }
}

Editor Command.js

RadEditorCommandList["InsertCustomSymbols"] = function(commandName, editor, args) {
    var val = args.get_value();
    editor.pasteHtml(val);
    args.set_cancel(true);
}

patch include.config

<clientscripts>
  <htmleditor>
    <script src="/location/to/custom.js" language="javascript" key="customJs" />
  </htmleditor>
</clientscripts>