An example how one might add custom file browsing handler for CKEditor.

customBrowseHandler.js

CKEDITOR.replace( 'editor', {
	on: {
		// You need to listen to dialogShow event.
		dialogShow: function( evt ) {
			// Getting browse button object.
			var browseButton = evt.data.getContentElement( 'info', 'browse' );
			
			if ( browseButton ) {
				browseButton.on( 'click', function( evt ) {
					// Do your custom logic here, e.g. display a floating div with UI.
			
					evt.stop(); // Stop the event if you want nothing more to happen.
				}, null, null, 9 );
			}
		}
	}
} );