GridPaletteUtilityを使ってPaletteWindowにPaletteを新規追加する。

GridPaletteUtilityCreateNewPalette.cs

var type = System.Reflection.Assembly.Load( "UnityEditor.dll" ).GetType( "UnityEditor.GridPaletteUtility" );
var method_info = type.GetMethods (System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public );

System.Reflection.MethodInfo find_method = null;
foreach (var method in method_info) {
	if ( ( method.Name == "CreateNewPalette" ) && ( method.GetParameters().Length == 2 ) ) {
		find_method = method;
		break;
	}
}

if (find_method != null) {
	GameObject go = (GameObject)find_method.Invoke (null, new object[] { "Assets/NewPalette1.prefab", Grid.CellLayout.Rectangle });

	Debug.Log ( go );
}