X11 Custom Window Decorations

README.md

X11 Custom Window Decorations

To create an X window with custom decorations, you must remove the default decorations and emulate their behavior with your own UI. The example below shows a basic implementation of this.

Steps

1. Create a window without decorations:
   The most common approach is to create a frameless window using the _MOTIF_WM_HINTS property.
2. Emulate button behaviors:
   Most floating window managers have 3 buttons in each titlebar: "Close", "Maximize", and "Minimize".
   • "Close" can just be handled by breaking the main loop or sending a close event to your application.
   • "Maximize" is handled by sending a ClientMessage event to the root window. See ToggleMaximize().
   • "Minimize" is handled by calling XIconifyWindow().
3. Emulate dragging behaviors:
   Moving and resizing is done by sending a ClientMessage event of type _NET_WM_MOVERESIZE to the root window.
   • You only have to send this event once on mouse-down to start the move/resize.
   • This event takes the button that was pressed to cause it, so it will know to stop when that button is released.
   • Depending on the direction you supply, the event can initiate a resize in one of eight directions, and if the direction is 8, it will initiate a move.
   • The direction value is specified to start at 0 = top-left, and continue clockwise.
   • Before sending this event, you must call XUngrabPointer().
     See InitiateMoveResize() for more details.

Considerations

This is by no means a robust implementation of custom window decorations. The example only exists to demonstrate the minimum code required to get started.
For a more robust implementation, consider implementing these features:

• Make them optional:
  There are a number of desktop environments that don't fit into the three-button titlebar paradigm (e.g. tiling WMs), and you may want to accomodate this use case. This isn't difficult to implement, as you just have to not do the things demonstrated here.
• Smooth resizing:
  On many desktop environments, the resizing of this window will be laggy and jittery. This is because it doesn't use the SYNC extension. Conveniently, I have another example for that!
• Change the appearance of the maximize button when maximized.
• Remove the borders and resize margins when maximized.
• Change border appearance when window is not in focus.
• Make the resize margins bigger for diagonal sections.
• Make double-clicking the titlebar work like the maximize button.
• Check to make sure Extended Window Manager Hints (EWMH) are supported:
  Without these hints, _NET_WM_MOVERESIZE will not work properly. There are two things you could do in this situation:
  • Fallback implementation:
    You could have a special fallback implementation of the decorations that manually adjusts the window's position and size.
  • Use default decorations, as discussed earlier.
• OPTIONAL: Use _GTK_FRAME_EXTENTS:
  The _GTK_FRAME_EXTENTS property exists to solve two problems:
  • It allows you to have the resize margin extend past the window's boundaries without the weirdness.
  • It forcibly disables all drop shadows for applications that want to draw their own.
    Keep in mind that this isn't as well supported as EWMH, as it is technically a GTK-specific extension.

References

• https://specifications.freedesktop.org/wm-spec/wm-spec-1.3.html
• https://specifications.freedesktop.org/wm-spec/1.3/ar01s05.html

Code

Compile and try out this demo:
If you notice any unexpected behaviors on a floating window manager, please let me know!
Compile with: cc custom_decorations.c -lX11

custom_decorations.c

#include <stdint.h>
#include <stdbool.h>
#include <X11/Xlib.h>
#include <X11/cursorfont.h>

#define BORDER_SIZE 5
#define TITLEBAR_SIZE 30
#define BUTTON_SIZE 50

typedef struct Rect Rect;
struct Rect
{
  int x, y, w, h;
};

static inline bool PointInRect(int x, int y, Rect r);
static inline void ToggleMaximize(Display* display, Window window);
static inline int  ResizeDirectionFromPoint(int x, int y, int window_width, int window_height);
static inline void InitiateMoveResize(Display* display, Window window, int direction, XButtonEvent button_event);
static inline void DrawRect(Display* display, Window window, GC gc, Rect rect, XColor color);

Atom _NET_WM_STATE;
Atom _NET_WM_STATE_MAXIMIZED_HORZ;
Atom _NET_WM_STATE_MAXIMIZED_VERT;
Atom _NET_WM_MOVERESIZE;
Atom _MOTIF_WM_HINTS;

int main(int argument_count, char* arguments[])
{
  Display* display = XOpenDisplay(0);
  int screen = DefaultScreen(display);
  
  _NET_WM_STATE = XInternAtom(display, "_NET_WM_STATE", 0);
  _NET_WM_STATE_MAXIMIZED_HORZ = XInternAtom(display, "_NET_WM_STATE_MAXIMIZED_HORZ", 0);
  _NET_WM_STATE_MAXIMIZED_VERT = XInternAtom(display, "_NET_WM_STATE_MAXIMIZED_VERT", 0);
  _NET_WM_MOVERESIZE = XInternAtom(display, "_NET_WM_MOVERESIZE", 0);
  _MOTIF_WM_HINTS = XInternAtom(display, "_MOTIF_WM_HINTS", 0);
  
  Window window = XCreateWindow(display,
                                XDefaultRootWindow(display),
                                0, 0, 700, 700,
                                0,
                                CopyFromParent,
                                InputOutput,
                                CopyFromParent,
                                0,
                                0);
  XStoreName(display, window, "Custom Window Decoration Demo");
  
  // Remove default decorations
  long motif_hints[5] = {2, 0, 0, 0, 0};
  XChangeProperty(display, window, _MOTIF_WM_HINTS, _MOTIF_WM_HINTS, 32, PropModeReplace, (unsigned char*)motif_hints, 5);
  XMapWindow(display, window);
  
  XSelectInput(display, window, ExposureMask | PointerMotionMask | ButtonPressMask | ButtonReleaseMask);
  
  Cursor direction_cursors[9];
  direction_cursors[0] = XCreateFontCursor(display, XC_top_left_corner);
  direction_cursors[1] = XCreateFontCursor(display, XC_top_side);
  direction_cursors[2] = XCreateFontCursor(display, XC_top_right_corner);
  direction_cursors[3] = XCreateFontCursor(display, XC_right_side);
  direction_cursors[4] = XCreateFontCursor(display, XC_bottom_right_corner);
  direction_cursors[5] = XCreateFontCursor(display, XC_bottom_side);
  direction_cursors[6] = XCreateFontCursor(display, XC_bottom_left_corner);
  direction_cursors[7] = XCreateFontCursor(display, XC_left_side);
  direction_cursors[8] = XCreateFontCursor(display, XC_left_ptr);
  
  // begin: Graphics setup
  GC gc = XCreateGC(display, window, 0, 0);
  Colormap colormap = XDefaultColormap(display, screen);
  
  XColor red = {0};
  red.red = 65535;
  red.flags = DoRed;
  
  XColor green = {0};
  green.green = 65535;
  green.flags = DoGreen;
  
  XColor blue = {0};
  blue.blue = 65535;
  blue.flags = DoBlue;
  
  XColor yellow = {0};
  yellow.red = 65535;
  yellow.green = 57777;
  yellow.flags = DoRed | DoGreen;
  
  XColor gray = {0};
  gray.red = 32500;
  gray.green = 32500;
  gray.blue = 32500;
  gray.flags = DoRed | DoGreen | DoBlue;
  
  XColor dark_gray = {};
  dark_gray.red = 15000;
  dark_gray.green = 15000;
  dark_gray.blue = 15000;
  dark_gray.flags = DoRed | DoGreen | DoBlue;
  
  XAllocColor(display, colormap, &red);
  XAllocColor(display, colormap, &green);
  XAllocColor(display, colormap, &blue);
  XAllocColor(display, colormap, &yellow);
  XAllocColor(display, colormap, &gray);
  XAllocColor(display, colormap, &dark_gray);
  // end: Graphics setup
  
  Rect titlebar_rect = {0};
  Rect content_rect = {0};
  Rect close_rect = {0};
  Rect maximize_rect = {0};
  Rect minimize_rect = {0};
  
  int window_width;
  int window_height;
  bool running = true;
  bool needs_redraw = true;
  while(running)
  {
    XEvent event;
    while(XPending(display) > 0)
    {
      XNextEvent(display, &event);
      if(event.type == Expose) needs_redraw = true;
      if(event.type == ButtonPress)
      {
        if(event.xbutton.button = Button1)
        {
          int x = event.xbutton.x;
          int y = event.xbutton.y;
          
          if(PointInRect(x, y, close_rect))
          {
            running = false;
          }
          else if(PointInRect(x, y, maximize_rect))
          {
            ToggleMaximize(display, window);
          }
          else if(PointInRect(x, y, minimize_rect))
          {
            XIconifyWindow(display, window, 0);
          }
          else if(!PointInRect(x, y, content_rect))
          {
            InitiateMoveResize(display, window, ResizeDirectionFromPoint(x, y, window_width, window_height), event.xbutton);
          }
        }
      }
      else if(event.type == MotionNotify)
      {
        int resize_direction = ResizeDirectionFromPoint(event.xmotion.x, event.xmotion.y, window_width, window_height);
        
        XDefineCursor(display, window, direction_cursors[resize_direction]);
      }
    }
    
    if(needs_redraw)
    {
      needs_redraw = false;
      
      // Refresh layout
      XWindowAttributes window_attribs;
      XGetWindowAttributes(display, window, &window_attribs);
      
      window_width = window_attribs.width;
      window_height = window_attribs.height;
      
      titlebar_rect.x = BORDER_SIZE;
      titlebar_rect.y = BORDER_SIZE;
      titlebar_rect.w = window_width - BORDER_SIZE*2;
      titlebar_rect.h = TITLEBAR_SIZE;
      
      content_rect.x = BORDER_SIZE;
      content_rect.y = BORDER_SIZE + TITLEBAR_SIZE;
      content_rect.w = window_width - BORDER_SIZE*2;
      content_rect.h = window_height - TITLEBAR_SIZE - BORDER_SIZE*2;
      
      close_rect.x = window_width - BORDER_SIZE - BUTTON_SIZE;
      close_rect.y = BORDER_SIZE;
      close_rect.w = BUTTON_SIZE;
      close_rect.h = TITLEBAR_SIZE;
      
      maximize_rect.x = window_width - BORDER_SIZE - BUTTON_SIZE - BUTTON_SIZE;
      maximize_rect.y = BORDER_SIZE;
      maximize_rect.w = BUTTON_SIZE;
      maximize_rect.h = TITLEBAR_SIZE;
      
      minimize_rect.x = window_width - BORDER_SIZE - BUTTON_SIZE - BUTTON_SIZE*2;
      minimize_rect.y = BORDER_SIZE;
      minimize_rect.w = BUTTON_SIZE;
      minimize_rect.h = TITLEBAR_SIZE;
      
      // Draw
      XSetForeground(display, gc, yellow.pixel);
      XFillRectangle(display, window, gc, 0, 0, window_width, window_height);
      
      DrawRect(display, window, gc, titlebar_rect, dark_gray);
      DrawRect(display, window, gc, content_rect, gray);
      DrawRect(display, window, gc, close_rect, red);
      DrawRect(display, window, gc, maximize_rect, green);
      DrawRect(display, window, gc, minimize_rect, blue);
    }
  }
  
  XFreeGC(display, gc);
  XDestroyWindow(display, window);
  XCloseDisplay(display);
  
  return 0;
}

static inline bool PointInRect(int x, int y, Rect r)
{
  return (x > r.x && x < r.x + r.w && y > r.y && y < r.y + r.h);
}

static inline void ToggleMaximize(Display* display, Window window)
{
  XEvent event = {0};
  event.type = ClientMessage;
  event.xclient.window = window;
  event.xclient.message_type = _NET_WM_STATE;
  event.xclient.format = 32;
  // 2 = toggle
  event.xclient.data.l[0] = 2;
  event.xclient.data.l[1] = _NET_WM_STATE_MAXIMIZED_HORZ;
  event.xclient.data.l[2] = _NET_WM_STATE_MAXIMIZED_VERT;
  event.xclient.data.l[3] = 1;
  
  XSendEvent(display, XDefaultRootWindow(display), 0, SubstructureRedirectMask | SubstructureNotifyMask, &event);
}

static inline int ResizeDirectionFromPoint(int x, int y, int window_width, int window_height)
{
  // 0 = NorthWest
  // 1 = North
  // 2 = NorthEast
  // 3 = East
  // etc
  int result = 8;
  if(x <= BORDER_SIZE && y <= BORDER_SIZE)
    result = 0;
  else if(x >= window_width - BORDER_SIZE && y <= BORDER_SIZE)
    result = 2;
  else if(x >= window_width - BORDER_SIZE && y >= window_height - BORDER_SIZE)
    result = 4;
  else if(x <= BORDER_SIZE && y >= window_height - BORDER_SIZE)
    result = 6;
  else if(x <= BORDER_SIZE)
    result = 7;
  else if(x >= window_width - BORDER_SIZE)
    result = 3;
  else if(y <= BORDER_SIZE)
    result = 1;
  else if(y >= window_height - BORDER_SIZE)
    result = 5;
  
  return result;
}

static inline void InitiateMoveResize(Display* display, Window window, int direction, XButtonEvent button_event)
{
  XUngrabPointer(display, 0);
  
  XEvent event = {0};
  event.type = ClientMessage;
  event.xclient.send_event = 1;
  event.xclient.display = display;
  event.xclient.window = window;
  event.xclient.message_type = _NET_WM_MOVERESIZE;
  event.xclient.format = 32;
  event.xclient.data.l[0] = button_event.x_root;
  event.xclient.data.l[1] = button_event.y_root;
  event.xclient.data.l[2] = direction;
  event.xclient.data.l[3] = button_event.button;
  event.xclient.data.l[4] = 1;
  
  XSendEvent(display, XDefaultRootWindow(display), 0, SubstructureRedirectMask | SubstructureNotifyMask, &event);
}

static inline void DrawRect(Display* display, Window window, GC gc, Rect rect, XColor color)
{
  XSetForeground(display, gc, color.pixel);
  XFillRectangle(display, window, gc, rect.x, rect.y, rect.w, rect.h);
}