https://stackoverflow.com/questions/62374427/javafx-how-to-prevent-painting-white-on-client-area

playground_build.gradle

plugins {
  id 'java'
  id 'application'
  id 'org.openjfx.javafxplugin' version '0.0.8'
}

repositories {
  jcenter()
}

javafx {
  version = "14"
  modules = ['javafx.controls', 'javafx.fxml']
}

sourceCompatibility = JavaVersion.VERSION_14
targetCompatibility = JavaVersion.VERSION_14

mainClassName = 'org.openjfx.MainApp'

project.ext.jnaVersion = '5.5.0'

dependencies {
  implementation "net.java.dev.jna:jna:${jnaVersion}"
  implementation "net.java.dev.jna:jna-platform:${jnaVersion}"
}

playground_src_main_java_com_example_CustomFrame.java

package com.example;

import com.sun.jna.Native;
import com.sun.jna.Pointer;
import com.sun.jna.Structure;
import com.sun.jna.platform.win32.BaseTSD;
import com.sun.jna.platform.win32.User32;
import com.sun.jna.platform.win32.WinDef;
import com.sun.jna.platform.win32.WinDef.HWND;
import com.sun.jna.platform.win32.WinDef.LRESULT;
import com.sun.jna.platform.win32.WinError;
import com.sun.jna.platform.win32.WinNT;
import com.sun.jna.platform.win32.WinUser;
import com.sun.jna.win32.StdCallLibrary;
import com.sun.jna.win32.W32APIOptions;
import javafx.application.Application;
import javafx.stage.Stage;

import java.util.Objects;

import static com.sun.jna.platform.win32.WinUser.GWL_WNDPROC;
import static com.sun.jna.platform.win32.WinUser.SWP_FRAMECHANGED;
import static com.sun.jna.platform.win32.WinUser.SWP_NOMOVE;
import static com.sun.jna.platform.win32.WinUser.SWP_NOSIZE;
import static com.sun.jna.platform.win32.WinUser.SWP_NOZORDER;

public class CustomFrame {

  private static final int WM_NCCALCSIZE = 0x0083;
  private static final int WM_ACTIVATE = 0x0006;

  public static final LRESULT LRESULT_ZERO = new LRESULT(0);

  /**
   * Starting CustomFrameApplication instead of calling this main, at least in IntelliJ, you'll get "Error: JavaFX
   * runtime components are missing, and are required to run this application".
   */
  public static void main(String[] args) {
    CustomFrameApplication.launch(CustomFrameApplication.class, args);
  }

  public static class CustomFrameApplication extends Application {

    @Override
    public void start(Stage primaryStage) {
      // No scene set: JavaFX will initially paint a white rectangle onto the client area which won't be updated when
      // resizing the window. Instead, you'll see the default black client area background.
      primaryStage.show();

      // Even with a transparent scene, a white background will be painted. It will be re-painted when resizing.
//      primaryStage.setScene(new Scene(new StackPane(), Color.TRANSPARENT));
//      primaryStage.show();

      // This causes the client area to be transparent, which is what I want, but it also removes window decorations.
//      primaryStage.initStyle(StageStyle.TRANSPARENT);
//      primaryStage.setScene(new Scene(new StackPane(), Color.TRANSPARENT));
//      primaryStage.show();

      HWND hwnd = new HWND();
      hwnd.setPointer(User32.INSTANCE.GetActiveWindow().getPointer());

      setCustomWindowProc(hwnd);
    }

    private void setCustomWindowProc(HWND hwnd) {
      BaseTSD.LONG_PTR defaultWindowProc = User32.INSTANCE.GetWindowLongPtr(hwnd, GWL_WNDPROC);

      // Since the window has already been created, this can't be called in WM_CREATE, so do it here
      extendFrameIntoClientArea(hwnd, new Margins(8, 8, 20, 27));

      WinUser.WindowProc windowProc = new WinUser.WindowProc() {
        @Override
        public LRESULT callback(HWND hwnd, int uMsg, WinDef.WPARAM wParam, WinDef.LPARAM lParam) {
          switch (uMsg) {
            // With this, the whole window becomes client-area. It does not remove the default frame but JavaFX covers
            // it in white paint
//          case WM_NCCALCSIZE:
//            if (wParam.intValue() != 0) {
//              return LRESULT_ZERO;
//            }
//            return user32Ex.CallWindowProc(defaultWindowProc, hwnd, uMsg, wParam, lParam);

            case WM_ACTIVATE:
              extendFrameIntoClientArea(hwnd, new Margins(8, 8, 20, 27));
              return User32.INSTANCE.DefWindowProc(hwnd, uMsg, wParam, lParam);

            default:
              return User32.INSTANCE.DefWindowProc(hwnd, uMsg, wParam, lParam);
          }
        }
      };

      User32Ex.INSTANCE.SetWindowLongPtr(hwnd, GWL_WNDPROC, windowProc);
      User32Ex.INSTANCE.SetWindowPos(hwnd, hwnd, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE | SWP_NOZORDER | SWP_FRAMECHANGED);
    }
  }

  private static void extendFrameIntoClientArea(WinDef.HWND hwnd, Margins margins) {
    WinNT.HRESULT hresult = DwmApi.INSTANCE.DwmExtendFrameIntoClientArea(hwnd, margins);
    if (!Objects.equals(hresult, WinError.S_OK)) {
      throw new IllegalStateException("Could not call DwmExtendFrameIntoClientArea");
    }
  }

  @Structure.FieldOrder({"cxLeftWidth", "cxRightWidth", "cyTopHeight", "cyBottomHeight"})
  public static class Margins extends Structure implements Structure.ByReference {

    public Margins(int cxLeftWidth, int cxRightWidth, int cyTopHeight, int cyBottomHeight) {
      this.cxLeftWidth = cxLeftWidth;
      this.cxRightWidth = cxRightWidth;
      this.cyTopHeight = cyTopHeight;
      this.cyBottomHeight = cyBottomHeight;
    }

    public int cxLeftWidth = 0;
    public int cxRightWidth = 0;
    public int cyTopHeight = 0;
    public int cyBottomHeight = 0;
  }

  public interface User32Ex extends User32 {
    User32Ex INSTANCE = Native.load("user32", User32Ex.class, W32APIOptions.DEFAULT_OPTIONS);

    Pointer SetWindowLongPtr(HWND hWnd, int nIndex, WindowProc wndProc);
  }

  public interface DwmApi extends StdCallLibrary, WinUser, WinNT {

    /** A value for the fEnable member has been specified. */
    int DWM_BB_ENABLE = 1;
    /** A value for the hRgnBlur member has been specified. */
    int DWM_BB_BLURREGION = 2;
    /** A value for the fTransitionOnMaximized member has been specified. */
    int DWM_BB_TRANSITIONONMAXIMIZED = 4;

    DwmApi INSTANCE = Native.load("dwmapi", DwmApi.class, W32APIOptions.DEFAULT_OPTIONS);

    /**
     * @see <a href="https://docs.microsoft.com/en-us/windows/win32/api/dwmapi/nf-dwmapi-dwmextendframeintoclientarea?redirectedfrom=MSDN">MSDN</a>
     */
    HRESULT DwmExtendFrameIntoClientArea(HWND hWnd, Margins pMarInset);
  }
}