Window Frame Effects for Windows (Transparency/Blur Behind/Client side decorations) in Java.

WinEffects.java

// START WIN7
// MARGINS class for WIN7 Frame
public class MARGINS extends Structure implements Structure.ByReference {

		public int cxLeftWidth;
		public int cxRightWidth;
		public int cyTopHeight;
		public int cyBottomHeight;

		@Override
		protected List<String> getFieldOrder() {
			return Arrays.asList("cxLeftWidth", "cxRightWidth", "cyTopHeight", "cyBottomHeight");
		}

	}

// WIN7 Frame using LWJGL API and JNA.

		//SetWindowLongPtr(hwnd, GWL_EXSTYLE, GetWindowLongPtr(hwnd, GWL_EXSTYLE) | WS_EX_LAYERED);

		NativeLibrary dwmapi = NativeLibrary.getInstance("dwmapi");

		HWND aeroFrameHWND = new HWND(new Pointer(hwnd)); // Modify pointer to window

		MARGINS margins = new MARGINS();

// Frame
		margins.cxLeftWidth = 8;
		margins.cxRightWidth = 8;
		margins.cyBottomHeight = 20;
		margins.cyTopHeight = 27;

		Function extendFrameIntoClientArea = dwmapi.getFunction("DwmExtendFrameIntoClientArea");
		HRESULT result = (HRESULT) extendFrameIntoClientArea.invoke(HRESULT.class,
				new Object[] { aeroFrameHWND, margins });

		if (result.intValue() != 0)
			System.err.println("Call to DwmExtendFrameIntoClientArea failed.");

		//SetLayeredWindowAttributes(hwnd, 0xffffffff, (byte) 0, LWA_COLORKEY); // Modify mask color.
		SetWindowPos(hwnd, 0, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE | SWP_NOZORDER | SWP_FRAMECHANGED);
// END WIN7

// START WIN10
// WIN10 helpers for compositing (alpha, blur behind)
	public interface Accent {
		public static final int ACCENT_DISABLED = 0;
		public static final int ACCENT_ENABLE_GRADIENT = 1;
		public static final int ACCENT_ENABLE_TRANSPARENTGRADIENT = 2;
		public static final int ACCENT_ENABLE_BLURBEHIND = 3;
		public static final int ACCENT_ENABLE_ACRYLIC = 4; // YES, available on build 17063
		public static final int ACCENT_INVALID_STATE = 5;
	}

	public static interface WindowCompositionAttribute {
		public static final int WCA_ACCENT_POLICY = 19;
	}

	public class AccentPolicy extends Structure implements Structure.ByReference {
		public static final List<String> FIELDS = createFieldsOrder("AccentState", "AccentFlags", "GradientColor",
				"AnimationId");
		public int AccentState;
		public int AccentFlags;
		public int GradientColor;
		public int AnimationId;

		@Override
		protected List<String> getFieldOrder() {
			return FIELDS;
		}
	}

	public class WindowCompositionAttributeData extends Structure implements Structure.ByReference {
		public static final List<String> FIELDS = createFieldsOrder("Attribute", "Data", "SizeOfData");
		public int Attribute;
		public Pointer Data;
		public int SizeOfData;

		@Override
		protected List<String> getFieldOrder() {
			return FIELDS;
		}
	}

// WIN10 Composite using JNA.

		HWND aeroFrameHWND = new HWND(new Pointer(hwnd)); // Modify pointer to window

		NativeLibrary user32 = NativeLibrary.getInstance("user32");

		AccentPolicy accent = new AccentPolicy();
		accent.AccentState = Accent.ACCENT_ENABLE_ACRYLIC;
		accent.GradientColor = 0x7F000000;
		accent.AccentFlags = AccentFlags.DrawAllBorders;
		accent.write();

		WindowCompositionAttributeData data = new WindowCompositionAttributeData();
		data.Attribute = WindowCompositionAttribute.WCA_ACCENT_POLICY;
		data.SizeOfData = accent.size();
		data.Data = accent.getPointer();

		Function setWindowCompositionAttribute = user32.getFunction("SetWindowCompositionAttribute");
		setWindowCompositionAttribute.invoke(HRESULT.class, new Object[] { aeroFrameHWND, data });
// END WIN10

// CLIENT SIDE DECORATIONS JNA/LWJGL

		long hwnd = //window HWND
		long dwp = GetWindowLongPtr(hwnd, GWL_WNDPROC);

		WindowProc proc = new WindowProc() {

			@Override
			public long invoke(long hwnd, int uMsg, long wParam, long lParam) {
				switch (uMsg) {
				case WM_NCCALCSIZE:
					if (wParam != 0) {
						return 0;
					} else {
						return JNI.callPPPP(dwp, hwnd, uMsg, wParam, lParam);
					}
				}
				return JNI.callPPPP(dwp, hwnd, uMsg, wParam, lParam);
			}
		};

		SetWindowLongPtr(hwnd, GWL_WNDPROC, proc.address());
		SetWindowPos(hwnd, 0, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE | SWP_NOZORDER | SWP_FRAMECHANGED);

		NativeLibrary dwmapi = NativeLibrary.getInstance("dwmapi");

		HWND aeroFrameHWND = new HWND(new Pointer(hwnd));

		MARGINS margins = new MARGINS();

		margins.cxLeftWidth = 8;
		margins.cxRightWidth = 8;
		margins.cyBottomHeight = 8;
		margins.cyTopHeight = 30;

		Function extendFrameIntoClientArea = dwmapi.getFunction("DwmExtendFrameIntoClientArea");
		HRESULT result = (HRESULT) extendFrameIntoClientArea.invoke(HRESULT.class,
				new Object[] { aeroFrameHWND, margins });

		if (result.intValue() != 0)
			System.err.println("Call to DwmExtendFrameIntoClientArea failed.");

// END CLIENT SIDE DECORATIONS