This is an example about how you can customize the table menu button in a JavaFX TableView using reflection.

CustomTableMenuDemo.java

import javafx.application.Application;
import javafx.beans.property.SimpleStringProperty;
import javafx.collections.FXCollections;
import javafx.collections.ObservableList;
import javafx.geometry.Insets;
import javafx.scene.Scene;
import javafx.scene.control.TableColumn;
import javafx.scene.control.TableView;
import javafx.scene.control.cell.PropertyValueFactory;
import javafx.scene.layout.BorderPane;
import javafx.scene.layout.VBox;
import javafx.scene.text.Text;
import javafx.stage.Stage;

public class CustomTableMenuDemo extends Application {

    private final ObservableList<Person> data =
            FXCollections.observableArrayList( 
            new Person("Jacob", "Smith", "jacob.smith@example.com"),
            new Person("Isabella", "Johnson", "isabella.johnson@example.com"),
            new Person("Ethan", "Williams", "ethan.williams@example.com"),
            new Person("Emma", "Jones", "emma.jones@example.com"),
            new Person("Isabella", "Johnson", "isabella.johnson@example.com"),
            new Person("Ethan", "Williams", "ethan.williams@example.com"),
            new Person("Emma", "Jones", "emma.jones@example.com"),
            new Person("Isabella", "Johnson", "isabella.johnson@example.com"),
            new Person("Ethan", "Williams", "ethan.williams@example.com"),
            new Person("Emma", "Jones", "emma.jones@example.com"),
            new Person("Isabella", "Johnson", "isabella.johnson@example.com"),
            new Person("Ethan", "Williams", "ethan.williams@example.com"),
            new Person("Emma", "Jones", "emma.jones@example.com"),
            new Person("Isabella", "Johnson", "isabella.johnson@example.com"),
            new Person("Ethan", "Williams", "ethan.williams@example.com"),
            new Person("Emma", "Jones", "emma.jones@example.com"),
            new Person("Isabella", "Johnson", "isabella.johnson@example.com"),
            new Person("Ethan", "Williams", "ethan.williams@example.com"),
            new Person("Emma", "Jones", "emma.jones@example.com"),
            new Person("Michael", "Brown", "michael.brown@example.com"));
    
	public static void main(String[] args) {
		launch(args);
	}

	@Override
	public void start(Stage stage) {
		
		stage.setTitle("Table Menu Demo");
		stage.setWidth(500);
		stage.setHeight(550);

		// create table columns
		TableColumn<Person, String> firstNameCol = new TableColumn<Person, String>("First Name");
		firstNameCol.setMinWidth(100);
		firstNameCol.setCellValueFactory(new PropertyValueFactory<Person, String>("firstName"));

		TableColumn<Person, String> lastNameCol = new TableColumn<Person, String>("Last Name");
		lastNameCol.setMinWidth(100);
		lastNameCol.setCellValueFactory(new PropertyValueFactory<Person, String>("lastName"));

		TableColumn<Person, String> emailCol = new TableColumn<Person, String>("Email");
		emailCol.setMinWidth(180);
		emailCol.setCellValueFactory(new PropertyValueFactory<Person, String>("email"));


		TableView<Person> tableView = new TableView<>();
		tableView.setPlaceholder(new Text("No content in table"));
		tableView.setItems(data);
		tableView.getColumns().addAll(firstNameCol, lastNameCol, emailCol);

		final VBox vbox = new VBox();
		vbox.setSpacing(5);
		vbox.setPadding(new Insets(10, 10, 10, 10));
		
		BorderPane borderPane = new BorderPane();
		borderPane.setCenter( tableView);
		
		vbox.getChildren().addAll( borderPane);

		Scene scene = new Scene( vbox);
		

		stage.setScene(scene);
		stage.show();

		// enable table menu button and add a custom menu to it
		TableUtils.addCustomTableMenu(tableView);
	}

	
    public static class Person {
      	 
        private final SimpleStringProperty firstName;
        private final SimpleStringProperty lastName;
        private final SimpleStringProperty email;
 
        private Person(String fName, String lName, String email) {
            this.firstName = new SimpleStringProperty(fName);
            this.lastName = new SimpleStringProperty(lName);
            this.email = new SimpleStringProperty(email);
        }
 
        public String getFirstName() {
            return firstName.get();
        }
 
        public void setFirstName(String fName) {
            firstName.set(fName);
        }
 
        public String getLastName() {
            return lastName.get();
        }
 
        public void setLastName(String fName) {
            lastName.set(fName);
        }
 
        public String getEmail() {
            return email.get();
        }
 
        public void setEmail(String fName) {
            email.set(fName);
        }
    }

}

TableUtils.java

import java.lang.reflect.Field;

import javafx.collections.ObservableList;
import javafx.event.EventHandler;
import javafx.scene.Node;
import javafx.scene.control.CheckBox;
import javafx.scene.control.ContextMenu;
import javafx.scene.control.CustomMenuItem;
import javafx.scene.control.Label;
import javafx.scene.control.SeparatorMenuItem;
import javafx.scene.control.TableColumn;
import javafx.scene.control.TableView;
import javafx.scene.input.MouseEvent;

import com.sun.javafx.scene.control.skin.TableHeaderRow;
import com.sun.javafx.scene.control.skin.TableViewSkin;

public class TableUtils {

	/**
	 * Make table menu button visible and replace the context menu with a custom context menu via reflection.
	 * The preferred height is modified so that an empty header row remains visible. This is needed in case you remove all columns, so that the menu button won't disappear with the row header.
	 * IMPORTANT: Modification is only possible AFTER the table has been made visible, otherwise you'd get a NullPointerException
	 * @param tableView
	 */
	public static void addCustomTableMenu( TableView tableView) {

		// enable table menu
		tableView.setTableMenuButtonVisible(true);
		
		// get the table  header row
		TableHeaderRow tableHeaderRow = getTableHeaderRow((TableViewSkin) tableView.getSkin());

		// get context menu via reflection
		ContextMenu contextMenu = getContextMenu(tableHeaderRow);

		// setting the preferred height for the table header row
		// if the preferred height isn't set, then the table header would disappear if there are no visible columns
		// and with it the table menu button
		// by setting the preferred height the header will always be visible
		// note: this may need adjustments in case you have different heights in columns (eg when you use grouping)
		double defaultHeight = tableHeaderRow.getHeight();
		tableHeaderRow.setPrefHeight(defaultHeight);
		
		// modify the table menu
		contextMenu.getItems().clear();

		addCustomMenuItems( contextMenu, tableView);

	}
	
	/**
	 * Create a menu with custom items. The important thing is that the menu remains open while you click on the menu items.
	 * @param cm
	 * @param table
	 */
	private static void addCustomMenuItems( ContextMenu cm, TableView table) {
		
		// create new context menu
		CustomMenuItem cmi;

		// select all item
		Label showAll = new Label("Show all");
		showAll.addEventHandler(MouseEvent.MOUSE_CLICKED, new EventHandler<MouseEvent>() {

			@Override
			public void handle(MouseEvent event) {
				for (Object obj : table.getColumns()) {
					((TableColumn<?, ?>) obj).setVisible(true);
				}
			}

		});

		cmi = new CustomMenuItem(showAll);
		cmi.setHideOnClick(false);
		cm.getItems().add(cmi);

		// deselect all item
		Label hideAll = new Label("Hide all");
		hideAll.addEventHandler(MouseEvent.MOUSE_CLICKED, new EventHandler<MouseEvent>() {

			@Override
			public void handle(MouseEvent event) {

				for (Object obj : table.getColumns()) {
					((TableColumn<?, ?>) obj).setVisible(false);
				}
			}

		});

		cmi = new CustomMenuItem(hideAll);
		cmi.setHideOnClick(false);
		cm.getItems().add(cmi);

		// separator
		cm.getItems().add(new SeparatorMenuItem());

		// menu item for each of the available columns
		for (Object obj : table.getColumns()) {

			TableColumn<?, ?> tableColumn = (TableColumn<?, ?>) obj;

			CheckBox cb = new CheckBox(tableColumn.getText());
			cb.selectedProperty().bindBidirectional(tableColumn.visibleProperty());

			cmi = new CustomMenuItem(cb);
			cmi.setHideOnClick(false);

			cm.getItems().add(cmi);
		}
		
	}

	/**
	 * Find the TableHeaderRow of the TableViewSkin
	 * 
	 * @param tableSkin
	 * @return
	 */
	private static TableHeaderRow getTableHeaderRow(TableViewSkin<?> tableSkin) {

		// get all children of the skin
		ObservableList<Node> children = tableSkin.getChildren();

		// find the TableHeaderRow child
		for (int i = 0; i < children.size(); i++) {

			Node node = children.get(i);

			if (node instanceof TableHeaderRow) {
				return (TableHeaderRow) node;
			}

		}
		return null;
	}

	/**
	 * Get the table menu, i. e. the ContextMenu of the given TableHeaderRow via
	 * reflection
	 * 
	 * @param headerRow
	 * @return
	 */
	private static ContextMenu getContextMenu(TableHeaderRow headerRow) {

		try {

			// get columnPopupMenu field
			Field privateContextMenuField = TableHeaderRow.class.getDeclaredField("columnPopupMenu");

			// make field public
			privateContextMenuField.setAccessible(true);

			// get field
			ContextMenu contextMenu = (ContextMenu) privateContextMenuField.get(headerRow);

			return contextMenu;

		} catch (Exception ex) {
			ex.printStackTrace();
		}

		return null;
	}

}

For JavaFX 21 and JDK 21, here is my implementation of his code, just changed a couple of imports to align with JavaFX 21 and JDK 21.

import javafx.collections.ObservableList;
import javafx.event.EventHandler;
import javafx.geometry.Side;
import javafx.scene.Node;
import javafx.scene.control.CheckBox;
import javafx.scene.control.ContextMenu;
import javafx.scene.control.CustomMenuItem;
import javafx.scene.control.Label;
import javafx.scene.control.SeparatorMenuItem;
import javafx.scene.control.TableColumn;
import javafx.scene.control.TableView;
import javafx.scene.input.MouseEvent;

import javafx.scene.control.skin.TableHeaderRow;
import javafx.scene.control.skin.TableViewSkin;

public class TableUtils {

	/**
	 * Make table menu button visible and replace the context menu with a custom context menu via reflection.
	 * The preferred height is modified so that an empty header row remains visible. This is needed in case you remove all columns, so that the menu button won't disappear with the row header.
	 * IMPORTANT: Modification is only possible AFTER the table has been made visible, otherwise you'd get a NullPointerException
	 * @param tableView
	 */
	public static void addCustomTableMenu( TableView<?> tableView) {

		// enable table menu
		tableView.setTableMenuButtonVisible(true);
		
		// replace internal mouse listener with custom listener 
		setCustomContextMenu( tableView);

	}
	
	private static void setCustomContextMenu( TableView<?> table) {

		TableViewSkin<?> tableSkin = (TableViewSkin<?>) table.getSkin();

		// get all children of the skin
		ObservableList<Node> children = tableSkin.getChildren();

		// find the TableHeaderRow child
		for (int i = 0; i < children.size(); i++) {

			Node node = children.get(i);

			if (node instanceof TableHeaderRow) {
				
				TableHeaderRow tableHeaderRow = (TableHeaderRow) node;
				
				// setting the preferred height for the table header row
				// if the preferred height isn't set, then the table header would disappear if there are no visible columns
				// and with it the table menu button
				// by setting the preferred height the header will always be visible
				// note: this may need adjustments in case you have different heights in columns (eg when you use grouping)
				double defaultHeight = tableHeaderRow.getHeight();
				tableHeaderRow.setPrefHeight(defaultHeight);
				
				for( Node child: tableHeaderRow.getChildren()) {

					// child identified as cornerRegion in TableHeaderRow.java
					if( child.getStyleClass().contains( "show-hide-columns-button")) {
						
						// get the context menu
						ContextMenu columnPopupMenu = createContextMenu( table);
						
						// replace mouse listener
				        child.setOnMousePressed(me -> {
				            // show a popupMenu which lists all columns
				            columnPopupMenu.show(child, Side.BOTTOM, 0, 0);
				            me.consume();
				        });
					}
				}
				
			}
		}
	}
	
	/**
	 * Create a menu with custom items. The important thing is that the menu remains open while you click on the menu items.
	 * @param cm
	 * @param table
	 */
	private static ContextMenu createContextMenu( TableView<?> table) {
		
		ContextMenu cm = new ContextMenu();
		
		// create new context menu
		CustomMenuItem cmi;

		// select all item
		Label showAll = new Label("Show all");
		showAll.addEventHandler(MouseEvent.MOUSE_CLICKED, new EventHandler<MouseEvent>() {

			@Override
			public void handle(MouseEvent event) {
				for (Object obj : table.getColumns()) {
					((TableColumn<?, ?>) obj).setVisible(true);
				}
			}

		});

		cmi = new CustomMenuItem(showAll);
		cmi.setHideOnClick(false);
		cm.getItems().add(cmi);

		// De-select all item
		Label hideAll = new Label("Hide all");
		hideAll.addEventHandler(MouseEvent.MOUSE_CLICKED, new EventHandler<MouseEvent>() {

			@Override
			public void handle(MouseEvent event) {

				for (Object obj : table.getColumns()) {
					((TableColumn<?, ?>) obj).setVisible(false);
				}
			}

		});

		cmi = new CustomMenuItem(hideAll);
		cmi.setHideOnClick(false);
		cm.getItems().add(cmi);

		// separator
		cm.getItems().add(new SeparatorMenuItem());

		// menu item for each of the available columns
		for (Object obj : table.getColumns()) {

			TableColumn<?, ?> tableColumn = (TableColumn<?, ?>) obj;

			CheckBox cb = new CheckBox(tableColumn.getText());
			cb.selectedProperty().bindBidirectional(tableColumn.visibleProperty());

			cmi = new CustomMenuItem(cb);
			cmi.setHideOnClick(false);

			cm.getItems().add(cmi);
		}
		
		return cm;
	}
}