BottomSheetDialog.java

private BottomSheetBehavior.BottomSheetCallback mBottomSheetCallback
        = new BottomSheetBehavior.BottomSheetCallback() {
    @Override
    public void onStateChanged(@NonNull View bottomSheet,
            @BottomSheetBehavior.State int newState) {
        if (newState == BottomSheetBehavior.STATE_HIDDEN) {
            dismiss();
        }
    }

    @Override
    public void onSlide(@NonNull View bottomSheet, float slideOffset) {
    }
};