BottomSheetDialog.java

private View wrapInBottomSheet(int layoutResId, View view, ViewGroup.LayoutParams params) {
    ...
    if (shouldWindowCloseOnTouchOutside()) {
        coordinator.findViewById(R.id.touch_outside).setOnClickListener(
                new View.OnClickListener() {
                    @Override
                    public void onClick(View view) {
                        if (isShowing()) {
                            cancel();
                        }
                    }
                });
    }
}