useOutsideClick.js

import { useEffect, useRef } from 'react';

export default function useOutsideClick(callback) {
  const refElement = useRef(null);

  useEffect(() => {
    function handleClickOutside(event) {
      if(refElement.current && !refElement.current.contains(event.target)) {
        callback();
      }
    }

    document.addEventListener('mousedown', handleClickOutside);
    return () => {
      document.removeEventListener('mousedown', handleClickOutside);
    }
  }, [refElement, callback]);

  return { refElement };
}

useOutsideClick.test.js

import React from 'react';
import { renderHook, act } from '@testing-library/react-hooks'
import userEvent from '@testing-library/user-event';
import { fireEvent, ireEvent } from "@testing-library/react";
import useOutsideClick from './useOutsideClick';


describe('useOutsideClick', () => {

  const mockCallback = jest.fn();
  const mockHandle = jest.fn();
  const elementMock = { addEventListener: jest.fn() };
  jest.spyOn(document, 'getElementById').mockImplementation(() => elementMock);

  it('refElement should be null at initial render', () => {
    const { result, rerender } = renderHook(() => useOutsideClick())
    const refElement = result.current.refElement.current;

    expect(refElement).toBeNull();
  })

  it('test', () => {
      const { result } = renderHook(() => useOutsideClick(mockCallback));
      const refElement = result.current.refElement = document.createElement('div');

      act(() => {
        fireEvent.click(refElement);
        fireEvent.mouseDown(document);
      });

      expect(elementMock.addEventListener).toBeCalledWith(mockCallback);
  })

});