Extension version of these PR changes: https://github.com/ueberdosis/tiptap/pull/3819 as of commit 09782a5; useful for quick tests in other editor environments

helpers.ts

import { getNodeType, RawCommands } from '@tiptap/core';
import { Node, NodeType } from '@tiptap/pm/model';
import { EditorState } from '@tiptap/pm/state';

/**
 * Finds the first node of a given type or name in the current selection.
 * @param state The editor state.
 * @param typeOrName The node type or name.
 * @param pos The position to start searching from.
 * @param maxDepth The maximum depth to search.
 * @returns The node and the depth as an array.
 */
export const getNodeAtPosition = (
  state: EditorState,
  typeOrName: string | NodeType,
  pos: number,
  maxDepth = 20
) => {
  const $pos = state.doc.resolve(pos);

  let currentDepth = maxDepth;
  let node: Node | null = null;

  while (currentDepth > 0 && node === null) {
    const currentNode = $pos.node(currentDepth);

    if (currentNode?.type.name === typeOrName) {
      node = currentNode;
    } else {
      currentDepth -= 1;
    }
  }

  return [node, currentDepth] as [Node | null, number];
};

export const isAtStartOfNode = (state: EditorState) => {
  const { $from, $to } = state.selection;

  if ($from.parentOffset > 0 || $from.pos !== $to.pos) {
    return false;
  }

  return true;
};

export const findListItemPos = (
  typeOrName: string | NodeType,
  state: EditorState
) => {
  const { $from } = state.selection;
  const nodeType = getNodeType(typeOrName, state.schema);

  let currentNode = null;
  let currentDepth = $from.depth;
  let currentPos = $from.pos;
  let targetDepth: number | null = null;

  while (currentDepth > 0 && targetDepth === null) {
    currentNode = $from.node(currentDepth);

    if (currentNode.type === nodeType) {
      targetDepth = currentDepth;
    } else {
      currentDepth -= 1;
      currentPos -= 1;
    }
  }

  if (targetDepth === null) {
    return null;
  }

  return { $pos: state.doc.resolve(currentPos), depth: targetDepth };
};

export const hasPreviousListItem = (typeOrName: string, state: EditorState) => {
  const listItemPos = findListItemPos(typeOrName, state);

  if (!listItemPos) {
    return false;
  }

  const $item = state.doc.resolve(listItemPos.$pos.pos);
  const $prev = state.doc.resolve(listItemPos.$pos.pos - 2);

  const prevNode = $prev.node($item.depth);

  if (!prevNode) {
    return false;
  }

  return prevNode.type.name === typeOrName;
};

export const listItemHasSubList = (
  typeOrName: string,
  state: EditorState,
  node?: Node
) => {
  if (!node) {
    return false;
  }

  const nodeType = getNodeType(typeOrName, state.schema);

  let hasSubList = false;

  node.descendants((child) => {
    if (child.type === nodeType) {
      hasSubList = true;
    }
  });

  return hasSubList;
};

export const isAtEndOfNode = (state: EditorState) => {
  const { $from, $to } = state.selection;

  if ($to.parentOffset < $to.parent.nodeSize - 2 || $from.pos !== $to.pos) {
    return false;
  }

  return true;
};

export const getNextListDepth = (typeOrName: string, state: EditorState) => {
  const listItemPos = findListItemPos(typeOrName, state);

  if (!listItemPos) {
    return false;
  }

  const [, depth] = getNodeAtPosition(
    state,
    typeOrName,
    listItemPos.$pos.pos + 4
  );

  return depth;
};

export const nextListIsDeeper = (typeOrName: string, state: EditorState) => {
  const listDepth = getNextListDepth(typeOrName, state);
  const listItemPos = findListItemPos(typeOrName, state);

  if (!listItemPos || !listDepth) {
    return false;
  }

  if (listDepth > listItemPos.depth) {
    return true;
  }

  return false;
};

export const nextListIsHigher = (typeOrName: string, state: EditorState) => {
  const listDepth = getNextListDepth(typeOrName, state);
  const listItemPos = findListItemPos(typeOrName, state);

  if (!listItemPos || !listDepth) {
    return false;
  }

  if (listDepth < listItemPos.depth) {
    return true;
  }

  return false;
};

list-item-v2.ts

import TapListItem from '@tiptap/extension-list-item';
import { isNodeActive, Range } from '@tiptap/core';
import { joinPoint } from '@tiptap/pm/transform';
import {
  findListItemPos,
  hasPreviousListItem,
  isAtEndOfNode,
  isAtStartOfNode,
  listItemHasSubList,
  nextListIsDeeper,
  nextListIsHigher,
} from './helpers';

declare module '@tiptap/core' {
  interface Commands<ReturnType> {
    listItem: {
      joinListItemForward: () => ReturnType;
      joinListItemBackward: () => ReturnType;
    };
  }
}

export const ListItem = TapListItem.extend({
  name: 'listItem',
  defining: true,

  addCommands() {
    return {
      joinListItemBackward:
        () =>
        ({ tr, state, dispatch }) => {
          try {
            const point = joinPoint(state.doc, state.selection.$from.pos, -1);

            if (point === null || point === undefined) {
              return false;
            }

            tr.join(point, 2);

            if (dispatch) {
              dispatch(tr);
            }

            return true;
          } catch {
            return false;
          }
        },

      joinListItemForward:
        () =>
        ({ tr, state, dispatch }) => {
          try {
            const point = joinPoint(state.doc, state.selection.$from.pos, +1);

            if (point === null || point === undefined) {
              return false;
            }

            tr.join(point, 2);

            if (dispatch) {
              dispatch(tr);
            }

            return true;
          } catch {
            return false;
          }
        },
    };
  },

  addKeyboardShortcuts() {
    return {
      Enter: () => this.editor.commands.splitListItem(this.name),
      Tab: () => this.editor.commands.sinkListItem(this.name),
      'Shift-Tab': () => this.editor.commands.liftListItem(this.name),
      Delete: ({ editor }) => {
        // if the cursor is not inside the current node type
        // do nothing and proceed
        if (!isNodeActive(editor.state, this.name)) {
          return false;
        }

        // if the cursor is not at the end of a node
        // do nothing and proceed
        if (!isAtEndOfNode(editor.state)) {
          return false;
        }

        // check if the next node is a list with a deeper depth
        if (nextListIsDeeper(this.name, editor.state)) {
          return editor
            .chain()
            .focus(editor.state.selection.from + 4)
            .lift(this.name)
            .joinBackward()
            .run();
        }

        if (nextListIsHigher(this.name, editor.state)) {
          return editor.chain().joinForward().joinBackward().run();
        }

        // check if the next node is also a listItem
        return editor.commands.joinListItemForward();
      },
      Backspace: ({ editor }) => {
        // this is required to still handle the undo handling
        if (this.editor.commands.undoInputRule()) {
          return true;
        }

        // if the cursor is not inside the current node type
        // do nothing and proceed
        if (!isNodeActive(editor.state, this.name)) {
          return false;
        }

        // if the cursor is not at the start of a node
        // do nothing and proceed
        if (!isAtStartOfNode(editor.state)) {
          return false;
        }

        const listItemPos = findListItemPos(this.name, editor.state);

        if (!listItemPos) {
          return false;
        }

        const $prev = editor.state.doc.resolve(listItemPos.$pos.pos - 2);
        const prevNode = $prev.node(listItemPos.depth);

        const previousListItemHasSubList = listItemHasSubList(
          this.name,
          editor.state,
          prevNode
        );

        // if the previous item is a list item and doesn't have a sublist, join the list items
        if (
          hasPreviousListItem(this.name, editor.state) &&
          !previousListItemHasSubList
        ) {
          return editor.commands.joinListItemBackward();
        }

        // otherwise in the end, a backspace should
        // always just lift the list item if
        // joining / merging is not possible
        return editor.chain().liftListItem(this.name).run();
      },
    };
  },
});