This class exhibits a case where the new JDK 11 work-stealing algorithm creates an issue. The code works with JDK 8.

task_tree.svg

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
 "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Title: G Pages: 1 -->
<svg width="721pt" height="337pt"
 viewBox="0.00 0.00 721.45 337.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 333)">
<title>G</title>
<polygon fill="white" stroke="none" points="-4,4 -4,-333 717.445,-333 717.445,4 -4,4"/>
<!-- root -->
<g id="node1" class="node"><title>root</title>
<ellipse fill="none" stroke="blue" cx="234.945" cy="-311" rx="27" ry="18"/>
</g>
<!-- p1 -->
<g id="node2" class="node"><title>p1</title>
<ellipse fill="none" stroke="blue" cx="194.945" cy="-224" rx="27" ry="18"/>
</g>
<!-- root&#45;&gt;p1 -->
<g id="edge1" class="edge"><title>root&#45;&gt;p1</title>
<path fill="none" stroke="black" d="M227.233,-293.611C221.478,-281.382 213.507,-264.443 206.925,-250.456"/>
<polygon fill="black" stroke="black" points="210.059,-248.896 202.634,-241.339 203.725,-251.877 210.059,-248.896"/>
<text text-anchor="middle" x="224.945" y="-263.8" font-family="Times New Roman,serif" font-size="14.00">2&#45; invokes</text>
</g>
<!-- p2 -->
<g id="node3" class="node"><title>p2</title>
<ellipse fill="none" stroke="red" cx="367.945" cy="-224" rx="33.5952" ry="18"/>
<text text-anchor="middle" x="367.945" y="-220.3" font-family="Times New Roman,serif" font-size="14.00">Stolen</text>
</g>
<!-- root&#45;&gt;p2 -->
<g id="edge2" class="edge"><title>root&#45;&gt;p2</title>
<path fill="none" stroke="black" d="M253.719,-298.002C275.753,-283.92 312.562,-260.395 338.427,-243.865"/>
<polygon fill="black" stroke="black" points="340.536,-246.671 347.077,-238.337 336.766,-240.773 340.536,-246.671"/>
<text text-anchor="middle" x="332.445" y="-263.8" font-family="Times New Roman,serif" font-size="14.00">1&#45; forks</text>
</g>
<!-- lt1 -->
<g id="node4" class="node"><title>lt1</title>
<ellipse fill="none" stroke="blue" cx="102.945" cy="-137" rx="64.189" ry="18"/>
<text text-anchor="middle" x="102.945" y="-133.3" font-family="Times New Roman,serif" font-size="14.00">Locking task 1</text>
</g>
<!-- p1&#45;&gt;lt1 -->
<g id="edge3" class="edge"><title>p1&#45;&gt;lt1</title>
<path fill="none" stroke="black" d="M179.753,-208.964C165.806,-196.078 144.866,-176.731 128.371,-161.491"/>
<polygon fill="black" stroke="black" points="130.4,-158.601 120.68,-154.386 125.65,-163.743 130.4,-158.601"/>
<text text-anchor="middle" x="183.945" y="-176.8" font-family="Times New Roman,serif" font-size="14.00">3&#45; invokes</text>
</g>
<!-- lt2 -->
<g id="node5" class="node"><title>lt2</title>
<ellipse fill="none" stroke="blue" cx="367.945" cy="-137" rx="64.189" ry="18"/>
<text text-anchor="middle" x="367.945" y="-133.3" font-family="Times New Roman,serif" font-size="14.00">Locking task 2</text>
</g>
<!-- p2&#45;&gt;lt2 -->
<g id="edge4" class="edge"><title>p2&#45;&gt;lt2</title>
<path fill="none" stroke="black" d="M367.945,-205.799C367.945,-194.163 367.945,-178.548 367.945,-165.237"/>
<polygon fill="black" stroke="black" points="371.445,-165.175 367.945,-155.175 364.445,-165.175 371.445,-165.175"/>
<text text-anchor="middle" x="390.445" y="-176.8" font-family="Times New Roman,serif" font-size="14.00">5&#45; forks</text>
</g>
<!-- lt3 -->
<g id="node6" class="node"><title>lt3</title>
<ellipse fill="none" stroke="red" cx="514.945" cy="-137" rx="64.189" ry="18"/>
<text text-anchor="middle" x="514.945" y="-133.3" font-family="Times New Roman,serif" font-size="14.00">Locking task 3</text>
</g>
<!-- p2&#45;&gt;lt3 -->
<g id="edge5" class="edge"><title>p2&#45;&gt;lt3</title>
<path fill="none" stroke="black" d="M389.953,-210.275C413.658,-196.568 451.724,-174.556 479.627,-158.422"/>
<polygon fill="black" stroke="black" points="481.393,-161.444 488.298,-153.408 477.889,-155.384 481.393,-161.444"/>
<text text-anchor="middle" x="471.445" y="-176.8" font-family="Times New Roman,serif" font-size="14.00">invokes</text>
</g>
<!-- lt1&#45;&gt;lt2 -->
<g id="edge8" class="edge"><title>lt1&#45;&gt;lt2</title>
<path fill="none" stroke="black" stroke-dasharray="1,5" d="M167.433,-137C205.344,-137 253.448,-137 293.036,-137"/>
<polygon fill="black" stroke="black" points="293.169,-140.5 303.169,-137 293.169,-133.5 293.169,-140.5"/>
<text text-anchor="middle" x="235.445" y="-158.8" font-family="Times New Roman,serif" font-size="14.00">7&#45; steals while joining</text>
<text text-anchor="middle" x="235.445" y="-143.8" font-family="Times New Roman,serif" font-size="14.00"> the stolen child</text>
</g>
<!-- st1 -->
<g id="node7" class="node"><title>st1</title>
<ellipse fill="none" stroke="yellow" cx="53.9452" cy="-34" rx="53.8905" ry="18"/>
<text text-anchor="middle" x="53.9452" y="-30.3" font-family="Times New Roman,serif" font-size="14.00">Stolen child</text>
</g>
<!-- lt1&#45;&gt;st1 -->
<g id="edge6" class="edge"><title>lt1&#45;&gt;st1</title>
<path fill="none" stroke="black" d="M94.633,-118.867C86.9489,-103.028 75.4195,-79.2633 66.5934,-61.0708"/>
<polygon fill="black" stroke="black" points="69.6486,-59.3496 62.1346,-51.8802 63.3506,-62.4051 69.6486,-59.3496"/>
<text text-anchor="middle" x="108.445" y="-89.8" font-family="Times New Roman,serif" font-size="14.00">4&#45; forks</text>
</g>
<!-- st2 -->
<g id="node8" class="node"><title>st2</title>
<ellipse fill="none" stroke="blue" cx="152.945" cy="-34" rx="27" ry="18"/>
</g>
<!-- lt1&#45;&gt;st2 -->
<g id="edge7" class="edge"><title>lt1&#45;&gt;st2</title>
<path fill="none" stroke="black" d="M111.427,-118.867C119.362,-102.837 131.317,-78.6895 140.363,-60.4157"/>
<polygon fill="black" stroke="black" points="143.619,-61.7282 144.919,-51.2134 137.345,-58.6225 143.619,-61.7282"/>
<text text-anchor="middle" x="164.945" y="-89.8" font-family="Times New Roman,serif" font-size="14.00">6&#45; invokes</text>
</g>
<!-- notes -->
<g id="node9" class="node"><title>notes</title>
<text text-anchor="start" x="206.445" y="-52.8" font-family="Times New Roman,serif" font-size="14.00">Notes:</text>
<text text-anchor="start" x="206.445" y="-37.8" font-family="Times New Roman,serif" font-size="14.00">The locking tasks share the same lock.</text>
<text text-anchor="start" x="206.445" y="-22.8" font-family="Times New Roman,serif" font-size="14.00">Locking task 1 executes the critcal section of the Locking task 2 within its critical section.</text>
<text text-anchor="start" x="206.445" y="-7.8" font-family="Times New Roman,serif" font-size="14.00">The color of a task depends of the thread executing it.</text>
</g>
</g>
</svg>

WorkStealingWhileLocking.java

import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.RecursiveAction;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

/**
 * This class exhibits a case where the new JDK 11 work-stealing algorithm creates an issue.
 * <p>
 * A thread steals a task in a critical section and executes its critical section: so 2 critical
 * sections are executed at the same time by the same thread.
 * <p>
 * Here is a description of what is happening (using a family analogy):
 * <ol>
 * <li>A {@link LockingTask task} has its uncle {@link ParentTask task} stolen by a thread.</li>
 * <li>Then it takes a lock and creates new {@link SubTask subtasks}.</li>
 * <li>One of its {@link SubTask subtasks} is stolen by another thread.</li>
 * <li>When joining the stolen {@link SubTask subtask}, it helps its uncle and steals a cousin
 * {@link LockingTask task} (this was not the case with JDK 8).</li>
 * <li>The cousin {@link LockingTask task} is executed and the issue happens as this task use the
 * same lock and the thread already own the lock.</li>
 * </ol>
 * <p>
 * This code works with JDK 8.
 */
public class WorkStealingWhileLocking {

	public static void main(String[] args) throws Exception {
		new ForkJoinPool(3).submit(new RecursiveAction() {

			@Override
			protected void compute() {
				log("creates the ParentTasks");
				invokeAll(new ParentTask(), new ParentTask());
			}
		}).get();

	}

	/**
	 * Used to execute the the threads' operations in the right order to reproduce the issue.
	 */
	static volatile State STATE = State.S0_START;

	static enum State {
		S0_START, S1_PARENT_TASK_IS_STOLEN, S2_SUBTASK_IS_STOLEN, S3_CAN_JOIN_SUBTASK;
	}

	/**
	 * Object shared by the {@link LockingTask}.
	 * <p>
	 * The methods of this class must be called after aquiring the lock.
	 */
	static class Data {

		protected final Lock lock = new ReentrantLock();
		// value1 and value2 must be equals when accessing them with the lock
		protected int value1 = 0;
		protected int value2 = 0;

		public void operation1() {
			++value1;
			if (value1 == 1) {
				maintenanceOperation();
			}
		}

		// always called after each call of operation1
		public void operation2() {
			++value2;
			if (value2 != value1) {
				String msg = Thread.currentThread().getName() + ": IllegalState  value1 != value2";
				System.out.println(msg);
				throw new IllegalStateException(msg);
			}
		}

		private void maintenanceOperation() {
			log("forks a SubTask");
			final SubTask t = new SubTask();
			t.fork();

			// Wait for t to be stolen
			while (STATE != State.S2_SUBTASK_IS_STOLEN) {
			}

			new SubTask().invoke();

			// Wait to have work to steal
			while (STATE != State.S3_CAN_JOIN_SUBTASK) {
			}

			// t was stolen
			log("joins the forked SubTask");
			t.join();
		}
	}

	static class ParentTask extends Task {

		@Override
		protected void doCompute() throws InterruptedException {
			// In the real case this class creates a variable number of LockingTasks and use
			// invokeAll to execute them
			if (isStolen()) {
				assert STATE == State.S0_START : STATE;
				STATE = State.S1_PARENT_TASK_IS_STOLEN;
				while (STATE != State.S2_SUBTASK_IS_STOLEN) {
				}

				log("forks a LockingTask");
				final LockingTask t = new LockingTask();
				t.fork();

				STATE = State.S3_CAN_JOIN_SUBTASK;
				new LockingTask().invoke();

				// Need to wait so that the forked LockingTask can be stolen in JDK 11
				Thread.sleep(2000);
				log("joins the forked LockingTask");
				t.join();

			} else {
				while (STATE != State.S1_PARENT_TASK_IS_STOLEN) {
				}
				new LockingTask().invoke();
			}
		}
	}

	static class LockingTask extends Task {
		static final Data DATA = new Data();

		@Override
		protected void doCompute() {
			// In the real case this class does most of its computations outside the lock and then
			// contributes its result in a shared object.
			log("will take the lock");
			// The critical section is below
			DATA.lock.lock();
			try {
				log("took the lock");
				DATA.operation1();
				DATA.operation2();
			} finally {
				log("releases the lock");
				DATA.lock.unlock();
			}
		}
	}

	/**
	 * Task executed by {@link Data#operation1()} in the critical section of {@link LockingTask}.
	 */
	static class SubTask extends Task {
		@Override
		protected void doCompute() throws InterruptedException {
			if (isStolen()) {
				assert STATE == State.S1_PARENT_TASK_IS_STOLEN : STATE;
				STATE = State.S2_SUBTASK_IS_STOLEN;
				// Need to wait so that a LockingTask can be stolen in JDK 11
				Thread.sleep(2000);
			} else {
				// Nothing interesting to do
			}
		}
	}

	private static void log(String action) {
		System.out.println(Thread.currentThread().getName() + " " + action);
	}

	/**
	 * Used to log what happens
	 */
	static abstract class Task extends RecursiveAction {

		/** The thread which created this task */
		final Thread creator;

		public Task() {
			this.creator = Thread.currentThread();
		}

		@Override
		protected final void compute() {
			if (isStolen()) {
				log("steals a " + getClass().getSimpleName() + " of " + creator.getName());
			} else {
				log("executes a " + getClass().getSimpleName());
			}
			try {
				doCompute();
			} catch (Exception e) {
				completeExceptionally(e);
			} finally {
				log("finishes a " + getClass().getSimpleName());
			}
		}

		protected boolean isStolen() {
			return Thread.currentThread() != creator;
		}

		protected abstract void doCompute() throws InterruptedException;
	}

}