JUnit 4 Runner to run a test with all possible feature toggle combinations

ToggleFeaturesRunner.java

import org.junit.internal.AssumptionViolatedException;
import org.junit.runners.BlockJUnit4ClassRunner;
import org.junit.runners.model.FrameworkMethod;
import org.junit.runners.model.InitializationError;
import org.junit.runners.model.Statement;
import org.junit.runners.model.TestClass;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.*;

import static pero.common.EnumMaps.newEnumMap;

/**
 * <p>Behaves like {@link BlockJUnit4ClassRunner} except when a test method is
 * annotated with {@link ToggleFeatures}. In this case the test will be
 * executed with every possible combination of the specified features.</p>
 * <p>Note: You need to call the {@link #getCurrentFeatureCombination} method
 * in your test (or in your test fixture) and set up our test environment
 * according to the returned feature combination.</p>
 */
public class ToggleFeaturesRunner extends BlockJUnit4ClassRunner {

    /**
     * This annotation indicates to the {@link ToggleFeaturesRunner} that
     * the annotated test method should be executed using all possible
     * enabled/disabled combinations of the features given in the value.
     */
    @Retention(RetentionPolicy.RUNTIME)
    @Target(ElementType.METHOD)
    public @interface ToggleFeatures {
        /**
         * The features that should be used to provide the combinations of
         * enabled and disabled features.
         */
        Feature[] value();
    }

    private static Map<Feature, Boolean> c_currentFeatureCombination = Collections.emptyMap();

    /**
     * You need to call this method in your test (or in your test fixture)
     * and set up our test environment according to the returned feature combination.
     */
    public static Map<Feature, Boolean> getCurrentFeatureCombination() {
        return c_currentFeatureCombination;
    }

    public ToggleFeaturesRunner(Class<?> klass) throws InitializationError {
        super(klass);
    }

    @Override
    protected Statement methodBlock(FrameworkMethod method) {
        ToggleFeatures toggleFeatures = method.getAnnotation(ToggleFeatures.class);
        if (toggleFeatures != null && toggleFeatures.value().length > 0) {
            return new TryAllPossibleFeatureCombinations(getTestClass(), method, toggleFeatures.value());
        } else {
            return super.methodBlock(method);
        }
    }

    private static class TryAllPossibleFeatureCombinations extends Statement {

        private final TestClass _testClass;
        private final FrameworkMethod _testMethod;
        private final Feature[] _features;

        private TryAllPossibleFeatureCombinations(TestClass testClass, FrameworkMethod testMethod, Feature[] features) {
            _testClass = testClass;
            _testMethod = testMethod;
            _features = features;
        }

        @Override
        public void evaluate() throws Throwable {
            List<AssumptionViolatedException> assumptionViolatedExceptions = new ArrayList();
            List<Map<Feature, Boolean>> allPossibleFeatureCombinations = getAllPossibleFeatureCombinations();
            for (Map<Feature, Boolean> featureCombination : allPossibleFeatureCombinations) {
                try {
                    runUsing(featureCombination);
                } catch (AssumptionViolatedException e) {
                    assumptionViolatedExceptions.add(e);
                }
                // Note: we do not catch any other exceptions here, so the test will fail fast.
            }
            if (assumptionViolatedExceptions.size() == allPossibleFeatureCombinations.size()) {
                throw new AssumptionViolatedException(_testMethod.getName() + "() ignored for all possible combinations of " + Arrays.toString(_features));
            }
        }

        private List<Map<Feature, Boolean>> getAllPossibleFeatureCombinations() {
            List<Map<Feature, Boolean>> result = new ArrayList();
            result.add(newEnumMap(_features[0], Boolean.FALSE));
            result.add(newEnumMap(_features[0], Boolean.TRUE));
            for (int i = 1; i < _features.length; ++i) {
                List<Map<Feature, Boolean>> temp = result;
                result = new ArrayList<Map<Feature, Boolean>>();
                for (Map<Feature, Boolean> incompleteCombination : temp) {
                    Map<Feature, Boolean> enhancedCombination1 = new EnumMap<Feature, Boolean>(incompleteCombination);
                    enhancedCombination1.put(_features[i], Boolean.FALSE);
                    result.add(enhancedCombination1);
                    Map<Feature, Boolean> enhancedCombination2 = new EnumMap<Feature, Boolean>(incompleteCombination);
                    enhancedCombination2.put(_features[i], Boolean.TRUE);
                    result.add(enhancedCombination2);
                }
            }
            return result;
        }

        private void runUsing(final Map<Feature, Boolean> featureCombination) throws Throwable {
            c_currentFeatureCombination = featureCombination;
            try {
                new BlockJUnit4ClassRunner(_testClass.getJavaClass()) {
                    @Override
                    protected Statement methodBlock(final FrameworkMethod method) {
                        final Statement runTest = super.methodBlock(method);
                        return new Statement() {
                            @Override
                            public void evaluate() throws Throwable {
                                try {
                                    runTest.evaluate();
                                } catch (AssumptionViolatedException e) {
                                    throw e;
                                } catch (Throwable t) {
                                    throw new RuntimeException(method.getName() + "() failed with feature combination " + featureCombination, t);
                                }
                            }
                        };
                    }
                }.methodBlock(_testMethod).evaluate();
            } finally {
                c_currentFeatureCombination = Collections.emptyMap();
            }
        }
    }
}

Note: This class does not compile, as long as you don't have an enum Feature.
But suppose you have one like this:

public enum Feature {
    coolFeature1, awesomeFeature2, greatFeature3
}

Then you could write the following test class:

@RunWith(ToggleFeaturesRunner.class)
public class FooTest {
    @Test
    @ToggleFeatures({ coolFeature1, awesomeFeature2 })
    public void test() {
        // Setup test environment using ToggleFeaturesRunner.getCurrentFeatureCombination()
        // ...
    }
}

and the test method would be called 4 times for all possible combinations for the 2 specified features.