Draft of CTR4J's API for testing frameworks

CTR4J API Draft.java

/**
 * Implemented by each testing framework and given as a parameter to the @RunWith annotation.
 */
public interface TestRunner {

    /**
     * The Executor should be used to run the tests, so that they can be executed in parallel,
     * each test in a different thread. If the Runnables passed to the Executor are Serializable,
     * then each of the tests in one class could potentially be executed on different machine in
     * a server cluster. Otherwise any potential clustering is at class-granularity (which may be
     * a hindrance for classes with many slow tests).
     */
    void findTests(Class<?> testClass, SuiteNotifier notifier, java.util.concurrent.Executor executor);
}


public interface SuiteNotifier {

    /**
     * Must be called before starting the test. Idempotent. IDEs can use the
     * Class or Method object to implement code navigation.
     */
    void fireTestFound(Path path, String name, Class<?> location);

    void fireTestFound(Path path, String name, java.lang.reflect.Method location);

    /**
     * May be called multiple times, before a test is finished, to produce nested tests.
     * The only limitation is that a nested test must be finished before the surrounding tests.
     * Everything printed to System.out and System.err after the call to this method will be
     * recorded from the current thread and threads which are started by it (possibly together
     * with timestamps and name of the thread which printed it).
     */
    TestNotifier fireTestStarted(Path path);
}


public interface TestNotifier {

    /**
     * May be called multiple times per test (although most testing frameworks
     * will stop execution on the first exception).
     */
    void fireFailure(Throwable cause);

    /**
     * Must be called last, exactly once. The test will fail if fireFailure() had
     * been called at least once, otherwise the test will pass. It is a runtime error
     * to call fireFailure() or fireTestFinished() after calling this method. If the
     * test started any threads, will wait for them to finish (except the AWT event
     * thread and maybe some others). It is an error (or at least a warning) for a
     * test to start threads without stopping them.
     */
    void fireTestFinished();
}


/**
 * Uniquely identifies a single test in the tree of all tests. Immutable.
 */
public final class Path {

    public static final Path ROOT = Path.to();

    public static Path to(int... indices) {
    }

    public Path firstChild() {
    }

    public Path nextSibling() {
    }

    public Path parent() {
    }

    // plus some more helper methods such as isRoot(), isFirstChild() etc.
}