testInstrumentationRunner (replacement for android.support.test.runner.AndroidJUnitRunner) and a org.junit.runner.Runner (replacement for cucumber.api.junit.Cucumber) that combines cucumber-android (2.40) and cucumber-junit (2.40) to run scenarios isolated in seperate processes via orchestrator 1.0.2

Cucumber.java

package cucumber.runtime.android;

import android.content.Context;
import android.support.test.InstrumentationRegistry;

import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.runner.Description;
import org.junit.runner.notification.RunNotifier;
import org.junit.runners.ParentRunner;
import org.junit.runners.model.InitializationError;
import org.junit.runners.model.Statement;

import java.io.IOException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import cucumber.api.CucumberOptions;
import cucumber.api.Plugin;
import cucumber.api.SnippetType;
import cucumber.api.StepDefinitionReporter;
import cucumber.api.SummaryPrinter;
import cucumber.api.event.TestRunFinished;
import cucumber.api.event.TestRunStarted;
import cucumber.api.formatter.Formatter;
import cucumber.api.java.ObjectFactory;
import cucumber.runner.EventBus;
import cucumber.runtime.CucumberException;
import cucumber.runtime.RuntimeOptionsFactory;
import cucumber.runtime.io.MultiLoader;
import cucumber.runtime.io.ResourceLoaderClassFinder;
import cucumber.runtime.junit.Assertions;
import cucumber.runtime.junit.FeatureRunner;
import cucumber.runtime.junit.JUnitOptions;
import cucumber.runtime.junit.JUnitReporter;
import cucumber.runtime.model.CucumberFeature;
import dalvik.system.DexFile;

import cucumber.runtime.Backend;
import cucumber.runtime.ClassFinder;
import cucumber.runtime.Env;
import cucumber.runtime.Runtime;
import cucumber.runtime.RuntimeOptions;
import cucumber.runtime.io.ResourceLoader;
import cucumber.runtime.java.JavaBackend;
import cucumber.runtime.java.ObjectFactoryLoader;

import static cucumber.runtime.model.CucumberFeature.load;

/**
 * <p>
 * Classes annotated with {@code @RunWith(Cucumber.class)} will run a Cucumber Feature.
 * In general, the runner class should be empty without any fields or methods.
 * For example:
 *
 * <blockquote><pre>
 * &#64;RunWith(Cucumber.class)
 * &#64;CucumberOptions(plugin = "pretty")
 * public class RunCukesTest {
 *
 * }
 * </pre></blockquote>
 * <p>
 * Cucumber will look for a {@code .feature} file on the classpath, using the same resource
 * path as the annotated class ({@code .class} substituted by {@code .feature}).
 * <p>
 * Additional hints can be given to Cucumber by annotating the class with {@link CucumberOptions}.
 * <p>
 * Cucumber also supports JUnits {@link ClassRule}, {@link BeforeClass} and {@link AfterClass} annotations.
 * These will be invoked around the suite of features" and moved to the end of the java doc.
 *
 * @see CucumberOptions
 */
public class Cucumber extends ParentRunner<Cucumber.FeatureRunner> {
    private final JUnitReporter jUnitReporter;
    private final List<FeatureRunner> children = new ArrayList<FeatureRunner>();
    private final Runtime runtime;
    private final Formatter formatter;

    /**
     * Constructor called by JUnit.
     *
     * @param clazz the class with the @RunWith annotation.
     * @throws java.io.IOException                         if there is a problem
     * @throws org.junit.runners.model.InitializationError if there is another problem
     */
    public Cucumber(Class clazz) throws InitializationError, IOException {
        this(clazz, null);
    }

    public Cucumber(Class clazz, String methodName) throws InitializationError, IOException {
        super(clazz);
        Context context = InstrumentationRegistry.getContext();
        ClassLoader classLoader = context.getClassLoader();
        Assertions.assertNoCucumberAnnotatedMethods(clazz);

        RuntimeOptionsFactory runtimeOptionsFactory = new RuntimeOptionsFactory(clazz);
        final RuntimeOptions originalRuntimeOptions = runtimeOptionsFactory.create();
        RuntimeOptions runtimeOptions = originalRuntimeOptions;
        if(methodName != null) {
            runtimeOptions = new RuntimeOptions(Collections.emptyList()) {
                @Override
                public List<CucumberFeature> cucumberFeatures(ResourceLoader resourceLoader, EventBus bus) {
                    List<CucumberFeature> features = load(resourceLoader, getFeaturePaths(), System.out);
                    getPlugins(); // to create the formatter objects
                    bus.send(new TestRunStarted(bus.getTime()));
                    for (CucumberFeature feature : features) {
                        feature.sendTestSourceRead(bus);
                    }
                    return features;
                }

                @Override
                public List<Plugin> getPlugins() {
                    return originalRuntimeOptions.getPlugins();
                }

                @Override
                public Formatter formatter(ClassLoader classLoader) {
                    return originalRuntimeOptions.formatter(classLoader);
                }

                @Override
                public StepDefinitionReporter stepDefinitionReporter(ClassLoader classLoader) {
                    return originalRuntimeOptions.stepDefinitionReporter(classLoader);
                }

                @Override
                public SummaryPrinter summaryPrinter(ClassLoader classLoader) {
                    return originalRuntimeOptions.summaryPrinter(classLoader);
                }

                @Override
                public List<String> getGlue() {
                    return originalRuntimeOptions.getGlue();
                }

                @Override
                public boolean isStrict() {
                    return originalRuntimeOptions.isStrict();
                }

                @Override
                public boolean isDryRun() {
                    return originalRuntimeOptions.isDryRun();
                }

                @Override
                public List<String> getFeaturePaths() {
                    return Collections.singletonList(URLDecoder.decode(methodName.substring(0, methodName.indexOf(':'))));
                }

                @Override
                public void addPlugin(Formatter plugin) {
                    originalRuntimeOptions.addPlugin(plugin);
                }

                @Override
                public List<Pattern> getNameFilters() {
                    return Collections.singletonList(Pattern.compile("^" + methodName.substring(methodName.indexOf(':')+1) + "$"));
                }

                @Override
                public List<String> getTagFilters() {
                    return originalRuntimeOptions.getTagFilters();
                }

                @Override
                public Map<String, List<Long>> getLineFilters(ResourceLoader resourceLoader) {
                    return originalRuntimeOptions.getLineFilters(resourceLoader);
                }

                @Override
                public boolean isMonochrome() {
                    return originalRuntimeOptions.isMonochrome();
                }

                @Override
                public SnippetType getSnippetType() {
                    return originalRuntimeOptions.getSnippetType();
                }

                @Override
                public List<String> getJunitOptions() {
                    return originalRuntimeOptions.getJunitOptions();
                }
            };
        }

        ResourceLoader resourceLoader = new AndroidResourceLoader(context);
        runtime = createRuntime(resourceLoader, classLoader, runtimeOptions);
        formatter = runtimeOptions.formatter(classLoader);
        final JUnitOptions junitOptions = new JUnitOptions(runtimeOptions.getJunitOptions());
        final List<CucumberFeature> cucumberFeatures = runtimeOptions.cucumberFeatures(resourceLoader, runtime.getEventBus());
        jUnitReporter = new JUnitReporter(runtime.getEventBus(), runtimeOptions.isStrict(), junitOptions);
        addChildren(cucumberFeatures);
    }

    /**
     * Create the Runtime. Can be overridden to customize the runtime or backend.
     *
     * @param resourceLoader used to load resources
     * @param classLoader    used to load classes
     * @param runtimeOptions configuration
     * @return a new runtime
     * @throws InitializationError if a JUnit error occurred
     * @throws IOException         if a class or resource could not be loaded
     * @deprecated Neither the runtime nor the backend or any of the classes involved in their construction are part of
     * the public API. As such they should not be  exposed. The recommended way to observe the cucumber process is to
     * listen to events by using a plugin. For example the JSONFormatter.
     */
    protected Runtime createRuntime(ResourceLoader resourceLoader, ClassLoader classLoader, RuntimeOptions runtimeOptions) throws InitializationError, IOException {
        return new Runtime(resourceLoader, classLoader, createBackends(createDexClassFinder(InstrumentationRegistry.getContext())) , runtimeOptions);
    }


    @Override
    public List<FeatureRunner> getChildren() {
        return children;
    }

    @Override
    protected Description describeChild(FeatureRunner child) {
        Description original = child.getDescription();
        Description translated = original.childlessCopy();
        for(Description description : original.getChildren()) {
            translated.addChild(Description.createTestDescription(getTestClass().getJavaClass(), URLEncoder.encode(child.cucumberFeature.getUri()) + ':' + description.getMethodName()));
        }
        return translated;
    }

    @Override
    protected void runChild(FeatureRunner child, RunNotifier notifier) {
        child.run(notifier);
    }

    @Override
    protected Statement childrenInvoker(RunNotifier notifier) {
        final Statement features = super.childrenInvoker(notifier);
        return new Statement() {
            @Override
            public void evaluate() throws Throwable {
                features.evaluate();
                runtime.getEventBus().send(new TestRunFinished(runtime.getEventBus().getTime()));
                runtime.printSummary();
            }
        };
    }

    private void addChildren(List<CucumberFeature> cucumberFeatures) throws InitializationError {
        for (CucumberFeature cucumberFeature : cucumberFeatures) {
            FeatureRunner featureRunner = new FeatureRunner(cucumberFeature, runtime, jUnitReporter);
            if (!featureRunner.isEmpty()) {
                children.add(featureRunner);
            }
        }
    }

    private ClassFinder createDexClassFinder(final Context context) {
        final String apkPath = context.getPackageCodePath();
        return new DexClassFinder(newDexFile(apkPath));
    }

    private DexFile newDexFile(final String apkPath) {
        try {
            return new DexFile(apkPath);
        } catch (final IOException e) {
            throw new CucumberException("Failed to open " + apkPath);
        }
    }

    private Collection<? extends Backend> createBackends(ClassFinder classFinder) {
        final ObjectFactory delegateObjectFactory = ObjectFactoryLoader.loadObjectFactory(classFinder, Env.INSTANCE.get(ObjectFactory.class.getName()));
        final AndroidObjectFactory objectFactory = new AndroidObjectFactory(delegateObjectFactory, InstrumentationRegistry.getInstrumentation());
        final List<Backend> backends = new ArrayList<>();
        backends.add(new JavaBackend(objectFactory, classFinder));
        return backends;
    }

    public static class FeatureRunner extends cucumber.runtime.junit.FeatureRunner {

        private final CucumberFeature cucumberFeature;

        public FeatureRunner(CucumberFeature cucumberFeature, Runtime runtime, JUnitReporter jUnitReporter) throws InitializationError {
            super(cucumberFeature, runtime, jUnitReporter);
            this.cucumberFeature = cucumberFeature;
        }
    }

}

CucumberRunner.java

package cucumber.runtime.android;

import static android.support.test.internal.util.ReflectionUtil.reflectivelyInvokeRemoteMethod;

import android.app.Activity;
import android.app.Application;
import android.app.Instrumentation;
import android.content.Context;
import android.os.Build;
import android.os.Bundle;
import android.os.Debug;
import android.support.annotation.VisibleForTesting;
import android.support.test.internal.runner.RunnerArgs;
import android.support.test.internal.runner.TestExecutor;
import android.support.test.internal.runner.TestRequestBuilder;
import android.support.test.internal.runner.listener.ActivityFinisherRunListener;
import android.support.test.internal.runner.listener.CoverageListener;
import android.support.test.internal.runner.listener.DelayInjector;
import android.support.test.internal.runner.listener.InstrumentationResultPrinter;
import android.support.test.internal.runner.listener.LogRunListener;
import android.support.test.internal.runner.listener.SuiteAssignmentPrinter;
import android.support.test.internal.runner.tracker.AnalyticsBasedUsageTracker;
import android.support.test.internal.runner.tracker.UsageTrackerRegistry.AtslVersions;
import android.support.test.runner.MonitoringInstrumentation;
import android.support.test.orchestrator.instrumentationlistener.OrchestratedInstrumentationListener;
import android.support.test.orchestrator.instrumentationlistener.OrchestratedInstrumentationListener.OnConnectListener;
import android.support.test.runner.UsageTrackerFacilitator;
import android.support.test.runner.lifecycle.ApplicationLifecycleCallback;
import android.support.test.runner.lifecycle.ApplicationLifecycleMonitorRegistry;
import android.support.test.runner.screenshot.ScreenCaptureProcessor;
import android.support.test.runner.screenshot.Screenshot;
import android.util.Log;
import java.util.HashSet;
import org.junit.runner.Request;
import org.junit.runner.notification.RunListener;

import cucumber.runtime.android.Cucumber;

public class CucumberRunner extends MonitoringInstrumentation implements OnConnectListener {

    private static final String LOG_TAG = "AndroidJUnitRunner";

    private Bundle mArguments;
    private InstrumentationResultPrinter mInstrumentationResultPrinter =
            new InstrumentationResultPrinter();
    private RunnerArgs mRunnerArgs;
    private UsageTrackerFacilitator mUsageTrackerFacilitator;
    private OrchestratedInstrumentationListener mOrchestratorListener;

    @Override
    public void onCreate(Bundle arguments) {
        mArguments = arguments;
        parseRunnerArgs(mArguments);

        if (mRunnerArgs.debug) {
            Log.i(LOG_TAG, "Waiting for debugger to connect...");
            Debug.waitForDebugger();
            Log.i(LOG_TAG, "Debugger connected.");
        }

        // We are only interested in tracking usage of the primary process.
        if (isPrimaryInstrProcess(mRunnerArgs.targetProcess)) {
            mUsageTrackerFacilitator = new UsageTrackerFacilitator(mRunnerArgs);
        } else {
            mUsageTrackerFacilitator = new UsageTrackerFacilitator(false);
        }

        super.onCreate(arguments);

        for (ApplicationLifecycleCallback listener : mRunnerArgs.appListeners) {
            ApplicationLifecycleMonitorRegistry.getInstance().addLifecycleCallback(listener);
        }

        addScreenCaptureProcessors(mRunnerArgs);

        if (mRunnerArgs.orchestratorService != null
                && isPrimaryInstrProcess(mRunnerArgs.targetProcess)) {
            // If orchestratorService is provided, and we are the primary process
            // we await onOrchestratorConnect() before we start().
            mOrchestratorListener = new OrchestratedInstrumentationListener(this);
            mOrchestratorListener.connect(getContext());
        } else {
            // If no orchestration service is given, or we are not the primary process we can
            // start() immediately.
            start();
        }
    }

    /**
     * Called when AndroidJUnitRunner connects to a test orchestrator, if the {@code
     * orchestratorService} parameter is set.
     *
     * @hide
     */
    @Override
    public void onOrchestratorConnect() {
        start();
    }

    /**
     * Build the arguments.
     *
     * <p>Read from manifest first so manifest-provided args can be overridden with command line
     * arguments
     *
     * @param arguments
     */
    private void parseRunnerArgs(Bundle arguments) {
        mRunnerArgs = new RunnerArgs.Builder().fromManifest(this).fromBundle(arguments).build();
    }

    /**
     * Get the Bundle object that contains the arguments passed to the instrumentation
     *
     * @return the Bundle object
     */
    private Bundle getArguments() {
        return mArguments;
    }

    @VisibleForTesting
    InstrumentationResultPrinter getInstrumentationResultPrinter() {
        return mInstrumentationResultPrinter;
    }

    @Override
    public void onStart() {
        setJsBridgeClassName("android.support.test.espresso.web.bridge.JavaScriptBridge");
        super.onStart();

        /*
         * The orchestrator cannot collect the list of tests as it is running in a different process
         * than the test app.  On first run, the Orchestrator will ask AJUR to list the tests
         * out that would be run for a given class parameter.  AJUR will then be successively
         * called with whatever it passes back to the orchestratorListener.
         */
        if (mRunnerArgs.listTestsForOrchestrator && isPrimaryInstrProcess(mRunnerArgs.targetProcess)) {
            Request testRequest = buildRequest(mRunnerArgs, getArguments());
            mOrchestratorListener.addTests(testRequest.getRunner().getDescription());
            finish(Activity.RESULT_OK, new Bundle());
            return;
        }

        if (mRunnerArgs.remoteMethod != null) {
            reflectivelyInvokeRemoteMethod(
                    mRunnerArgs.remoteMethod.testClassName, mRunnerArgs.remoteMethod.methodName);
        }

        if (!isPrimaryInstrProcess(mRunnerArgs.targetProcess)) {
            Log.i(LOG_TAG, "Runner is idle...");
            return;
        }

        Bundle results = new Bundle();
        try {
            TestExecutor.Builder executorBuilder = new TestExecutor.Builder(this);

            addListeners(mRunnerArgs, executorBuilder);

            try {
                Cucumber cucumber = new Cucumber(Class.forName(mRunnerArgs.tests.get(0).testClassName), mRunnerArgs.tests.get(0).methodName);
                Request testRequest = Request.runner(cucumber.getChildren().get(0));
                results = executorBuilder.build().execute(testRequest);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        } catch (RuntimeException e) {
            final String msg = "Fatal exception when running tests";
            Log.e(LOG_TAG, msg, e);
            // report the exception to instrumentation out
            results.putString(
                    Instrumentation.REPORT_KEY_STREAMRESULT, msg + "\n" + Log.getStackTraceString(e));
        }
        finish(Activity.RESULT_OK, results);
    }

    @Override
    public void finish(int resultCode, Bundle results) {
        try {
            mUsageTrackerFacilitator.trackUsage("AndroidJUnitRunner", AtslVersions.RUNNER_VERSION);
            mUsageTrackerFacilitator.sendUsages();
        } catch (RuntimeException re) {
            Log.w(LOG_TAG, "Failed to send analytics.", re);
        }
        super.finish(resultCode, results);
    }

    @VisibleForTesting
    final void addListeners(RunnerArgs args, TestExecutor.Builder builder) {
        if (args.newRunListenerMode) {
            addListenersNewOrder(args, builder);
        } else {
            addListenersLegacyOrder(args, builder);
        }
    }

    private void addListenersLegacyOrder(RunnerArgs args, TestExecutor.Builder builder) {
        if (args.logOnly) {
            // Only add the listener that will report the list of tests when running in logOnly
            // mode.
            builder.addRunListener(getInstrumentationResultPrinter());
        } else if (args.suiteAssignment) {
            builder.addRunListener(new SuiteAssignmentPrinter());
        } else {
            builder.addRunListener(new LogRunListener());
            if (mOrchestratorListener != null) {
                builder.addRunListener(mOrchestratorListener);
            } else {
                builder.addRunListener(getInstrumentationResultPrinter());
            }
            builder.addRunListener(
                    new ActivityFinisherRunListener(
                            this,
                            new MonitoringInstrumentation.ActivityFinisher(),
                            new Runnable() {
                                // Yes, this is terrible and weird but avoids adding a new public API
                                // outside the internal package.
                                @Override
                                public void run() {
                                    waitForActivitiesToComplete();
                                }
                            }));
            addDelayListener(args, builder);
            addCoverageListener(args, builder);
        }
        addListenersFromArg(args, builder);
    }

    private void addListenersNewOrder(RunnerArgs args, TestExecutor.Builder builder) {
        // User defined listeners go first, to guarantee running before InstrumentationResultPrinter
        // and ActivityFinisherRunListener. Delay and Coverage Listener are also moved before for the
        // same reason.
        addListenersFromArg(args, builder);
        if (args.logOnly) {
            // Only add the listener that will report the list of tests when running in logOnly
            // mode.
            builder.addRunListener(getInstrumentationResultPrinter());
        } else if (args.suiteAssignment) {
            builder.addRunListener(new SuiteAssignmentPrinter());
        } else {
            builder.addRunListener(new LogRunListener());
            addDelayListener(args, builder);
            addCoverageListener(args, builder);
            if (mOrchestratorListener != null) {
                builder.addRunListener(mOrchestratorListener);
            } else {
                builder.addRunListener(getInstrumentationResultPrinter());
            }
            builder.addRunListener(
                    new ActivityFinisherRunListener(
                            this,
                            new MonitoringInstrumentation.ActivityFinisher(),
                            new Runnable() {
                                // Yes, this is terrible and weird but avoids adding a new public API
                                // outside the internal package.
                                @Override
                                public void run() {
                                    waitForActivitiesToComplete();
                                }
                            }));
        }
    }

    private void addScreenCaptureProcessors(RunnerArgs args) {
        Screenshot.addScreenCaptureProcessors(
                new HashSet<ScreenCaptureProcessor>(args.screenCaptureProcessors));
    }

    private void addCoverageListener(RunnerArgs args, TestExecutor.Builder builder) {
        if (args.codeCoverage) {
            builder.addRunListener(new CoverageListener(args.codeCoveragePath));
        }
    }

    /** Sets up listener to inject a delay between each test, if specified. */
    private void addDelayListener(RunnerArgs args, TestExecutor.Builder builder) {
        if (args.delayInMillis > 0) {
            builder.addRunListener(new DelayInjector(args.delayInMillis));
        } else if (args.logOnly && Build.VERSION.SDK_INT < 16) {
            // On older platforms, collecting tests can fail for large volume of tests.
            // Insert a small delay between each test to prevent this
            builder.addRunListener(new DelayInjector(15 /* msec */));
        }
    }

    private void addListenersFromArg(RunnerArgs args, TestExecutor.Builder builder) {
        for (RunListener listener : args.listeners) {
            builder.addRunListener(listener);
        }
    }

    @Override
    public boolean onException(Object obj, Throwable e) {
        InstrumentationResultPrinter instResultPrinter = getInstrumentationResultPrinter();
        if (instResultPrinter != null) {
            // report better error message back to Instrumentation results.
            instResultPrinter.reportProcessCrash(e);
        }
        return super.onException(obj, e);
    }

    /** Builds a {@link Request} based on given input arguments. */
    @VisibleForTesting
    Request buildRequest(RunnerArgs runnerArgs, Bundle bundleArgs) {

        TestRequestBuilder builder = createTestRequestBuilder(this, bundleArgs);
        builder.addPathsToScan(runnerArgs.classpathToScan);
        if (runnerArgs.classpathToScan.isEmpty()) {
            // Only scan for tests for current apk aka testContext
            // Note that this represents a change from InstrumentationTestRunner where
            // getTargetContext().getPackageCodePath() aka app under test was also scanned
            // Only add the package classpath when no custom classpath is provided in order to
            // avoid duplicate class issues.
            builder.addPathToScan(getContext().getPackageCodePath());
        }
        builder.addFromRunnerArgs(runnerArgs);

        registerUserTracker();

        return builder.build();
    }

    private void registerUserTracker() {
        Context targetContext = getTargetContext();
        if (targetContext != null) {
            mUsageTrackerFacilitator.registerUsageTracker(
                    new AnalyticsBasedUsageTracker.Builder(targetContext).buildIfPossible());
        }
    }

    /** Factory method for {@link TestRequestBuilder}. */
    TestRequestBuilder createTestRequestBuilder(Instrumentation instr, Bundle arguments) {
        return new TestRequestBuilder(instr, arguments);
    }
}