TestRail extension which reports test run results to TestRail and creates test run for each configuration related to some test plan.

TestRailReportExtension.java

public class TestRailReportExtension implements TestWatcher, BeforeAllCallback {

    private static boolean started = false;
    private static final String TESTRAIL_REPORT = "TEST_RAIL";
    private static List<Result> results = new CopyOnWriteArrayList<>();

    @Override
    public void testDisabled(ExtensionContext extensionContext, Optional<String> optional) {
        addResult(extensionContext, TestRailStatus.DISABLED);
    }

    @Override
    public void testSuccessful(ExtensionContext extensionContext) {
        addResult(extensionContext, TestRailStatus.PASSED);
    }

    @Override
    public void testAborted(ExtensionContext extensionContext, Throwable throwable) {
        addResult(extensionContext, TestRailStatus.RETEST);
    }

    @Override
    public void testFailed(ExtensionContext extensionContext, Throwable throwable) {
        addResult(extensionContext, TestRailStatus.FAILED);
    }

    @Override
    public void beforeAll(ExtensionContext extensionContext) {
        if (!started) {
            getStore(extensionContext).put(TESTRAIL_REPORT, new CloseableOnlyOnceResource());
            started = true;
        }

    }

    private static class CloseableOnlyOnceResource implements
            ExtensionContext.Store.CloseableResource {
        @Override
        public void close() {
            //After all tests run hook.
            //Any additional desired action goes here
            reportResults();
        }
    }

    private void addResult(ExtensionContext extensionContext, TestRailStatus status) {

        if (extensionContext.getElement().isPresent() && extensionContext.getElement().get().isAnnotationPresent(
                TmsLink.class)) {
            TmsLink element = extensionContext.getElement().get().getAnnotation(TmsLink.class);
            String exceptionMessage = extensionContext.getExecutionException().map(Throwable::getMessage).orElse(null);
            Result result = new Result().setComment(exceptionMessage).setTestId(Integer.parseInt(element.value())).setStatusId(status.getId())
                    .setCaseId(Integer.valueOf(element.value()));
            addResult(result);
        }
    }

    private ExtensionContext.Store getStore(ExtensionContext context) {
        return context.getRoot().getStore(ExtensionContext.Namespace.GLOBAL);
    }

    private static void addResult(Result result) {
        results.add(result);
    }

    private static void reportResults() {
        if (testRailEnabled()) { // replace with your own method of reading data from prom properties file
          final String url = "http://testrail.example.com";
            final String userId = "email@example.com";
            final String pwd = "1q2w3e";
            final String projectId = "18";
            final Integer testSuiteId = 1234; // Suite ID should be unique per project
            final Integer planId = 1111; // Test plan reflects current version which is tested
            final Integer milestone = 666; // Milestone is set per each project and should reflect release version(e.g. 1.0, 666)
            final String browserName = "browserName";
            final String description = "Test Run description. It can contain links to build or some other useful information";
            final String name = "Test run name";
            TestRail testRail = TestRail.builder(url, userId, pwd).build();
   
            // First, we need to create list of configuration ID's. They are taken from TestRail! See http://docs.gurock.com/testrail-api2/reference-configs
            List<Integer> configIds = new ArrayList<>();
            List<Plan.Entry.Run> runList = new ArrayList<>();
              List<Integer> emptyCasesArray = new ArrayList<>();
            
            // Here we create 1 run per 1 configuration. TestRail API does not allow to add new runs to existing plan entry. 
           switch (browserName.toLowerCase()) {
                case "chrome":
                    configIds.add(258);
                    Plan.Entry.Run chromeRun = (Plan.Entry.Run) new Plan.Entry.Run().setConfigIds(Collections.singletonList(258)).setCaseIds(emptyCasesArray);
                    runList.add(chromeRun);
                    break;
                case "firefox":
                    configIds.add(259);
                    Plan.Entry.Run firefoxRun = (Plan.Entry.Run) new Plan.Entry.Run().setConfigIds(Collections.singletonList(259)).setCaseIds(emptyCasesArray);
                    runList.add(firefoxRun);
                    break;
                case "ie":
                case "internet explorer":
                    configIds.add(257);
                    Plan.Entry.Run ieRun = (Plan.Entry.Run) new Plan.Entry.Run().setConfigIds(Collections.singletonList(257)).setCaseIds(emptyCasesArray);
                    runList.add(ieRun);
                    break;
                case "api":
                    configIds.add(273);
                    Plan.Entry.Run apiRun = (Plan.Entry.Run) new Plan.Entry.Run().setConfigIds(Collections.singletonList(273)).setCaseIds(emptyCasesArray);
                    runList.add(apiRun);
                    break;
                case "unknown":
                    configIds.add(274);
                    Plan.Entry.Run unknownRun = (Plan.Entry.Run) new Plan.Entry.Run().setConfigIds(Collections.singletonList(274)).setCaseIds(emptyCasesArray);
                    runList.add(unknownRun);
                    break;
            }

            List<ResultField> customResultFields = testRail.resultFields().list().execute();
            List<CaseField> caseFields = testRail.caseFields().list().execute();

            for (Result result : results) {
                try {
                    testRail.cases().get(result.getCaseId(), caseFields).execute();
                } catch (Exception e) {
                    results.remove(result);
                    System.out.println("\n Test Case with ID " + result.getCaseId() + " does not exist in TestRail! \n");
                }
            }

            Plan.Entry planEntry;
            List<Plan.Entry> entries = testRail.plans().get(planId).execute().getEntries();
            List<Plan.Entry> existingEntries = entries.stream().filter(p -> p.getName().equals(name)).collect(Collectors.toList());

            if (existingEntries
                    .stream()
                    .filter(entry -> entry.getRuns().get(0).getConfigIds().get(0).equals(configIds.get(0)))
                    .map(Plan.Entry::getName)
                    .anyMatch(name::equals)) {
                Plan.Entry existingEntry = testRail.plans().get(planId).execute().getEntries()
                        .stream()
                        .filter(entry -> entry.getName().equals(name))
                        .findFirst()
                        .orElse(null);
                Objects.requireNonNull(existingEntry).setRuns(runList).setCaseIds(results
                        .stream()
                        .map(Result::getCaseId)
                        .collect(Collectors.toList()));

                planEntry = testRail.plans().updateEntry(planId, existingEntry).execute();
            } else {
                Plan.Entry newPlanEntry = new Plan.Entry();
                newPlanEntry
                        .setName(name)
                        .setRuns(runList)
                        .setConfigIds(configIds)
                        .setSuiteId(testSuiteId)
                        .setIncludeAll(true);
                planEntry = testRail.plans().addEntry(planId, newPlanEntry).execute();
            }

            // Here we modify entry and update it
            Plan.Entry finalPlanEntry = planEntry;
            finalPlanEntry
                    .setIncludeAll(false)
                    .setConfigIds(configIds)
                    .setRuns(runList)
                    .setDescription(description)
                    .setCaseIds(results
                            .stream()
                            .map(Result::getCaseId)
                            .collect(Collectors.toList()));

            Plan.Entry.Run run = Objects.requireNonNull(testRail.plans().get(planId).execute().getEntries()
                    .stream()
                    .filter(entry -> entry.getName().equals(finalPlanEntry.getName()))
                    .filter(br -> br.getRuns().get(0).getConfigIds().get(0).equals(configIds.get(0)))
                    .findFirst()
                    .orElse(null))
                    .getRuns()
                    .stream()
                    .filter(run1 -> StringUtils.containsIgnoreCase(run1.getConfig(), browserName))
                    .findFirst()
                    .orElse(null);

            testRail.results().addForCases(Objects.requireNonNull(run).getId(), results, customResultFields).execute();
            testRail.plans().updateEntry(planId, finalPlanEntry).execute();

        } else System.out.println("TestRail reporting is disabled... Skipping");
    }

    private enum TestRailStatus {
        PASSED(1),
        BLOCKED(2),
        UNTESTED(3),
        RETEST(4),
        FAILED(5),
        DISABLED(6);

        private int id;

        public int getId() {
            return id;
        }

        TestRailStatus(int id) {
            this.id = id;
        }
    }
}